├── .coveragerc ├── .gitignore ├── .travis.yml ├── CONTRIBUTING.md ├── LICENSE ├── MANIFEST.in ├── README.rst ├── examples ├── logging.conf ├── pubsub_logging_example.py └── sync.conf ├── pubsub-integration-test.json.enc ├── pubsub_logging ├── __init__.py ├── async_handler.py ├── errors.py ├── pubsub_handler.py └── utils.py ├── requirements.txt ├── setup.py ├── tests └── pubsub_logging_test.py └── tox.ini /.coveragerc: -------------------------------------------------------------------------------- 1 | [report] 2 | exclude_lines = 3 | pragma: NO COVER 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Copied from 2 | # https://github.com/github/gitignore/blob/master/Python.gitignore 3 | 4 | # Byte-compiled / optimized / DLL files 5 | __pycache__/ 6 | *.py[cod] 7 | 8 | # C extensions 9 | *.so 10 | 11 | # Distribution / packaging 12 | .Python 13 | env/ 14 | build/ 15 | develop-eggs/ 16 | dist/ 17 | eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # Installer logs 28 | pip-log.txt 29 | pip-delete-this-directory.txt 30 | 31 | # Unit test / coverage reports 32 | htmlcov/ 33 | .tox/ 34 | .coverage 35 | .cache 36 | nosetests.xml 37 | coverage.xml 38 | 39 | # Translations 40 | *.mo 41 | *.pot 42 | 43 | # Django stuff: 44 | *.log 45 | 46 | # Sphinx documentation 47 | docs/_build/ 48 | 49 | # pip env 50 | bin/ 51 | include/ 52 | lib/ 53 | local/ 54 | man/ 55 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: python 3 | install: 4 | - pip install tox 5 | script: tox 6 | env: 7 | - GOOGLE_APPLICATION_CREDENTIALS=pubsub-integration-test.json 8 | before_install: 9 | - openssl aes-256-cbc -K $encrypted_360b288b17ae_key 10 | -iv $encrypted_360b288b17ae_iv 11 | -in pubsub-integration-test.json.enc 12 | -out pubsub-integration-test.json 13 | -d 14 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Contributor License Agreements 2 | ------------------------------ 3 | 4 | Before we can accept your pull requests you'll need to sign a Contributor License Agreement (CLA): 5 | 6 | * If you are an individual writing original source code and you own the intellectual property, then you'll need to sign an [individual CLA](https://developers.google.com/open-source/cla/individual). 7 | 8 | * If you work for a company that wants to allow you to contribute your work, then you'll need to sign a [corporate CLA](https://developers.google.com/open-source/cla/corporate>). 9 | 10 | You can sign these electronically (just scroll to the bottom). After that, we'll be able to accept your pull requests. 11 | 12 | Travis test 13 | =========== 14 | 15 | Unfortunately, the tests depend on the travis secret variables for 16 | decrypting the json key, and the secret variables are not available 17 | for pull requests from other repos. 18 | 19 | So if you want to create a pull request, consider creating a pull 20 | request to the `contribution` branch first. We'll review the pull 21 | requests then merge to the `contribution`. After that we'll be able to 22 | run the travis tests with another pull request from `contribution` to 23 | `master` branch. 24 | 25 | Run the tests 26 | ============= 27 | 28 | Before sending a pull request, consider running the tests. To run the 29 | tests, follow the instructions below. 30 | 31 | * Install tox 32 | 33 | $ pip install tox 34 | 35 | * Create a Cloud Project if you don't have it. 36 | * Create a service account if you don't have it. 37 | * Download a JSON key of that service account. 38 | * Set environment variable 39 | 40 | * GOOGLE_APPLICATION_CREDENTIALS: the file name of the JSON key 41 | * PUBSUB_LOGGING_TEST_PROJECT: your project id 42 | 43 | * Run tox 44 | 45 | Note: Don't submit your JSON key!! 46 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst 2 | global-exclude *.pyc 3 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | cloud-pubsub-logging-python 2 | =========================== 3 | 4 | Logging handlers for publishing the logs to Cloud Pub/Sub. 5 | 6 | |pypi| |build| |coverage| 7 | 8 | You can use the `pubsub_logging.PubsubHandler` or `pubsub_logging.AsyncPubsubHandler` to publish the logs to `Cloud Pub/Sub`_. You can use this module with `the standard Python logging module`_. It's recommended that you use `AsyncPubsubHandler`. `PubsubHandler` exists only for backward compatibility. 9 | 10 | .. _Cloud Pub/Sub: https://cloud.google.com/pubsub/docs/ 11 | .. _the standard Python logging module: https://docs.python.org/2/library/logging.html 12 | 13 | Supported version 14 | ----------------- 15 | 16 | Python 2.7 and Python 3.4 are supported. 17 | 18 | Installation 19 | ------------ 20 | 21 | :: 22 | 23 | $ pip install pubsub-logging 24 | 25 | How to use 26 | ---------- 27 | 28 | Here is an example configuration file. 29 | 30 | .. code:: ini 31 | 32 | [loggers] 33 | keys=root 34 | 35 | [handlers] 36 | keys=asyncPubsubHandler 37 | 38 | [formatters] 39 | keys=simpleFormatter 40 | 41 | [logger_root] 42 | level=NOTSET 43 | handlers=asyncPubsubHandler 44 | 45 | [handler_asyncPubsubHandler] 46 | class=pubsub_logging.AsyncPubsubHandler 47 | level=DEBUG 48 | formatter=simpleFormatter 49 | # Replace {project-name} and {topic-name} with actual ones. 50 | # The second argument indicates number of workers. 51 | args=('projects/{project-name}/topics/{topic-name}', 10) 52 | 53 | [formatter_simpleFormatter] 54 | format=%(asctime)s - %(name)s - %(levelname)s - %(message)s 55 | 56 | How to use this config file. 57 | 58 | .. code:: python 59 | 60 | logging.config.fileConfig(os.path.join('examples', 'logging.conf')) 61 | logger = logging.getLogger('root') 62 | logger.info('My first message.') 63 | 64 | Here is a dynamic usage example. 65 | 66 | .. code:: python 67 | 68 | pubsub_handler = AsyncPubsubHandler(topic=topic) 69 | pubsub_handler.setFormatter( 70 | logging.Formatter( 71 | '%(asctime)s - %(name)s - %(levelname)s - %(message)s')) 72 | 73 | logger = logging.getLogger('root') 74 | logger.setLevel(logging.DEBUG) 75 | logger.addHandler(pubsub_handler) 76 | logger.info('My first message.') 77 | 78 | The logs are kept in a buffer first, then moved to the process safe queue, and then the background child processes automatically pick up and send them to Cloud Pub/Sub. The flush call blocks until all of the logs are sent to Cloud Pub/Sub. 79 | 80 | Authentication 81 | -------------- 82 | 83 | The module uses the `Application Default Credentials`_. You can configure the authentication as follows. 84 | 85 | .. _Application Default Credentials: https://developers.google.com/accounts/docs/application-default-credentials 86 | 87 | Authentication on App Engine 88 | ---------------------------- 89 | 90 | It should work out of the box. If you're getting an authorization error, please make sure that your App Engine service account has an `Editor` or greater permission on your Cloud project. 91 | 92 | Authentication on Google Compute Engine 93 | --------------------------------------- 94 | 95 | When creating a new instance, please add the Cloud Pub/Sub scope `https://www.googleapis.com/auth/pubsub` to the service account of the instance. 96 | 97 | Authentication anywhere else 98 | ---------------------------- 99 | 100 | As `the documentation suggests`_, create a new service account and download its JSON key file, then set the environment variable `GOOGLE_APPLICATION_CREDENTIALS` pointing to the JSON key file. Please note that this service account must have `Editor` or greater permissions on your Cloud project. 101 | 102 | .. _the documentation suggests: https://developers.google.com/accounts/docs/application-default-credentials#whentouse 103 | 104 | 105 | .. |build| image:: https://travis-ci.org/GoogleCloudPlatform/cloud-pubsub-logging-python.svg?branch=master 106 | :target: https://travis-ci.org/GoogleCloudPlatform/cloud-pubsub-logging-python 107 | .. |pypi| image:: https://img.shields.io/pypi/v/pubsub-logging.svg 108 | :target: https://pypi.python.org/pypi/pubsub-logging 109 | .. |coverage| image:: https://coveralls.io/repos/GoogleCloudPlatform/cloud-pubsub-logging-python/badge.png?branch=master 110 | :target: https://coveralls.io/r/GoogleCloudPlatform/cloud-pubsub-logging-python?branch=master 111 | -------------------------------------------------------------------------------- /examples/logging.conf: -------------------------------------------------------------------------------- 1 | [loggers] 2 | keys=root 3 | 4 | [handlers] 5 | keys=asyncPubsubHandler 6 | 7 | [formatters] 8 | keys=simpleFormatter 9 | 10 | [logger_root] 11 | level=NOTSET 12 | handlers=asyncPubsubHandler 13 | 14 | [handler_asyncPubsubHandler] 15 | class=pubsub_logging.AsyncPubsubHandler 16 | level=DEBUG 17 | formatter=simpleFormatter 18 | # replace {project-name} and {topic-name} with actual ones 19 | args=('projects/{project-name}/topics/{topic-name}', 10) 20 | 21 | [formatter_simpleFormatter] 22 | format=%(asctime)s - %(name)s - %(levelname)s - %(message)s 23 | -------------------------------------------------------------------------------- /examples/pubsub_logging_example.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2015 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | """A example script for Pub/Sub logging handlers.""" 17 | 18 | 19 | from __future__ import print_function 20 | 21 | import argparse 22 | import logging 23 | import logging.config 24 | import logging.handlers 25 | import time 26 | 27 | from pubsub_logging import AsyncPubsubHandler 28 | from pubsub_logging import PubsubHandler 29 | from pubsub_logging import utils 30 | 31 | 32 | def benchmark(f): 33 | """A simple decorator for timing output for publish_body.""" 34 | def inner(*args, **kwargs): 35 | before = time.time() 36 | ret = f(*args, **kwargs) 37 | print('Took %f secs for sending %d messages.' % 38 | (time.time() - before, len(args[1]['messages']))) 39 | return ret 40 | return inner 41 | 42 | 43 | def main(): 44 | parser = argparse.ArgumentParser(description='Testing AsyncPubsubHandler') 45 | parser.add_argument('-m', '--num_messages', metavar='N', type=int, 46 | default=100000, help='number of messages') 47 | parser.add_argument('-w', '--num_workers', metavar='N', type=int, 48 | default=20, help='number of workers') 49 | parser.add_argument('--async', dest='async', action='store_true') 50 | parser.add_argument('--no-async', dest='async', action='store_false') 51 | parser.set_defaults(async=True) 52 | parser.add_argument('--bench', dest='bench', action='store_true') 53 | parser.add_argument('--no-bench', dest='bench', action='store_false') 54 | parser.set_defaults(bench=False) 55 | parser.add_argument('topic', default='') 56 | args = parser.parse_args() 57 | num = args.num_messages 58 | workers = args.num_workers 59 | topic = args.topic 60 | publish_body = utils.publish_body 61 | if args.bench: 62 | publish_body = benchmark(publish_body) 63 | if args.async: 64 | print('Using AsyncPubsubHandler.\n') 65 | pubsub_handler = AsyncPubsubHandler(topic, workers, 66 | publish_body=publish_body) 67 | else: 68 | print('Using PubsubHandler.\n') 69 | pubsub_handler = PubsubHandler(topic, publish_body=publish_body) 70 | pubsub_handler.setFormatter( 71 | logging.Formatter( 72 | '%(asctime)s - %(name)s - %(levelname)s - %(message)s')) 73 | logger = logging.getLogger('root') 74 | logger.setLevel(logging.DEBUG) 75 | logger.addHandler(pubsub_handler) 76 | 77 | before = time.time() 78 | for i in range(num): 79 | logger.info('log message %03d.', i) 80 | elapsed = time.time() - before 81 | print('Took %f secs for buffering %d messages: %f mps.\n' % 82 | (elapsed, num, num/elapsed)) 83 | pubsub_handler.flush() 84 | elapsed = time.time() - before 85 | print('Took %f secs for sending %d messages: %f mps.\n' % 86 | (elapsed, num, num/elapsed)) 87 | 88 | 89 | if __name__ == '__main__': 90 | main() 91 | -------------------------------------------------------------------------------- /examples/sync.conf: -------------------------------------------------------------------------------- 1 | [loggers] 2 | keys=root 3 | 4 | [handlers] 5 | keys=pubsubHandler 6 | 7 | [formatters] 8 | keys=simpleFormatter 9 | 10 | [logger_root] 11 | level=NOTSET 12 | handlers=pubsubHandler 13 | 14 | [handler_pubsubHandler] 15 | class=pubsub_logging.PubsubHandler 16 | level=DEBUG 17 | formatter=simpleFormatter 18 | # replace {project-name} and {topic-name} with actual ones 19 | args=('projects/{project-name}/topics/{topic-name}', 1000) 20 | 21 | [formatter_simpleFormatter] 22 | format=%(asctime)s - %(name)s - %(levelname)s - %(message)s 23 | -------------------------------------------------------------------------------- /pubsub-integration-test.json.enc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/cloud-pubsub-logging-python/a115bc8d772de06cae0f9c514e6a8b41f9fb2919/pubsub-integration-test.json.enc -------------------------------------------------------------------------------- /pubsub_logging/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2015 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | """Python logging handler implementations for Cloud Pub/Sub. 17 | 18 | Modules in this package send the logs to Cloud Pub/Sub[1]. The 19 | pubsub_handler module has a sync version of the handler. 20 | 21 | [1]: https://cloud.google.com/pubsub/docs 22 | 23 | """ 24 | 25 | import logging 26 | import logging.handlers 27 | import os 28 | import sys 29 | 30 | from .pubsub_handler import PubsubHandler # flake8: noqa 31 | from .async_handler import AsyncPubsubHandler # flake8: noqa 32 | 33 | __version__ = '0.2.1' 34 | -------------------------------------------------------------------------------- /pubsub_logging/async_handler.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2015 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | """Python logging handler implementation for Cloud Pub/Sub. 17 | 18 | This module provides a logging.Handler implementation which sends the 19 | logs to Cloud Pub/Sub[1] asynchronously. The logs are kept in an 20 | internal queue and child workers will pick them up and send them in 21 | background. 22 | 23 | [1]: https://cloud.google.com/pubsub/docs 24 | 25 | """ 26 | 27 | import logging 28 | import multiprocessing as mp 29 | 30 | # For Python 2 and Python 3 compatibility. 31 | try: 32 | from queue import Empty 33 | except ImportError: 34 | from Queue import Empty 35 | 36 | from pubsub_logging import errors 37 | 38 | from pubsub_logging.utils import check_topic 39 | from pubsub_logging.utils import compat_urlsafe_b64encode 40 | from pubsub_logging.utils import get_pubsub_client 41 | from pubsub_logging.utils import publish_body 42 | 43 | 44 | BATCH_SIZE = 1000 45 | DEFAULT_POOL_SIZE = 1 46 | DEFAULT_RETRY_COUNT = 10 47 | 48 | 49 | def send_loop(client, q, topic, retry, logger, format_func, 50 | publish_body): # pragma: NO COVER 51 | """Process loop for indefinitely sending logs to Cloud Pub/Sub. 52 | 53 | Args: 54 | client: Pub/Sub client. If it's None, a new client will be created. 55 | q: mp.JoinableQueue instance to get the message from. 56 | topic: Cloud Pub/Sub topic name to send the logs. 57 | retry: How many times to retry upon Cloud Pub/Sub API failure. 58 | logger: A logger for informing failures within this function. 59 | format_func: A callable for formatting the logs. 60 | publish_body: A callable for sending the logs. 61 | """ 62 | if client is None: 63 | client = get_pubsub_client() 64 | while True: 65 | try: 66 | logs = q.get() 67 | except Empty: 68 | continue 69 | try: 70 | body = {'messages': 71 | [{'data': compat_urlsafe_b64encode(format_func(r))} 72 | for r in logs]} 73 | publish_body(client, body, topic, retry) 74 | except errors.RecoverableError as e: 75 | # Records the exception and puts the logs back to the deque 76 | # and prints the exception to stderr. 77 | q.put(logs) 78 | logger.exception(e) 79 | except Exception as e: 80 | logger.exception(e) 81 | logger.warn('There was a non recoverable error, exiting.') 82 | return 83 | q.task_done() 84 | 85 | 86 | class AsyncPubsubHandler(logging.Handler): 87 | """A logging handler to publish logs to Cloud Pub/Sub in background.""" 88 | def __init__(self, topic, worker_num=DEFAULT_POOL_SIZE, 89 | retry=DEFAULT_RETRY_COUNT, client=None, 90 | publish_body=publish_body, stderr_logger=None): 91 | """The constructor of the handler. 92 | 93 | Args: 94 | topic: Cloud Pub/Sub topic name to send the logs. 95 | worker_num: The number of workers, defaults to 1. 96 | retry: How many times to retry upon Cloud Pub/Sub API failure, 97 | defaults to 5. 98 | client: An optional Cloud Pub/Sub client to use. If not set, one is 99 | built automatically, defaults to None. 100 | publish_body: A callable for publishing the Pub/Sub message, 101 | just for testing and benchmarking purposes. 102 | stderr_logger: A logger for informing failures with this 103 | logger, defaults to None and if not specified, a last 104 | resort logger will be used. 105 | """ 106 | super(AsyncPubsubHandler, self).__init__() 107 | self._q = mp.JoinableQueue() 108 | self._batch_size = BATCH_SIZE 109 | self._buf = [] 110 | if not check_topic(client or get_pubsub_client(), topic, retry): 111 | raise EnvironmentError( 112 | 'Failed to confirm the existence of the topic "%s".' % topic) 113 | if not stderr_logger: 114 | stderr_logger = logging.Logger('last_resort') 115 | stderr_logger.addHandler(logging.StreamHandler()) 116 | for _ in range(worker_num): 117 | p = mp.Process(target=send_loop, 118 | args=(client, self._q, topic, retry, stderr_logger, 119 | self.format, publish_body)) 120 | p.daemon = True 121 | p.start() 122 | 123 | def emit(self, record): 124 | """Puts the record to the internal queue.""" 125 | self._buf.append(record) 126 | if len(self._buf) == self._batch_size: 127 | self._q.put(self._buf) 128 | self._buf = [] 129 | 130 | def flush(self): 131 | """Blocks until the queue becomes empty.""" 132 | with self.lock: 133 | if self._buf: 134 | self._q.put(self._buf) 135 | self._buf = [] 136 | self._q.join() 137 | 138 | def close(self): 139 | """Joins the child processes and call the superclass's close.""" 140 | with self.lock: 141 | self.flush() 142 | super(AsyncPubsubHandler, self).close() 143 | -------------------------------------------------------------------------------- /pubsub_logging/errors.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2015 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | """Errors for the Python logging handlers.""" 17 | 18 | 19 | class RecoverableError(Exception): 20 | """A special error case we'll ignore.""" 21 | pass 22 | -------------------------------------------------------------------------------- /pubsub_logging/pubsub_handler.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2015 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | """Python logging handler implementation for Cloud Pub/Sub. 17 | 18 | This module has logging.handlers.BufferingHandler implementation which 19 | sends the logs to Cloud Pub/Sub[1]. The logs are kept in an internal 20 | buffer by default. If the buffer becomes full (capacity), or the given 21 | log record has a level higher than flush_level, the handler will 22 | transmit the buffered logs to Cloud Pub/Sub. 23 | 24 | By default, this log handler will try to keep the logs as much as 25 | possible, even upon intermittent failure on Cloud Pub/Sub API. If 26 | you're concerned about indifinitely growing buffer size in such cases, 27 | you should set buf_hard_limit, then the buffer will be cut off at the 28 | specified size. In that case, you may want to consider having another 29 | backup logging handler backed by the local disk or something. 30 | 31 | [1]: https://cloud.google.com/pubsub/docs 32 | 33 | """ 34 | 35 | import logging 36 | 37 | from pubsub_logging.errors import RecoverableError 38 | from pubsub_logging.utils import check_topic 39 | from pubsub_logging.utils import compat_urlsafe_b64encode 40 | from pubsub_logging.utils import get_pubsub_client 41 | from pubsub_logging.utils import publish_body 42 | 43 | 44 | DEFAULT_BATCH_NUM = 1000 45 | DEFAULT_RETRY_COUNT = 5 46 | MAX_BATCH_SIZE = 1000 47 | 48 | 49 | class PubsubHandler(logging.handlers.BufferingHandler): 50 | """A logging handler to publish log messages to Cloud Pub/Sub.""" 51 | def __init__(self, topic, capacity=DEFAULT_BATCH_NUM, 52 | retry=DEFAULT_RETRY_COUNT, flush_level=logging.CRITICAL, 53 | buf_hard_limit=-1, client=None, publish_body=publish_body): 54 | """The constructor of the handler. 55 | 56 | Args: 57 | topic: Cloud Pub/Sub topic name to send the logs. 58 | capacity: The maximum buffer size, defaults to 1000. 59 | retry: How many times to retry upon Cloud Pub/Sub API failure, 60 | defaults to 5. 61 | flush_level: Minimum log level that, when seen, will flush the logs, 62 | defaults to logging.CRITICAL. 63 | buf_hard_limit: Maximum buffer size to hold the logs, defaults to -1 64 | which means unlimited size. 65 | client: An optional Cloud Pub/Sub client to use. If not set, one is 66 | built automatically, defaults to None. 67 | publish_body: A callable for publishing the Pub/Sub message, 68 | just for testing and benchmarking purposes. 69 | """ 70 | super(PubsubHandler, self).__init__(capacity) 71 | self._topic = topic 72 | self._retry = retry 73 | self._flush_level = flush_level 74 | self._buf_hard_limit = buf_hard_limit 75 | self._publish_body = publish_body 76 | if client: 77 | self._client = client 78 | else: 79 | self._client = get_pubsub_client() 80 | if not check_topic(self._client, topic, retry): 81 | raise EnvironmentError( 82 | 'Failed to confirm the existence of the topic "%s".' % topic) 83 | 84 | def flush(self): 85 | """Transmits the buffered logs to Cloud Pub/Sub.""" 86 | self.acquire() 87 | try: 88 | while self.buffer: 89 | body = {'messages': 90 | [{'data': compat_urlsafe_b64encode(self.format(r))} 91 | for r in self.buffer[:MAX_BATCH_SIZE]]} 92 | self._publish_body(self._client, body, self._topic, 93 | self._retry) 94 | self.buffer = self.buffer[MAX_BATCH_SIZE:] 95 | except RecoverableError: 96 | # Cloud Pub/Sub API didn't receive the logs, most 97 | # likely because of intermittent errors. This handler 98 | # ignores this case and keep the logs in its buffer in 99 | # a hope of future success. 100 | pass 101 | finally: 102 | # Cut off the buffer at the _buf_hard_limit. 103 | # The default value of -1 means unlimited buffer. 104 | if self._buf_hard_limit != -1: 105 | self.buffer = self.buffer[:self._buf_hard_limit] 106 | self.release() 107 | 108 | def shouldFlush(self, record): 109 | """Should the handler flush its buffer? 110 | 111 | Returns true if the buffer is up to capacity, or the given 112 | record has a level greater or equal to flush_level. 113 | """ 114 | return (record.levelno >= self._flush_level 115 | or (len(self.buffer) >= self.capacity)) 116 | -------------------------------------------------------------------------------- /pubsub_logging/utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2015 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | """Utilities for the Python logging handlers.""" 17 | 18 | 19 | import base64 20 | import sys 21 | import threading 22 | import traceback 23 | 24 | from googleapiclient import discovery 25 | from googleapiclient import errors 26 | import httplib2 27 | from oauth2client.client import GoogleCredentials 28 | 29 | from pubsub_logging.errors import RecoverableError 30 | 31 | 32 | PUBSUB_SCOPES = ["https://www.googleapis.com/auth/pubsub"] 33 | 34 | clients = threading.local() 35 | 36 | 37 | def compat_urlsafe_b64encode(v): 38 | """A urlsafe ba64encode which is compatible with Python 2 and 3. 39 | 40 | Args: 41 | v: A string to encode. 42 | Returns: 43 | The encoded string. 44 | """ 45 | if sys.version_info[0] >= 3: # pragma: NO COVER 46 | return base64.urlsafe_b64encode(v.encode('UTF-8')).decode('ascii') 47 | else: 48 | return base64.urlsafe_b64encode(v) 49 | 50 | 51 | def get_pubsub_client(http=None, credentials=None): 52 | """Return a Pub/Sub client. 53 | 54 | Args: 55 | http: httplib2.Http instance. Defaults to None. 56 | Returns: 57 | Cloud Pub/Sub client. 58 | """ 59 | if not credentials: 60 | credentials = GoogleCredentials.get_application_default() 61 | if credentials.create_scoped_required(): 62 | credentials = credentials.create_scoped(PUBSUB_SCOPES) 63 | if not http: 64 | http = httplib2.Http() 65 | credentials.authorize(http=http) 66 | return discovery.build('pubsub', 'v1beta2', http=http) 67 | 68 | 69 | def check_topic(client, topic, retry=3): 70 | """Checks the existance of a topic of the given name. 71 | 72 | Args: 73 | client: Cloud Pub/Sub client. 74 | topic: topic name that we publish the records to. 75 | retry: number of retry upon intermittent failures, defaults to 3. 76 | 77 | Returns: 78 | True when it confirmed that the topic exists, and False otherwise. 79 | """ 80 | try: 81 | client.projects().topics().get(topic=topic).execute(num_retries=retry) 82 | return True 83 | except Exception: 84 | traceback.print_exc(file=sys.stderr) 85 | return False 86 | 87 | 88 | def publish_body(client, body, topic, retry): 89 | """Publishes the specified body to Cloud Pub/Sub. 90 | 91 | Args: 92 | client: Cloud Pub/Sub client. 93 | body: Post body for Pub/Sub publish call. 94 | topic: topic name that we publish the records to. 95 | retry: number of retry upon intermittent failures. 96 | 97 | Raises: 98 | errors.HttpError When the Cloud Pub/Sub API call fails with 99 | unrecoverable reasons. 100 | RecoverableError When the Cloud Pub/Sub API call fails with 101 | intermittent errors. 102 | """ 103 | try: 104 | client.projects().topics().publish( 105 | topic=topic, body=body).execute(num_retries=retry) 106 | except errors.HttpError as e: 107 | if e.resp.status >= 400 and e.resp.status < 500: 108 | # Publishing failed for some non-recoverable reason. For 109 | # example, perhaps the service account doesn't have a 110 | # permission to publish to the specified topic, or the topic 111 | # simply doesn't exist. 112 | raise 113 | else: 114 | # Treat this as a recoverable error. 115 | raise RecoverableError() 116 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | google-api-python-client 2 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from __future__ import with_statement 2 | 3 | try: 4 | from setuptools import setup 5 | except ImportError: 6 | from distutils.core import setup 7 | 8 | pubsub_logging_classifiers = [ 9 | 'Programming Language :: Python :: 2', 10 | 'Programming Language :: Python :: 3', 11 | 'Intended Audience :: Developers', 12 | 'License :: OSI Approved :: Apache Software License', 13 | 'Topic :: Software Development :: Libraries', 14 | 'Topic :: Utilities', 15 | ] 16 | 17 | with open('README.rst', 'r') as fp: 18 | pubsub_logging_long_description = fp.read() 19 | 20 | REQUIREMENTS = [ 21 | 'google-api-python-client >= 1.4.0' 22 | ] 23 | 24 | setup( 25 | name='pubsub-logging', 26 | version='0.2.1', 27 | author='Takashi Matsuo', 28 | author_email='tmatsuo@google.com', 29 | url='https://github.com/GoogleCloudPlatform/cloud-pubsub-logging-python', 30 | packages=['pubsub_logging'], 31 | description="Logging handlers for publishing the logs to Cloud Pub/Sub", 32 | install_requires=REQUIREMENTS, 33 | long_description=pubsub_logging_long_description, 34 | license='Apache 2.0', 35 | classifiers=pubsub_logging_classifiers 36 | ) 37 | -------------------------------------------------------------------------------- /tests/pubsub_logging_test.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2015 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | """Unit tests for pubsub_logging.""" 17 | 18 | 19 | import logging 20 | import multiprocessing as mp 21 | import os 22 | import sys 23 | import time 24 | import unittest 25 | 26 | import httplib2 27 | import mock 28 | 29 | from apiclient import errors 30 | from oauth2client.client import GoogleCredentials 31 | 32 | import pubsub_logging # flake8: noqa 33 | 34 | from pubsub_logging.errors import RecoverableError 35 | from pubsub_logging.utils import check_topic 36 | from pubsub_logging.utils import compat_urlsafe_b64encode 37 | from pubsub_logging.utils import get_pubsub_client 38 | from pubsub_logging.utils import publish_body 39 | from pubsub_logging.utils import PUBSUB_SCOPES 40 | 41 | 42 | DEFAULT_TEST_PROJECT = 'pubsub-integration-test' 43 | TEST_PROJECT_ENV = 'PUBSUB_LOGGING_TEST_PROJECT' 44 | 45 | 46 | class CompatBase64Test(unittest.TestCase): 47 | """Test for compat_urlsafe_b64encode function.""" 48 | 49 | def test_compat_urlsafe_b64encode(self): 50 | v = 'test' 51 | expected = 'dGVzdA==' 52 | result = compat_urlsafe_b64encode(v) 53 | self.assertEqual(expected, result) 54 | 55 | 56 | class GetPubsubClientTest(unittest.TestCase): 57 | """Tests for utils.get_pubsub_client function. 58 | 59 | You have to set GOOGLE_APPLICATION_CREDENTIALS pointing to the 60 | json file of the service account. 61 | """ 62 | RETRY = 3 63 | 64 | def test_get_pubsub_client_with_service_account(self): 65 | """Tests the client obtained by the service account method.""" 66 | client = get_pubsub_client() 67 | project = ('projects/%s' 68 | % os.environ.get(TEST_PROJECT_ENV, DEFAULT_TEST_PROJECT)) 69 | client.projects().topics().list(project=project).execute( 70 | num_retries=self.RETRY) 71 | # Providing an Http object this time. 72 | self.assertIsNotNone(get_pubsub_client(http=httplib2.Http())) 73 | 74 | def test_get_pubsub_client_with_scoped_credentials(self): 75 | """Tests the client obtained by scoped credentials.""" 76 | credentials = GoogleCredentials.get_application_default() 77 | credentials = credentials.create_scoped(PUBSUB_SCOPES) 78 | self.assertIsNotNone(get_pubsub_client(credentials=credentials)) 79 | 80 | 81 | class CheckTopicTest(unittest.TestCase): 82 | """Tests for utils.check_topic function. 83 | 84 | You have to set GOOGLE_APPLICATION_CREDENTIALS pointing to the 85 | json file of the service account. 86 | """ 87 | RETRY = 3 88 | 89 | def setUp(self): 90 | self.client = get_pubsub_client() 91 | self.project = os.environ.get(TEST_PROJECT_ENV, DEFAULT_TEST_PROJECT) 92 | self.topic = 'projects/%s/topics/test-topic-%f' % (self.project, 93 | time.time()) 94 | self.nonexistence = 'projects/%s/topics/nonexistence' % self.project 95 | try: 96 | self.client.projects().topics().create( 97 | name=self.topic, body={}).execute() 98 | except errors.HttpError as e: 99 | if e.resp.status == 409: 100 | pass 101 | else: 102 | raise 103 | 104 | def tearDown(self): 105 | self.client.projects().topics().delete(topic=self.topic).execute() 106 | 107 | def test_check_topic(self): 108 | """Basic test for check_topic.""" 109 | self.assertTrue(check_topic(self.client, self.topic, self.RETRY)) 110 | 111 | def test_check_topic_failure(self): 112 | """Tests if the check_topic raises when getting a 404 error.""" 113 | self.assertFalse( 114 | check_topic(self.client, self.nonexistence, self.RETRY)) 115 | 116 | 117 | class PublishBodyTest(unittest.TestCase): 118 | """Tests for utils.publish_body function.""" 119 | RETRY = 3 120 | 121 | def setUp(self): 122 | self.mocked_client = mock.MagicMock() 123 | self.topic = 'projects/test-project/topics/test-topic' 124 | self.projects = self.mocked_client.projects.return_value 125 | self.topics = self.projects.topics.return_value 126 | self.topics_publish = self.topics.publish.return_value 127 | self.log_msg = 'Test message' 128 | self.expected_payload = compat_urlsafe_b64encode( 129 | self.log_msg) 130 | self.expected_body = {'messages': [{'data': self.expected_payload}]} 131 | self.r = logging.LogRecord('test', logging.INFO, None, 0, self.log_msg, 132 | [], None) 133 | 134 | def publish(self): 135 | publish_body(self.mocked_client, self.expected_body, self.topic, 136 | self.RETRY) 137 | 138 | def test_publish_body(self): 139 | """Basic test for publish_body.""" 140 | self.publish() 141 | self.topics.publish.assert_called_once_with( 142 | topic=self.topic, body=self.expected_body) 143 | self.topics_publish.execute.assert_called_with(num_retries=self.RETRY) 144 | 145 | def test_publish_body_raise_on_publish_404(self): 146 | """Tests if the flush method raises when publish gets a 404 error.""" 147 | mocked_resp = mock.MagicMock() 148 | mocked_resp.status = 404 149 | mocked_resp.reason = 'Not Found' 150 | # 404 error 151 | self.topics_publish.execute.side_effect = [ 152 | errors.HttpError(mocked_resp, 'Not found') 153 | ] 154 | self.assertRaises(errors.HttpError, self.publish) 155 | 156 | def test_flush_raise_on_publish_403(self): 157 | """Tests if the flush method raises when publish gets a 403 error.""" 158 | mocked_resp = mock.MagicMock() 159 | mocked_resp.status = 403 160 | mocked_resp.reason = 'Access not allowed' 161 | # 403 error 162 | self.topics_publish.execute.side_effect = [ 163 | errors.HttpError(mocked_resp, 'Access not allowed'), 164 | ] 165 | self.assertRaises(errors.HttpError, self.publish) 166 | 167 | def test_flush_ignore_recoverable(self): 168 | """Tests if we raise upon getting 503 error from Cloud Pub/Sub.""" 169 | mocked_resp = mock.MagicMock() 170 | mocked_resp.status = 503 171 | mocked_resp.reason = 'Server Error' 172 | # 503 error 173 | self.topics_publish.execute.side_effect = [ 174 | errors.HttpError(mocked_resp, 'Server Error'), 175 | ] 176 | self.assertRaises(RecoverableError, self.publish) 177 | self.topics.publish.assert_called_once_with( 178 | topic=self.topic, body=self.expected_body) 179 | self.topics_publish.execute.assert_called_once_with( 180 | num_retries=self.RETRY) 181 | 182 | 183 | class CountPublishBody(object): 184 | """A simple counter that counts total number of messages.""" 185 | def __init__(self, mock=None): 186 | """Initializes this mock. 187 | 188 | Args: 189 | mock: A mock object that we call before update the counter. 190 | """ 191 | self.cnt = mp.Value('i', 0) 192 | self.lock = mp.Lock() 193 | self._mock = mock 194 | 195 | def __call__(self, client, body, topic, retry): 196 | if self._mock: 197 | self._mock(client, body, topic, retry) 198 | with self.lock: 199 | self.cnt.value += len(body['messages']) 200 | 201 | 202 | class AsyncPubsubHandlerTest(unittest.TestCase): 203 | """Tests for async_handler.AsyncPubsubHandler.""" 204 | RETRY = 10 205 | 206 | def setUp(self): 207 | self.mocked_client = mock.MagicMock() 208 | self.topic = 'projects/test-project/topics/test-topic' 209 | 210 | @mock.patch('pubsub_logging.async_handler.check_topic') 211 | def test_fail_fast_when_topic_not_exist(self, check_topic): 212 | check_topic.return_value = False 213 | def create_handler(): 214 | pubsub_logging.AsyncPubsubHandler(topic=self.topic, 215 | client=self.mocked_client, 216 | worker_num=1) 217 | self.assertRaises(EnvironmentError, create_handler) 218 | 219 | def test_single_message(self): 220 | """Tests if utils.publish_body is called with one message.""" 221 | self.counter = CountPublishBody() 222 | self.handler = pubsub_logging.AsyncPubsubHandler( 223 | topic=self.topic, client=self.mocked_client, retry=self.RETRY, 224 | worker_num=1, publish_body=self.counter) 225 | log_msg = 'Test message' 226 | r = logging.LogRecord('test', logging.CRITICAL, None, 0, log_msg, [], 227 | None) 228 | self.handler.emit(r) 229 | self.handler.close() 230 | with self.counter.lock: 231 | self.assertEqual(1, self.counter.cnt.value) 232 | 233 | def test_handler_ignores_error(self): 234 | """Tests if the handler ignores errors and throws the logs away.""" 235 | mock_publish_body = mock.MagicMock() 236 | mock_publish_body.side_effect = [RecoverableError(), mock.DEFAULT] 237 | self.counter = CountPublishBody(mock=mock_publish_body) 238 | # For suppressing the output. 239 | devnull = logging.Logger('devnull') 240 | devnull.addHandler(logging.NullHandler()) 241 | self.handler = pubsub_logging.AsyncPubsubHandler( 242 | topic=self.topic, client=self.mocked_client, retry=self.RETRY, 243 | worker_num=1, publish_body=self.counter, 244 | stderr_logger=devnull) 245 | log_msg = 'Test message' 246 | r = logging.LogRecord('test', logging.CRITICAL, None, 0, log_msg, [], 247 | None) 248 | 249 | # RecoverableError should be ignored, and retried. 250 | self.handler.emit(r) 251 | self.handler.close() 252 | with self.counter.lock: 253 | self.assertEqual(1, self.counter.cnt.value) 254 | 255 | def test_total_message_count(self): 256 | """Tests if utils.publish_body is called with 10000 message.""" 257 | self.counter = CountPublishBody() 258 | self.handler = pubsub_logging.AsyncPubsubHandler( 259 | topic=self.topic, client=self.mocked_client, retry=self.RETRY, 260 | worker_num=10, publish_body=self.counter) 261 | log_msg = 'Test message' 262 | r = logging.LogRecord('test', logging.CRITICAL, None, 0, log_msg, [], 263 | None) 264 | num = 10000 265 | for i in range(num): 266 | self.handler.emit(r) 267 | self.handler.close() 268 | with self.counter.lock: 269 | self.assertEqual(num, self.counter.cnt.value) 270 | 271 | 272 | class PubsubHandlerTest(unittest.TestCase): 273 | """Tests for the emit method.""" 274 | RETRY = 3 275 | BATCH_NUM = 2 276 | 277 | def setUp(self): 278 | self.mocked_client = mock.MagicMock() 279 | self.topic = 'projects/test-project/topics/test-topic' 280 | self.handler = pubsub_logging.PubsubHandler( 281 | topic=self.topic, client=self.mocked_client, retry=self.RETRY, 282 | capacity=self.BATCH_NUM) 283 | self.handler.flush = mock.MagicMock() 284 | 285 | @mock.patch('pubsub_logging.pubsub_handler.check_topic') 286 | def test_fail_fast_when_topic_not_exist(self, check_topic): 287 | check_topic.return_value = False 288 | def create_handler(): 289 | pubsub_logging.PubsubHandler(topic=self.topic, 290 | client=self.mocked_client) 291 | self.assertRaises(EnvironmentError, create_handler) 292 | 293 | @mock.patch('pubsub_logging.pubsub_handler.get_pubsub_client') 294 | def test_constructor_without_client(self, get_pubsub_client): 295 | """Tests if the constructor create a new Pub/Sub client.""" 296 | get_pubsub_client.return_value = self.mocked_client 297 | handler = pubsub_logging.PubsubHandler( 298 | topic=self.topic, client=None, retry=self.RETRY, 299 | capacity=self.BATCH_NUM) 300 | self.assertEqual(self.mocked_client, handler._client) 301 | 302 | def test_single_buff(self): 303 | """Tests if the log is stored in the internal buffer.""" 304 | log_msg = 'Test message' 305 | r = logging.LogRecord('test', logging.INFO, None, 0, log_msg, [], None) 306 | 307 | self.handler.emit(r) 308 | self.assertEqual(1, len(self.handler.buffer)) 309 | self.assertIs(r, self.handler.buffer[0]) 310 | 311 | def test_critical_forces_flush(self): 312 | """Tests if a single CRITICAL level log forces flushing.""" 313 | log_msg = 'Test message' 314 | r = logging.LogRecord('test', logging.CRITICAL, None, 0, log_msg, [], 315 | None) 316 | 317 | self.handler.emit(r) 318 | self.handler.flush.assert_called_once() 319 | 320 | def test_custom_level_forces_flush(self): 321 | """Tests if a single INFO level log forces flushing.""" 322 | self.handler._flush_level = logging.INFO 323 | log_msg = 'Test message' 324 | r = logging.LogRecord('test', logging.INFO, None, 0, log_msg, [], None) 325 | 326 | self.handler.emit(r) 327 | self.handler.flush.assert_called_once() 328 | 329 | def test_flush_when_full(self): 330 | """Tests if the flush is called when the buffer is full.""" 331 | log_msg1 = 'Test message' 332 | log_msg2 = 'Test message2' 333 | r1 = logging.LogRecord('test', logging.INFO, None, 0, log_msg1, [], 334 | None) 335 | r2 = logging.LogRecord('test', logging.INFO, None, 0, log_msg2, [], 336 | None) 337 | 338 | self.handler.emit(r1) 339 | self.handler.flush.assert_not_called() 340 | 341 | self.handler.emit(r2) 342 | self.handler.flush.assert_called_once() 343 | 344 | 345 | class PubsubHandlerFlushTest(unittest.TestCase): 346 | """Tests for the flush method of PubsubHandler.""" 347 | RETRY = 3 348 | BATCH_NUM = 2 349 | 350 | def setUp(self): 351 | self.mocked_client = mock.MagicMock() 352 | self.topic = 'projects/test-project/topics/test-topic' 353 | self.publish_body = mock.MagicMock() 354 | self.handler = pubsub_logging.PubsubHandler( 355 | topic=self.topic, client=self.mocked_client, retry=self.RETRY, 356 | capacity=self.BATCH_NUM, publish_body=self.publish_body) 357 | self.log_msg = 'Test message' 358 | self.expected_payload = compat_urlsafe_b64encode( 359 | self.log_msg) 360 | self.expected_body = {'messages': [{'data': self.expected_payload}]} 361 | self.r = logging.LogRecord('test', logging.INFO, None, 0, self.log_msg, 362 | [], None) 363 | 364 | def test_flush(self): 365 | """Tests if the flush method calls publish_body.""" 366 | self.handler.emit(self.r) 367 | 368 | self.handler.flush() 369 | self.publish_body.assert_called_once_with( 370 | self.mocked_client, self.expected_body, self.topic, self.RETRY) 371 | self.assertEqual(0, len(self.handler.buffer)) 372 | 373 | def test_flush_raise_on_publish_404(self): 374 | """Tests if the flush raises upon 404 error from publish_body.""" 375 | self.handler.emit(self.r) 376 | mocked_resp = mock.MagicMock() 377 | mocked_resp.status = 404 378 | mocked_resp.reason = 'Not Found' 379 | # 404 error and None for atexit. 380 | self.publish_body.side_effect = [ 381 | errors.HttpError(mocked_resp, 'Not found'), 382 | None] 383 | self.assertRaises(errors.HttpError, self.handler.flush) 384 | 385 | def test_flush_ignore_recoverable(self): 386 | """Tests if we ignore Recoverable error from publish_body.""" 387 | self.handler.emit(self.r) 388 | self.publish_body.side_effect = RecoverableError() 389 | self.handler.flush() 390 | 391 | self.publish_body.assert_called_once_with( 392 | self.mocked_client, self.expected_body, self.topic, self.RETRY) 393 | self.assertEqual(1, len(self.handler.buffer)) 394 | 395 | def test_cut_buffer(self): 396 | """Tests if we cut the buffer upon recoverale errors.""" 397 | self.handler._buf_hard_limit = 0 398 | self.handler.emit(self.r) 399 | self.publish_body.side_effect = RecoverableError() 400 | self.handler.flush() 401 | 402 | self.publish_body.assert_called_once_with( 403 | self.mocked_client, self.expected_body, self.topic, self.RETRY) 404 | self.assertEqual(0, len(self.handler.buffer)) 405 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | skipsdist = True 3 | envlist = {py27,py34}-{nosetest,pep8}, cover 4 | 5 | [testenv] 6 | basepython = 7 | cover,py27: python2.7 8 | py34: python3.4 9 | deps = 10 | google-api-python-client 11 | pep8: flake8 12 | pep8: flake8-import-order 13 | nosetest,cover: nose 14 | nosetest,cover: mock 15 | cover: coverage 16 | cover: nosexcover 17 | cover: coveralls 18 | commands = 19 | nosetest: nosetests 20 | pep8: flake8 --exclude lib,bin --max-complexity=10 \ 21 | pep8: --import-order-style=google 22 | cover: nosetests --with-xunit --with-xcoverage \ 23 | cover: --cover-package=pubsub_logging --nocapture --cover-erase \ 24 | cover: --cover-tests --cover-branches --cover-min-percentage=100 25 | cover: coveralls 26 | --------------------------------------------------------------------------------