├── MANIFEST.in ├── requirements.txt ├── test-requirements.txt ├── .travis.yml ├── pylintrc ├── docs ├── index.rst ├── Makefile ├── make.bat └── conf.py ├── .gitignore ├── LICENSE ├── test ├── __init__.py ├── test_auth.py ├── test_gax.py ├── test_path_template.py ├── test_grpc.py ├── test_api_callable.py └── test_bundling.py ├── google ├── __init__.py └── gax │ ├── auth.py │ ├── config.py │ ├── errors.py │ ├── grpc.py │ ├── path_template.py │ ├── bundling.py │ ├── __init__.py │ └── api_callable.py ├── tox.ini ├── README.rst ├── setup.py └── CONTRIBUTING.rst /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | grpcio==0.13.1 2 | oauth2client>=1.5.2 3 | ply==3.8 4 | protobuf>=3.0.0b1.post1 5 | -------------------------------------------------------------------------------- /test-requirements.txt: -------------------------------------------------------------------------------- 1 | mock>=1.3.0 2 | pytest>=2.8.3 3 | pytest-cov>=1.8.1 4 | pytest-timeout>=1.0.0 5 | unittest2>=1.1.0 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: python 3 | python: 4 | - "2.7" 5 | install: pip install codecov tox-travis 6 | script: tox 7 | after_success: 8 | - codecov 9 | -------------------------------------------------------------------------------- /pylintrc: -------------------------------------------------------------------------------- 1 | # checks for sign of poor/misdesign: 2 | # * number of methods, attributes, local variables... 3 | # * size, complexity of functions, methods 4 | # 5 | [DESIGN] 6 | 7 | # Maximum number of arguments for function / method 8 | max-args=8 9 | 10 | # Maximum number of public methods for a class (see R0904). 11 | max-public-methods=20 12 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. google-gax documentation master file, created by 2 | sphinx-quickstart on Sun Feb 04 16:22:46 2016. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | 7 | .. toctree:: 8 | :maxdepth: 2 9 | :caption: Google APIs Extensions 10 | 11 | This is the API documentation for Google API Extensions for Python (gax-python), 12 | a set of libraries which aids the development of APIs for clients and servers 13 | based on `GRPC`_ and `Google APIs`_ conventions. 14 | 15 | 16 | .. autosummary:: 17 | :toctree: generated 18 | 19 | google.gax 20 | google.gax.auth 21 | google.gax.api_callable 22 | google.gax.bundling 23 | google.gax.config 24 | google.gax.errors 25 | google.gax.grpc 26 | google.gax.path_template 27 | 28 | 29 | Indices and tables 30 | ================== 31 | 32 | * :ref:`genindex` 33 | * :ref:`modindex` 34 | * :ref:`search` 35 | 36 | 37 | .. _`GRPC`: http://grpc.io 38 | .. _`Google APIs`: https://github.com/googleapis/googleapis/ 39 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | .DS_Store 3 | 4 | # https://github.com/github/gitignore/blob/master/Python.gitignore 5 | 6 | # Byte-compiled / optimized / DLL files 7 | __pycache__/ 8 | *.py[cod] 9 | *$py.class 10 | 11 | # C extensions 12 | *.so 13 | 14 | # Distribution / packaging 15 | .Python 16 | env/ 17 | build/ 18 | develop-eggs/ 19 | docs/generated 20 | dist/ 21 | downloads/ 22 | eggs/ 23 | .eggs/ 24 | lib/ 25 | lib64/ 26 | parts/ 27 | sdist/ 28 | var/ 29 | *.egg-info/ 30 | .installed.cfg 31 | *.egg 32 | 33 | # PyInstaller 34 | # Usually these files are written by a python script from a template 35 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 36 | *.manifest 37 | *.spec 38 | 39 | # Installer logs 40 | pip-log.txt 41 | pip-delete-this-directory.txt 42 | 43 | # Unit test / coverage reports 44 | htmlcov/ 45 | .tox/ 46 | .coverage 47 | .coverage.* 48 | .cache 49 | nosetests.xml 50 | coverage.xml 51 | *,cover 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Django stuff: 58 | *.log 59 | 60 | # Sphinx documentation 61 | docs/_build/ 62 | _build 63 | 64 | # PyBuilder 65 | target/ 66 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2015, Google Inc. 2 | All rights reserved. 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | * Redistributions of source code must retain the above copyright 7 | notice, this list of conditions and the following disclaimer. 8 | * Redistributions in binary form must reproduce the above 9 | copyright notice, this list of conditions and the following disclaimer 10 | in the documentation and/or other materials provided with the 11 | distribution. 12 | * Neither the name of Google Inc. nor the names of its 13 | contributors may be used to endorse or promote products derived from 14 | this software without specific prior written permission. 15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 18 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 19 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 21 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015, Google Inc. 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are 6 | # met: 7 | # 8 | # * Redistributions of source code must retain the above copyright 9 | # notice, this list of conditions and the following disclaimer. 10 | # * Redistributions in binary form must reproduce the above 11 | # copyright notice, this list of conditions and the following disclaimer 12 | # in the documentation and/or other materials provided with the 13 | # distribution. 14 | # * Neither the name of Google Inc. nor the names of its 15 | # contributors may be used to endorse or promote products derived from 16 | # this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | -------------------------------------------------------------------------------- /google/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016, Google Inc. 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are 6 | # met: 7 | # 8 | # * Redistributions of source code must retain the above copyright 9 | # notice, this list of conditions and the following disclaimer. 10 | # * Redistributions in binary form must reproduce the above 11 | # copyright notice, this list of conditions and the following disclaimer 12 | # in the documentation and/or other materials provided with the 13 | # distribution. 14 | # * Neither the name of Google Inc. nor the names of its 15 | # contributors may be used to endorse or promote products derived from 16 | # this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | # pylint: disable=missing-docstring 31 | __import__('pkg_resources').declare_namespace(__name__) # pragma: no cover 32 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py27,pep8,pylint-errors,pylint-full 3 | 4 | [tox:travis] 5 | 2.7 = py27, pep8, pylint-full, docs 6 | 7 | [testenv] 8 | setenv = 9 | PYTHONPATH = {toxinidir} 10 | PIP_FIND_LINKS = https://gapi-pypi.appspot.com/admin/nurpc-dev 11 | 12 | deps = -r{toxinidir}/test-requirements.txt 13 | -r{toxinidir}/requirements.txt 14 | commands = py.test --timeout=30 --cov-report html --cov-report=term --cov {toxinidir}/google 15 | 16 | [testenv:pep8] 17 | deps = flake8 18 | commands = flake8 --max-complexity=10 google test --ignore=E501 19 | 20 | [testenv:pylint-errors] 21 | deps = pylint 22 | -r{toxinidir}/test-requirements.txt 23 | -r{toxinidir}/requirements.txt 24 | commands = pylint -f colorized -E google test 25 | 26 | [testenv:pylint-warnings] 27 | deps = pylint 28 | commands = pylint -f colorized -d all -e W -r n google test 29 | 30 | [testenv:pylint-full] 31 | deps = pylint 32 | -r{toxinidir}/test-requirements.txt 33 | -r{toxinidir}/requirements.txt 34 | commands = pylint -f colorized -e E,W,R -d fixme,locally-disabled google test 35 | 36 | [testenv:devenv] 37 | commands = 38 | envdir = {toxworkdir}/develop 39 | basepython = python2.7 40 | usedevelop = True 41 | deps= -r{toxinidir}/test-requirements.txt 42 | -r{toxinidir}/requirements.txt 43 | 44 | 45 | [testenv:docs] 46 | basepython = python2.7 47 | commands = 48 | python -c "import shutil; shutil.rmtree('docs/_build', ignore_errors=True)" 49 | python -c "import shutil; shutil.rmtree('docs/generated', ignore_errors=True)" 50 | python -c "import shutil; shutil.rmtree('docs/_static', ignore_errors=True)" 51 | python -c "import os; os.makedirs('docs/_static')" 52 | sphinx-build -W -b html -d docs/_build/doctrees docs docs/_build/html 53 | sphinx-build -b latex -D language=en -d _build/doctrees docs _build/latex 54 | deps = 55 | Sphinx 56 | sphinx_rtd_theme 57 | -------------------------------------------------------------------------------- /google/gax/auth.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015, Google Inc. 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are 6 | # met: 7 | # 8 | # * Redistributions of source code must retain the above copyright 9 | # notice, this list of conditions and the following disclaimer. 10 | # * Redistributions in binary form must reproduce the above 11 | # copyright notice, this list of conditions and the following disclaimer 12 | # in the documentation and/or other materials provided with the 13 | # distribution. 14 | # * Neither the name of Google Inc. nor the names of its 15 | # contributors may be used to endorse or promote products derived from 16 | # this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | """Provides a generic authorization callback.""" 31 | 32 | from __future__ import absolute_import 33 | 34 | from oauth2client import client as auth_client 35 | 36 | 37 | def make_auth_func(scopes): 38 | """Creates the callback that provides per rpc auth creds.""" 39 | google_creds = auth_client.GoogleCredentials.get_application_default() 40 | scoped_creds = google_creds.create_scoped(scopes) 41 | 42 | def auth_func(): 43 | """Adds the access token from the creds as the authorization token.""" 44 | authn = scoped_creds.get_access_token().access_token 45 | return [ 46 | ('authorization', 'Bearer %s' % (authn,)) 47 | ] 48 | 49 | return auth_func 50 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Google API Extensions for Python 2 | ================================ 3 | 4 | .. image:: https://img.shields.io/travis/googleapis/gax-python.svg 5 | :target: https://travis-ci.org/googleapis/gax-python 6 | 7 | .. image:: https://img.shields.io/pypi/dw/google-gax.svg 8 | :target: https://pypi.python.org/pypi/google-gax 9 | 10 | .. image:: https://readthedocs.org/projects/gax-python/badge/?version=latest 11 | :target: http://gax-python.readthedocs.org/ 12 | 13 | .. image:: https://img.shields.io/codecov/c/github/googleapis/gax-python.svg 14 | :target: https://codecov.io/github/googleapis/gax-python 15 | 16 | 17 | Google API Extensions for Python (gax-python) is a set of modules which aids the 18 | development of APIs for clients and servers based on `gRPC`_ and Google API 19 | conventions. 20 | 21 | Application code will rarely need to use most of the classes within this library 22 | directly, but code generated automatically from the API definition files in 23 | `Google APIs`_ can use services such as page streaming and request bundling to 24 | provide a more convenient and idiomatic API surface to callers. 25 | 26 | .. _`gRPC`: http://grpc.io 27 | .. _`Google APIs`: https://github.com/googleapis/googleapis/ 28 | 29 | 30 | Python Versions 31 | --------------- 32 | 33 | gax-python is currently tested with Python 2.7. 34 | 35 | 36 | Contributing 37 | ------------ 38 | 39 | Contributions to this library are always welcome and highly encouraged. 40 | 41 | See the `CONTRIBUTING`_ documentation for more information on how to get started. 42 | 43 | .. _`CONTRIBUTING`: https://github.com/googleapis/gax-python/blob/master/CONTRIBUTING.rst 44 | 45 | 46 | Versioning 47 | ---------- 48 | 49 | This library follows `Semantic Versioning`_ 50 | 51 | It is currently in major version zero (``0.y.z``), which means that anything 52 | may change at any time and the public API should not be considered 53 | stable. 54 | 55 | .. _`Semantic Versioning`: http://semver.org/ 56 | 57 | 58 | Details 59 | ------- 60 | 61 | For detailed documentation of the modules in gax-python, please watch `DOCUMENTATION`_. 62 | 63 | .. _`DOCUMENTATION`: https://gax-python.readthedocs.org/ 64 | 65 | 66 | License 67 | ------- 68 | 69 | BSD - See `LICENSE`_ for more information. 70 | 71 | .. _`LICENSE`: https://github.com/googleapis/gax-python/blob/master/LICENSE 72 | -------------------------------------------------------------------------------- /google/gax/config.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015, Google Inc. 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are 6 | # met: 7 | # 8 | # * Redistributions of source code must retain the above copyright 9 | # notice, this list of conditions and the following disclaimer. 10 | # * Redistributions in binary form must reproduce the above 11 | # copyright notice, this list of conditions and the following disclaimer 12 | # in the documentation and/or other materials provided with the 13 | # distribution. 14 | # * Neither the name of Google Inc. nor the names of its 15 | # contributors may be used to endorse or promote products derived from 16 | # this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | """Runtime configuration shared by gax modules.""" 31 | 32 | from __future__ import absolute_import 33 | 34 | from . import grpc 35 | 36 | 37 | exc_to_code = grpc.exc_to_code # pylint: disable=invalid-name 38 | """A function that takes an exception and returns a status code. 39 | 40 | May return None if the exception is not associated with a status code. 41 | """ 42 | 43 | 44 | STATUS_CODE_NAMES = grpc.STATUS_CODE_NAMES 45 | """Maps strings used in client config to the status codes they represent. 46 | 47 | This is necessary for google.gax.api_callable.construct_settings to translate 48 | the client constants configuration for retrying into the correct gRPC objects. 49 | """ 50 | 51 | 52 | create_stub = grpc.create_stub # pylint: disable=invalid-name, 53 | """The function to use to create stubs.""" 54 | 55 | 56 | API_ERRORS = grpc.API_ERRORS 57 | """Errors that indicate that an RPC was aborted.""" 58 | -------------------------------------------------------------------------------- /google/gax/errors.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016, Google Inc. 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are 6 | # met: 7 | # 8 | # * Redistributions of source code must retain the above copyright 9 | # notice, this list of conditions and the following disclaimer. 10 | # * Redistributions in binary form must reproduce the above 11 | # copyright notice, this list of conditions and the following disclaimer 12 | # in the documentation and/or other materials provided with the 13 | # distribution. 14 | # * Neither the name of Google Inc. nor the names of its 15 | # contributors may be used to endorse or promote products derived from 16 | # this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | """Provides GAX exceptions.""" 31 | 32 | 33 | class GaxError(Exception): 34 | """Common base class for exceptions raised by GAX. 35 | 36 | Attributes: 37 | msg (string): describes the error that occurred. 38 | cause (Exception, optional): the exception raised by a lower 39 | layer of the RPC stack (for example, gRPC) that caused this 40 | exception, or None if this exception originated in GAX. 41 | """ 42 | def __init__(self, msg, cause=None): 43 | super(GaxError, self).__init__(msg) 44 | self.cause = cause 45 | 46 | def __str__(self): 47 | msg = super(GaxError, self).__str__() 48 | if not self.cause: 49 | return msg 50 | 51 | return 'GaxError({}, caused by {})'.format(msg, self.cause) 52 | 53 | 54 | class RetryError(GaxError): 55 | """Indicates an error during automatic GAX retrying.""" 56 | pass 57 | -------------------------------------------------------------------------------- /test/test_auth.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016, Google Inc. 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are 6 | # met: 7 | # 8 | # * Redistributions of source code must retain the above copyright 9 | # notice, this list of conditions and the following disclaimer. 10 | # * Redistributions in binary form must reproduce the above 11 | # copyright notice, this list of conditions and the following disclaimer 12 | # in the documentation and/or other materials provided with the 13 | # distribution. 14 | # * Neither the name of Google Inc. nor the names of its 15 | # contributors may be used to endorse or promote products derived from 16 | # this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | # pylint: disable=missing-docstring,no-self-use,no-init,invalid-name 31 | """Unit tests for auth.""" 32 | 33 | from __future__ import absolute_import 34 | 35 | import mock 36 | import unittest2 37 | 38 | from google.gax import auth 39 | 40 | 41 | class TestMakeAuthFunc(unittest2.TestCase): 42 | TEST_TOKEN = 'an_auth_token' 43 | 44 | @mock.patch('oauth2client.client.GoogleCredentials.get_application_default') 45 | def test_uses_application_default_credentials(self, factory): 46 | creds = mock.Mock() 47 | creds.get_access_token.return_value = mock.Mock( 48 | access_token=self.TEST_TOKEN) 49 | factory_mock_config = {'create_scoped.return_value': creds} 50 | factory.return_value = mock.Mock(**factory_mock_config) 51 | fake_scopes = ['fake', 'scopes'] 52 | the_func = auth.make_auth_func(fake_scopes) 53 | factory.return_value.create_scoped.assert_called_once_with(fake_scopes) 54 | got = the_func() 55 | want = [('authorization', 'Bearer an_auth_token')] 56 | self.assertEqual(got, want) 57 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 2015, Google Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are 8 | # met: 9 | # 10 | # * Redistributions of source code must retain the above copyright 11 | # notice, this list of conditions and the following disclaimer. 12 | # * Redistributions in binary form must reproduce the above 13 | # copyright notice, this list of conditions and the following disclaimer 14 | # in the documentation and/or other materials provided with the 15 | # distribution. 16 | # * Neither the name of Google Inc. nor the names of its 17 | # contributors may be used to endorse or promote products derived from 18 | # this software without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | import os 33 | import re 34 | import sys 35 | from setuptools import setup, find_packages 36 | 37 | if sys.argv[-1] == 'publish': 38 | os.system('python setup.py sdist upload') 39 | sys.exit() 40 | 41 | # Get the version 42 | version_regex = r'__version__ = ["\']([^"\']*)["\']' 43 | with open('google/gax/__init__.py', 'r') as f: 44 | text = f.read() 45 | match = re.search(version_regex, text) 46 | if match: 47 | version = match.group(1) 48 | else: 49 | raise RuntimeError("No version number found!") 50 | 51 | install_requires = [ 52 | 'grpcio==0.13.1', 53 | 'ply==3.8', 54 | 'protobuf>=3.0.0b1.post1', 55 | 'oauth2client>=1.5.2', 56 | ] 57 | 58 | setup( 59 | name='google-gax', 60 | version=version, 61 | description='Google API Extensions', 62 | long_description=open('README.rst').read(), 63 | author='Google API Authors', 64 | author_email='googleapis-packages@google.com', 65 | url='https://github.com/googleapis/gax-python', 66 | namespace_packages = ['google'], 67 | packages=find_packages(), 68 | package_dir={'google-gax': 'google'}, 69 | license='BSD-3-Clause', 70 | classifiers=[ 71 | 'Development Status :: 4 - Beta', 72 | 'Intended Audience :: Developers', 73 | 'License :: OSI Approved :: BSD License', 74 | 'Programming Language :: Python', 75 | 'Programming Language :: Python :: 2', 76 | 'Programming Language :: Python :: 2.7', 77 | 'Programming Language :: Python :: Implementation :: CPython', 78 | ], 79 | tests_require=['pytest'], 80 | install_requires=install_requires, 81 | ) 82 | -------------------------------------------------------------------------------- /test/test_gax.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016, Google Inc. 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are 6 | # met: 7 | # 8 | # * Redistributions of source code must retain the above copyright 9 | # notice, this list of conditions and the following disclaimer. 10 | # * Redistributions in binary form must reproduce the above 11 | # copyright notice, this list of conditions and the following disclaimer 12 | # in the documentation and/or other materials provided with the 13 | # distribution. 14 | # * Neither the name of Google Inc. nor the names of its 15 | # contributors may be used to endorse or promote products derived from 16 | # this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | # pylint: disable=missing-docstring,no-self-use,no-init,invalid-name 31 | """Unit tests for gax package globals.""" 32 | 33 | from __future__ import absolute_import 34 | 35 | import unittest2 36 | 37 | from google.gax import ( 38 | BundleOptions, CallOptions, CallSettings, OPTION_INHERIT, RetryOptions) 39 | 40 | 41 | class TestBundleOptions(unittest2.TestCase): 42 | 43 | def test_cannont_construct_with_noarg_options(self): 44 | self.assertRaises(AssertionError, 45 | BundleOptions) 46 | 47 | def test_cannont_construct_with_bad_options(self): 48 | not_an_int = 'i am a string' 49 | self.assertRaises(AssertionError, 50 | BundleOptions, 51 | element_count_threshold=not_an_int) 52 | self.assertRaises(AssertionError, 53 | BundleOptions, 54 | request_byte_threshold=not_an_int) 55 | self.assertRaises(AssertionError, 56 | BundleOptions, 57 | delay_threshold=not_an_int) 58 | 59 | 60 | class TestCallSettings(unittest2.TestCase): 61 | 62 | def test_call_options_simple(self): 63 | options = CallOptions(timeout=23) 64 | self.assertEqual(options.timeout, 23) 65 | self.assertEqual(options.retry, OPTION_INHERIT) 66 | self.assertEqual(options.is_page_streaming, OPTION_INHERIT) 67 | 68 | def test_settings_merge_options1(self): 69 | retry = RetryOptions(None, None) 70 | options = CallOptions(timeout=46, retry=retry) 71 | settings = CallSettings(timeout=9, page_descriptor=None, retry=None) 72 | final = settings.merge(options) 73 | self.assertEqual(final.timeout, 46) 74 | self.assertEqual(final.retry, retry) 75 | self.assertIsNone(final.page_descriptor) 76 | 77 | def test_settings_merge_options2(self): 78 | options = CallOptions(retry=None) 79 | settings = CallSettings( 80 | timeout=9, page_descriptor=None, retry=RetryOptions(None, None)) 81 | final = settings.merge(options) 82 | self.assertEqual(final.timeout, 9) 83 | self.assertIsNone(final.page_descriptor) 84 | self.assertIsNone(final.retry) 85 | 86 | def test_settings_merge_options_page_streaming(self): 87 | retry = RetryOptions(None, None) 88 | options = CallOptions(timeout=46, is_page_streaming=False) 89 | settings = CallSettings(timeout=9, retry=retry) 90 | final = settings.merge(options) 91 | self.assertEqual(final.timeout, 46) 92 | self.assertIsNone(final.page_descriptor) 93 | self.assertEqual(final.retry, retry) 94 | 95 | def test_settings_merge_none(self): 96 | settings = CallSettings( 97 | timeout=23, page_descriptor=object(), bundler=object(), 98 | retry=object()) 99 | final = settings.merge(None) 100 | self.assertEqual(final.timeout, settings.timeout) 101 | self.assertEqual(final.retry, settings.retry) 102 | self.assertEqual(final.page_descriptor, settings.page_descriptor) 103 | self.assertEqual(final.bundler, settings.bundler) 104 | self.assertEqual(final.bundle_descriptor, settings.bundle_descriptor) 105 | -------------------------------------------------------------------------------- /google/gax/grpc.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015, Google Inc. 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are 6 | # met: 7 | # 8 | # * Redistributions of source code must retain the above copyright 9 | # notice, this list of conditions and the following disclaimer. 10 | # * Redistributions in binary form must reproduce the above 11 | # copyright notice, this list of conditions and the following disclaimer 12 | # in the documentation and/or other materials provided with the 13 | # distribution. 14 | # * Neither the name of Google Inc. nor the names of its 15 | # contributors may be used to endorse or promote products derived from 16 | # this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | """Adapts the grpc surface.""" 31 | 32 | from __future__ import absolute_import 33 | from grpc.beta import implementations 34 | from grpc.beta.interfaces import StatusCode 35 | from grpc.framework.interfaces.face import face 36 | from . import auth 37 | 38 | 39 | API_ERRORS = (face.AbortionError, ) 40 | """gRPC exceptions that indicate that an RPC was aborted.""" 41 | 42 | 43 | STATUS_CODE_NAMES = { 44 | 'ABORTED': StatusCode.ABORTED, 45 | 'CANCELLED': StatusCode.CANCELLED, 46 | 'DATA_LOSS': StatusCode.DATA_LOSS, 47 | 'DEADLINE_EXCEEDED': StatusCode.DEADLINE_EXCEEDED, 48 | 'FAILED_PRECONDITION': StatusCode.FAILED_PRECONDITION, 49 | 'INTERNAL': StatusCode.INTERNAL, 50 | 'INVALID_ARGUMENT': StatusCode.INVALID_ARGUMENT, 51 | 'NOT_FOUND': StatusCode.NOT_FOUND, 52 | 'OUT_OF_RANGE': StatusCode.OUT_OF_RANGE, 53 | 'PERMISSION_DENIED': StatusCode.PERMISSION_DENIED, 54 | 'RESOURCE_EXHAUSTED': StatusCode.RESOURCE_EXHAUSTED, 55 | 'UNAUTHENTICATED': StatusCode.UNAUTHENTICATED, 56 | 'UNAVAILABLE': StatusCode.UNAVAILABLE, 57 | 'UNIMPLEMENTED': StatusCode.UNIMPLEMENTED, 58 | 'UNKNOWN': StatusCode.UNKNOWN} 59 | """Maps strings used in client config to gRPC status codes.""" 60 | 61 | 62 | def exc_to_code(exc): 63 | """Retrieves the status code from an exception""" 64 | if not isinstance(exc, face.AbortionError): 65 | return None 66 | elif isinstance(exc, face.ExpirationError): 67 | return StatusCode.DEADLINE_EXCEEDED 68 | else: 69 | return getattr(exc, 'code', None) 70 | 71 | 72 | def _make_grpc_auth_func(auth_func): 73 | """Creates the auth func expected by the grpc callback.""" 74 | 75 | def grpc_auth(dummy_context, callback): 76 | """The auth signature required by grpc.""" 77 | callback(auth_func(), None) 78 | 79 | return grpc_auth 80 | 81 | 82 | def _make_channel_creds(auth_func, ssl_creds): 83 | """Converts the auth func into the composite creds expected by grpc.""" 84 | grpc_auth_func = _make_grpc_auth_func(auth_func) 85 | call_creds = implementations.metadata_call_credentials(grpc_auth_func) 86 | return implementations.composite_channel_credentials(ssl_creds, call_creds) 87 | 88 | 89 | def create_stub(generated_create_stub, service_path, port, ssl_creds=None, 90 | channel=None, metadata_transformer=None, scopes=None): 91 | """Creates a gRPC client stub. 92 | 93 | Args: 94 | generated_create_stub: The generated gRPC method to create a stub. 95 | service_path: The domain name of the API remote host. 96 | port: The port on which to connect to the remote host. 97 | ssl_creds: A ClientCredentials object for use with an SSL-enabled 98 | Channel. If none, credentials are pulled from a default location. 99 | channel: A Channel object through which to make calls. If none, a secure 100 | channel is constructed. 101 | metadata_transformer: A function that transforms the metadata for 102 | requests, e.g., to give OAuth credentials. 103 | scopes: The OAuth scopes for this service. This parameter is ignored if 104 | a custom metadata_transformer is supplied. 105 | 106 | Returns: 107 | A gRPC client stub. 108 | """ 109 | if channel is None: 110 | if ssl_creds is None: 111 | ssl_creds = implementations.ssl_channel_credentials( 112 | None, None, None) 113 | if metadata_transformer is None: 114 | if scopes is None: 115 | scopes = [] 116 | metadata_transformer = auth.make_auth_func(scopes) 117 | 118 | channel_creds = _make_channel_creds(metadata_transformer, ssl_creds) 119 | channel = implementations.secure_channel( 120 | service_path, port, channel_creds) 121 | 122 | return generated_create_stub(channel) 123 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | Contributing 2 | ============ 3 | 4 | Here are some guidelines for hacking on `gax-python`_. 5 | 6 | - Please **sign** one of the `Contributor License Agreements`_ below. 7 | - `File an issue`_ to notify the maintainers about what you're working on. 8 | - `Fork the repo`_; develop and `test your code changes`_; add docs. 9 | - Make sure that your `commit messages`_ clearly describe the changes. 10 | - `Make the pull request`_. 11 | 12 | .. _`Fork the repo`: https://help.github.com/articles/fork-a-repo 13 | .. _`forking`: https://help.github.com/articles/fork-a-repo 14 | .. _`commit messages`: http://chris.beams.io/posts/git-commit/ 15 | 16 | .. _`File an issue`: 17 | 18 | Before writing code, file an issue 19 | ---------------------------------- 20 | 21 | Use the issue tracker to start the discussion. It is possible that someone else 22 | is already working on your idea, your approach is not quite right, or that the 23 | functionality exists already. The ticket you file in the issue tracker will be 24 | used to hash that all out. 25 | 26 | Fork `gax-python` 27 | ------------------- 28 | 29 | We will use GitHub's mechanism for `forking`_ repositories and making pull 30 | requests. Fork the repository, and make your changes in the forked repository. 31 | 32 | .. _`test your code changes`: 33 | 34 | Include tests 35 | ------------- 36 | 37 | Be sure to add relevant tests and run then them using :code:`tox` before making the pull request. 38 | Docs will be updated automatically when we merge to `master`, but 39 | you should also build the docs yourself via :code:`tox -e docs`, making sure that the docs build OK 40 | and that they are readable. 41 | 42 | .. _`tox`: https://tox.readthedocs.org/en/latest/ 43 | 44 | Make the pull request 45 | --------------------- 46 | 47 | Once you have made all your changes, tested, and updated the documentation, 48 | make a pull request to move everything back into the main `gax-python`_ 49 | repository. Be sure to reference the original issue in the pull request. 50 | Expect some back-and-forth with regards to style and compliance of these 51 | rules. 52 | 53 | Using a Development Checkout 54 | ---------------------------- 55 | 56 | You’ll have to create a development environment to hack on 57 | `gax-python`_, using a Git checkout: 58 | 59 | - While logged into your GitHub account, navigate to the `gax-python repo`_ on GitHub. 60 | - Fork and clone the `gax-python` repository to your GitHub account 61 | by clicking the "Fork" button. 62 | - Clone your fork of `gax-python` from your GitHub account to your 63 | local computer, substituting your account username and specifying 64 | the destination as `hack-on-gax-python`. For example: 65 | 66 | .. code:: bash 67 | 68 | cd ${HOME} 69 | git clone git@github.com:USERNAME/gax-python.git hack-on-gax-python 70 | cd hack-on-gax-python 71 | 72 | # Configure remotes such that you can pull changes from the gax-python 73 | # repository into your local repository. 74 | git remote add upstream https://github.com:google/gax-python 75 | 76 | # fetch and merge changes from upstream into master 77 | git fetch upstream 78 | git merge upstream/master 79 | 80 | 81 | Now your local repo is set up such that you will push changes to your 82 | GitHub repo, from which you can submit a pull request. 83 | 84 | - Create use tox to create development virtualenv in which `gax-python`_ is installed: 85 | 86 | .. code:: bash 87 | 88 | sudo pip install tox 89 | cd ~/hack-on-gax-python 90 | tox -e devenv 91 | 92 | - This is creates a tox virtualenv named `development` that has gax-python installed. 93 | Activate it to use gax-python locally, e.g, from the python prompt. 94 | 95 | .. code:: bash 96 | 97 | cd ~/hack-on-gax-python 98 | . ./tox/develop/bin/activate 99 | 100 | .. _`gax-python`: https://github.com/googleapis/gax-python 101 | .. _`gax-python repo`: https://github.com/googleapis/gax-python 102 | 103 | 104 | Running Tests 105 | ------------- 106 | 107 | - To run the full set of `gax-python` tests on all platforms, install 108 | `tox`_ into a system Python. The :code:`tox` console script will be 109 | installed into the scripts location for that Python. While in the 110 | `gax-python` checkout root directory (it contains :code:`tox.ini`), 111 | invoke the `tox` console script. This will read the :code:`tox.ini` file and 112 | execute the tests on multiple Python versions and platforms; while it runs, 113 | it creates a virtualenv for each version/platform combination. For 114 | example: 115 | 116 | .. code:: bash 117 | 118 | sudo pip install tox 119 | cd ~/hack-on-gax-python 120 | tox 121 | 122 | Contributor License Agreements 123 | ------------------------------ 124 | 125 | Before we can accept your pull requests you'll need to sign a Contributor 126 | License Agreement (CLA): 127 | 128 | - **If you are an individual writing original source code** and **you own 129 | the intellectual property**, then you'll need to sign an 130 | `individual CLA`_. 131 | - **If you work for a company that wants to allow you to contribute your 132 | work**, then you'll need to sign a `corporate CLA`_. 133 | 134 | You can sign these electronically (just scroll to the bottom). After that, 135 | we'll be able to accept your pull requests. 136 | 137 | .. _`individual CLA`: https://developers.google.com/open-source/cla/individual 138 | .. _`corporate CLA`: https://developers.google.com/open-source/cla/corporate 139 | -------------------------------------------------------------------------------- /test/test_path_template.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015, Google Inc. 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are 6 | # met: 7 | # 8 | # * Redistributions of source code must retain the above copyright 9 | # notice, this list of conditions and the following disclaimer. 10 | # * Redistributions in binary form must reproduce the above 11 | # copyright notice, this list of conditions and the following disclaimer 12 | # in the documentation and/or other materials provided with the 13 | # distribution. 14 | # * Neither the name of Google Inc. nor the names of its 15 | # contributors may be used to endorse or promote products derived from 16 | # this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | # pylint: disable=missing-docstring,no-self-use,no-init,invalid-name 31 | """Unit tests for the path_template module.""" 32 | 33 | from __future__ import absolute_import 34 | import unittest2 35 | 36 | from google.gax.path_template import PathTemplate, ValidationException 37 | 38 | 39 | class TestPathTemplate(unittest2.TestCase): 40 | """Unit tests for PathTemplate.""" 41 | 42 | def test_len(self): 43 | self.assertEqual(len(PathTemplate('a/b/**/*/{a=hello/world}')), 6) 44 | 45 | def test_fail_invalid_token(self): 46 | self.assertRaises(ValidationException, 47 | PathTemplate, 'hello/wor*ld') 48 | 49 | def test_fail_when_impossible_match(self): 50 | template = PathTemplate('hello/world') 51 | self.assertRaises(ValidationException, 52 | template.match, 'hello') 53 | template = PathTemplate('hello/world') 54 | self.assertRaises(ValidationException, 55 | template.match, 'hello/world/fail') 56 | 57 | def test_fail_mismatched_literal(self): 58 | template = PathTemplate('hello/world') 59 | self.assertRaises(ValidationException, 60 | template.match, 'hello/world2') 61 | 62 | def test_fail_when_multiple_path_wildcards(self): 63 | self.assertRaises(ValidationException, 64 | PathTemplate, 'buckets/*/**/**/objects/*') 65 | 66 | def test_fail_if_inner_binding(self): 67 | self.assertRaises(ValidationException, 68 | PathTemplate, 'buckets/{hello={world}}') 69 | 70 | def test_fail_unexpected_eof(self): 71 | self.assertRaises(ValidationException, 72 | PathTemplate, 'a/{hello=world') 73 | 74 | def test_match_atomic_resource_name(self): 75 | template = PathTemplate('buckets/*/*/objects/*') 76 | self.assertEqual({'$0': 'f', '$1': 'o', '$2': 'bar'}, 77 | template.match('buckets/f/o/objects/bar')) 78 | template = PathTemplate('/buckets/{hello}') 79 | self.assertEqual({'hello': 'world'}, 80 | template.match('buckets/world')) 81 | template = PathTemplate('/buckets/{hello=*}') 82 | self.assertEqual({'hello': 'world'}, 83 | template.match('buckets/world')) 84 | 85 | def test_match_escaped_chars(self): 86 | template = PathTemplate('buckets/*/objects') 87 | self.assertEqual({'$0': 'hello%2F%2Bworld'}, 88 | template.match('buckets/hello%2F%2Bworld/objects')) 89 | 90 | def test_match_template_with_unbounded_wildcard(self): 91 | template = PathTemplate('buckets/*/objects/**') 92 | self.assertEqual({'$0': 'foo', '$1': 'bar/baz'}, 93 | template.match('buckets/foo/objects/bar/baz')) 94 | 95 | def test_match_with_unbound_in_middle(self): 96 | template = PathTemplate('bar/**/foo/*') 97 | self.assertEqual({'$0': 'foo/foo', '$1': 'bar'}, 98 | template.match('bar/foo/foo/foo/bar')) 99 | 100 | def test_render_atomic_resource(self): 101 | template = PathTemplate('buckets/*/*/*/objects/*') 102 | url = template.render({ 103 | '$0': 'f', '$1': 'o', '$2': 'o', '$3': 'google.com:a-b'}) 104 | self.assertEqual(url, 'buckets/f/o/o/objects/google.com:a-b') 105 | 106 | def test_render_fail_when_too_few_variables(self): 107 | template = PathTemplate('buckets/*/*/*/objects/*') 108 | self.assertRaises(ValidationException, 109 | template.render, 110 | {'$0': 'f', '$1': 'l', '$2': 'o'}) 111 | 112 | def test_render_with_unbound_in_middle(self): 113 | template = PathTemplate('bar/**/foo/*') 114 | url = template.render({'$0': '1/2', '$1': '3'}) 115 | self.assertEqual(url, 'bar/1/2/foo/3') 116 | 117 | def test_to_string(self): 118 | template = PathTemplate('bar/**/foo/*') 119 | self.assertEqual(str(template), 'bar/{$0=**}/foo/{$1=*}') 120 | template = PathTemplate('buckets/*/objects/*') 121 | self.assertEqual(str(template), 'buckets/{$0=*}/objects/{$1=*}') 122 | template = PathTemplate('/buckets/{hello}') 123 | self.assertEqual(str(template), 'buckets/{hello=*}') 124 | template = PathTemplate('/buckets/{hello=what}/{world}') 125 | self.assertEqual(str(template), 'buckets/{hello=what}/{world=*}') 126 | template = PathTemplate('/buckets/helloazAZ09-.~_what') 127 | self.assertEqual(str(template), 'buckets/helloazAZ09-.~_what') 128 | -------------------------------------------------------------------------------- /test/test_grpc.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016, Google Inc. 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are 6 | # met: 7 | # 8 | # * Redistributions of source code must retain the above copyright 9 | # notice, this list of conditions and the following disclaimer. 10 | # * Redistributions in binary form must reproduce the above 11 | # copyright notice, this list of conditions and the following disclaimer 12 | # in the documentation and/or other materials provided with the 13 | # distribution. 14 | # * Neither the name of Google Inc. nor the names of its 15 | # contributors may be used to endorse or promote products derived from 16 | # this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | # pylint: disable=missing-docstring,no-self-use,no-init,invalid-name 31 | """Unit tests for grpc.""" 32 | 33 | from __future__ import absolute_import 34 | 35 | import mock 36 | import unittest2 37 | 38 | from google.gax import grpc 39 | 40 | 41 | def _fake_create_stub(channel): 42 | return channel 43 | 44 | 45 | class TestCreateStub(unittest2.TestCase): 46 | FAKE_SERVICE_PATH = 'service_path' 47 | FAKE_PORT = 10101 48 | 49 | @mock.patch('grpc.beta.implementations.composite_channel_credentials') 50 | @mock.patch('grpc.beta.implementations.ssl_channel_credentials') 51 | @mock.patch('grpc.beta.implementations.secure_channel') 52 | @mock.patch('google.gax.auth.make_auth_func') 53 | def test_creates_a_stub_ok_with_no_scopes( 54 | self, auth, chan, chan_creds, comp): 55 | got_channel = grpc.create_stub( 56 | _fake_create_stub, self.FAKE_SERVICE_PATH, self.FAKE_PORT) 57 | chan_creds.assert_called_once_with(None, None, None) 58 | chan.assert_called_once_with(self.FAKE_SERVICE_PATH, self.FAKE_PORT, 59 | comp.return_value) 60 | auth.assert_called_once_with([]) 61 | self.assertEquals(got_channel, chan.return_value) 62 | 63 | @mock.patch('grpc.beta.implementations.composite_channel_credentials') 64 | @mock.patch('grpc.beta.implementations.ssl_channel_credentials') 65 | @mock.patch('grpc.beta.implementations.secure_channel') 66 | @mock.patch('google.gax.auth.make_auth_func') 67 | def test_creates_a_stub_ok_with_scopes( 68 | self, auth, chan, chan_creds, comp): 69 | fake_scopes = ['dummy', 'scopes'] 70 | grpc.create_stub( 71 | _fake_create_stub, self.FAKE_SERVICE_PATH, self.FAKE_PORT, 72 | scopes=fake_scopes) 73 | chan_creds.assert_called_once_with(None, None, None) 74 | chan.assert_called_once_with(self.FAKE_SERVICE_PATH, self.FAKE_PORT, 75 | comp.return_value) 76 | auth.assert_called_once_with(fake_scopes) 77 | 78 | @mock.patch('grpc.beta.implementations.metadata_call_credentials') 79 | @mock.patch('grpc.beta.implementations.composite_channel_credentials') 80 | @mock.patch('grpc.beta.implementations.ssl_channel_credentials') 81 | @mock.patch('grpc.beta.implementations.secure_channel') 82 | @mock.patch('google.gax.auth.make_auth_func') 83 | def test_creates_a_stub_with_given_channel( 84 | self, auth, chan, chan_creds, comp, md): 85 | fake_channel = object() 86 | got_channel = grpc.create_stub( 87 | _fake_create_stub, self.FAKE_SERVICE_PATH, self.FAKE_PORT, 88 | channel=fake_channel) 89 | self.assertEquals(got_channel, fake_channel) 90 | self.assertFalse(auth.called) 91 | self.assertFalse(chan_creds.called) 92 | self.assertFalse(chan.called) 93 | self.assertFalse(comp.called) 94 | self.assertFalse(md.called) 95 | 96 | @mock.patch('grpc.beta.implementations.metadata_call_credentials') 97 | @mock.patch('grpc.beta.implementations.composite_channel_credentials') 98 | @mock.patch('grpc.beta.implementations.ssl_channel_credentials') 99 | @mock.patch('grpc.beta.implementations.secure_channel') 100 | @mock.patch('google.gax.auth.make_auth_func') 101 | def test_creates_a_stub_ok_with_given_creds(self, auth, chan, chan_creds, 102 | comp, md): 103 | fake_creds = object() 104 | got_channel = grpc.create_stub( 105 | _fake_create_stub, self.FAKE_SERVICE_PATH, self.FAKE_PORT, 106 | ssl_creds=fake_creds) 107 | chan.assert_called_once_with(self.FAKE_SERVICE_PATH, self.FAKE_PORT, 108 | comp.return_value) 109 | auth.assert_called_once_with([]) 110 | self.assertTrue(chan.called) 111 | self.assertFalse(chan_creds.called) 112 | self.assertTrue(comp.called) 113 | self.assertTrue(md.called) 114 | self.assertEquals(got_channel, chan.return_value) 115 | 116 | @mock.patch('grpc.beta.implementations.composite_channel_credentials') 117 | @mock.patch('grpc.beta.implementations.ssl_channel_credentials') 118 | @mock.patch('grpc.beta.implementations.secure_channel') 119 | @mock.patch('google.gax.auth.make_auth_func') 120 | def test_creates_a_stub_ok_with_given_auth_func(self, auth, dummy_chan, 121 | dummy_chan_creds, dummy_md): 122 | grpc.create_stub( 123 | _fake_create_stub, self.FAKE_SERVICE_PATH, self.FAKE_PORT, 124 | metadata_transformer=lambda x: tuple()) 125 | self.assertFalse(auth.called) 126 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " applehelp to make an Apple Help Book" 34 | @echo " devhelp to make HTML files and a Devhelp project" 35 | @echo " epub to make an epub" 36 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 37 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 38 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 39 | @echo " text to make text files" 40 | @echo " man to make manual pages" 41 | @echo " texinfo to make Texinfo files" 42 | @echo " info to make Texinfo files and run them through makeinfo" 43 | @echo " gettext to make PO message catalogs" 44 | @echo " changes to make an overview of all changed/added/deprecated items" 45 | @echo " xml to make Docutils-native XML files" 46 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 47 | @echo " linkcheck to check all external links for integrity" 48 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 49 | @echo " coverage to run coverage check of the documentation (if enabled)" 50 | 51 | clean: 52 | rm -rf $(BUILDDIR)/* 53 | 54 | html: 55 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 56 | @echo 57 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 58 | 59 | dirhtml: 60 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 61 | @echo 62 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 63 | 64 | singlehtml: 65 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 66 | @echo 67 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 68 | 69 | pickle: 70 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 71 | @echo 72 | @echo "Build finished; now you can process the pickle files." 73 | 74 | json: 75 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 76 | @echo 77 | @echo "Build finished; now you can process the JSON files." 78 | 79 | htmlhelp: 80 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 81 | @echo 82 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 83 | ".hhp project file in $(BUILDDIR)/htmlhelp." 84 | 85 | qthelp: 86 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 87 | @echo 88 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 89 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 90 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/nurpc-hyper.qhcp" 91 | @echo "To view the help file:" 92 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/nurpc-hyper.qhc" 93 | 94 | applehelp: 95 | $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp 96 | @echo 97 | @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." 98 | @echo "N.B. You won't be able to view it unless you put it in" \ 99 | "~/Library/Documentation/Help or install it in your application" \ 100 | "bundle." 101 | 102 | devhelp: 103 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 104 | @echo 105 | @echo "Build finished." 106 | @echo "To view the help file:" 107 | @echo "# mkdir -p $$HOME/.local/share/devhelp/nurpc-hyper" 108 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/nurpc-hyper" 109 | @echo "# devhelp" 110 | 111 | epub: 112 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 113 | @echo 114 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 115 | 116 | latex: 117 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 118 | @echo 119 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 120 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 121 | "(use \`make latexpdf' here to do that automatically)." 122 | 123 | latexpdf: 124 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 125 | @echo "Running LaTeX files through pdflatex..." 126 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 127 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 128 | 129 | latexpdfja: 130 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 131 | @echo "Running LaTeX files through platex and dvipdfmx..." 132 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 133 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 134 | 135 | text: 136 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 137 | @echo 138 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 139 | 140 | man: 141 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 142 | @echo 143 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 144 | 145 | texinfo: 146 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 147 | @echo 148 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 149 | @echo "Run \`make' in that directory to run these through makeinfo" \ 150 | "(use \`make info' here to do that automatically)." 151 | 152 | info: 153 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 154 | @echo "Running Texinfo files through makeinfo..." 155 | make -C $(BUILDDIR)/texinfo info 156 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 157 | 158 | gettext: 159 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 160 | @echo 161 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 162 | 163 | changes: 164 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 165 | @echo 166 | @echo "The overview file is in $(BUILDDIR)/changes." 167 | 168 | linkcheck: 169 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 170 | @echo 171 | @echo "Link check complete; look for any errors in the above output " \ 172 | "or in $(BUILDDIR)/linkcheck/output.txt." 173 | 174 | doctest: 175 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 176 | @echo "Testing of doctests in the sources finished, look at the " \ 177 | "results in $(BUILDDIR)/doctest/output.txt." 178 | 179 | coverage: 180 | $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage 181 | @echo "Testing of coverage in the sources finished, look at the " \ 182 | "results in $(BUILDDIR)/coverage/python.txt." 183 | 184 | xml: 185 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 186 | @echo 187 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 188 | 189 | pseudoxml: 190 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 191 | @echo 192 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 193 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. xml to make Docutils-native XML files 37 | echo. pseudoxml to make pseudoxml-XML files for display purposes 38 | echo. linkcheck to check all external links for integrity 39 | echo. doctest to run all doctests embedded in the documentation if enabled 40 | echo. coverage to run coverage check of the documentation if enabled 41 | goto end 42 | ) 43 | 44 | if "%1" == "clean" ( 45 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 46 | del /q /s %BUILDDIR%\* 47 | goto end 48 | ) 49 | 50 | 51 | REM Check if sphinx-build is available and fallback to Python version if any 52 | %SPHINXBUILD% 1>NUL 2>NUL 53 | if errorlevel 9009 goto sphinx_python 54 | goto sphinx_ok 55 | 56 | :sphinx_python 57 | 58 | set SPHINXBUILD=python -m sphinx.__init__ 59 | %SPHINXBUILD% 2> nul 60 | if errorlevel 9009 ( 61 | echo. 62 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 63 | echo.installed, then set the SPHINXBUILD environment variable to point 64 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 65 | echo.may add the Sphinx directory to PATH. 66 | echo. 67 | echo.If you don't have Sphinx installed, grab it from 68 | echo.http://sphinx-doc.org/ 69 | exit /b 1 70 | ) 71 | 72 | :sphinx_ok 73 | 74 | 75 | if "%1" == "html" ( 76 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 77 | if errorlevel 1 exit /b 1 78 | echo. 79 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 80 | goto end 81 | ) 82 | 83 | if "%1" == "dirhtml" ( 84 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 85 | if errorlevel 1 exit /b 1 86 | echo. 87 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 88 | goto end 89 | ) 90 | 91 | if "%1" == "singlehtml" ( 92 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 93 | if errorlevel 1 exit /b 1 94 | echo. 95 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 96 | goto end 97 | ) 98 | 99 | if "%1" == "pickle" ( 100 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 101 | if errorlevel 1 exit /b 1 102 | echo. 103 | echo.Build finished; now you can process the pickle files. 104 | goto end 105 | ) 106 | 107 | if "%1" == "json" ( 108 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 109 | if errorlevel 1 exit /b 1 110 | echo. 111 | echo.Build finished; now you can process the JSON files. 112 | goto end 113 | ) 114 | 115 | if "%1" == "htmlhelp" ( 116 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 117 | if errorlevel 1 exit /b 1 118 | echo. 119 | echo.Build finished; now you can run HTML Help Workshop with the ^ 120 | .hhp project file in %BUILDDIR%/htmlhelp. 121 | goto end 122 | ) 123 | 124 | if "%1" == "qthelp" ( 125 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 129 | .qhcp project file in %BUILDDIR%/qthelp, like this: 130 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\nurpc-hyper.qhcp 131 | echo.To view the help file: 132 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\nurpc-hyper.ghc 133 | goto end 134 | ) 135 | 136 | if "%1" == "devhelp" ( 137 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 138 | if errorlevel 1 exit /b 1 139 | echo. 140 | echo.Build finished. 141 | goto end 142 | ) 143 | 144 | if "%1" == "epub" ( 145 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 146 | if errorlevel 1 exit /b 1 147 | echo. 148 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 149 | goto end 150 | ) 151 | 152 | if "%1" == "latex" ( 153 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 154 | if errorlevel 1 exit /b 1 155 | echo. 156 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 157 | goto end 158 | ) 159 | 160 | if "%1" == "latexpdf" ( 161 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 162 | cd %BUILDDIR%/latex 163 | make all-pdf 164 | cd %~dp0 165 | echo. 166 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 167 | goto end 168 | ) 169 | 170 | if "%1" == "latexpdfja" ( 171 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 172 | cd %BUILDDIR%/latex 173 | make all-pdf-ja 174 | cd %~dp0 175 | echo. 176 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 177 | goto end 178 | ) 179 | 180 | if "%1" == "text" ( 181 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 182 | if errorlevel 1 exit /b 1 183 | echo. 184 | echo.Build finished. The text files are in %BUILDDIR%/text. 185 | goto end 186 | ) 187 | 188 | if "%1" == "man" ( 189 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 190 | if errorlevel 1 exit /b 1 191 | echo. 192 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 193 | goto end 194 | ) 195 | 196 | if "%1" == "texinfo" ( 197 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 198 | if errorlevel 1 exit /b 1 199 | echo. 200 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 201 | goto end 202 | ) 203 | 204 | if "%1" == "gettext" ( 205 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 206 | if errorlevel 1 exit /b 1 207 | echo. 208 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 209 | goto end 210 | ) 211 | 212 | if "%1" == "changes" ( 213 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 214 | if errorlevel 1 exit /b 1 215 | echo. 216 | echo.The overview file is in %BUILDDIR%/changes. 217 | goto end 218 | ) 219 | 220 | if "%1" == "linkcheck" ( 221 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 222 | if errorlevel 1 exit /b 1 223 | echo. 224 | echo.Link check complete; look for any errors in the above output ^ 225 | or in %BUILDDIR%/linkcheck/output.txt. 226 | goto end 227 | ) 228 | 229 | if "%1" == "doctest" ( 230 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 231 | if errorlevel 1 exit /b 1 232 | echo. 233 | echo.Testing of doctests in the sources finished, look at the ^ 234 | results in %BUILDDIR%/doctest/output.txt. 235 | goto end 236 | ) 237 | 238 | if "%1" == "coverage" ( 239 | %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage 240 | if errorlevel 1 exit /b 1 241 | echo. 242 | echo.Testing of coverage in the sources finished, look at the ^ 243 | results in %BUILDDIR%/coverage/python.txt. 244 | goto end 245 | ) 246 | 247 | if "%1" == "xml" ( 248 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 249 | if errorlevel 1 exit /b 1 250 | echo. 251 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 252 | goto end 253 | ) 254 | 255 | if "%1" == "pseudoxml" ( 256 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 257 | if errorlevel 1 exit /b 1 258 | echo. 259 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 260 | goto end 261 | ) 262 | 263 | :end 264 | -------------------------------------------------------------------------------- /google/gax/path_template.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016, Google Inc. 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are 6 | # met: 7 | # 8 | # * Redistributions of source code must retain the above copyright 9 | # notice, this list of conditions and the following disclaimer. 10 | # * Redistributions in binary form must reproduce the above 11 | # copyright notice, this list of conditions and the following disclaimer 12 | # in the documentation and/or other materials provided with the 13 | # distribution. 14 | # * Neither the name of Google Inc. nor the names of its 15 | # contributors may be used to endorse or promote products derived from 16 | # this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | """Implements a utility for parsing and formatting path templates.""" 31 | 32 | from __future__ import absolute_import 33 | from collections import namedtuple 34 | 35 | from ply import lex, yacc 36 | 37 | _BINDING = 1 38 | _END_BINDING = 2 39 | _TERMINAL = 3 40 | _Segment = namedtuple('_Segment', ['kind', 'literal']) 41 | 42 | 43 | def _format(segments): 44 | template = '' 45 | slash = True 46 | for segment in segments: 47 | if segment.kind == _TERMINAL: 48 | if slash: 49 | template += '/' 50 | template += segment.literal 51 | slash = True 52 | if segment.kind == _BINDING: 53 | template += '/{%s=' % segment.literal 54 | slash = False 55 | if segment.kind == _END_BINDING: 56 | template += '%s}' % segment.literal 57 | return template[1:] # Remove the leading / 58 | 59 | 60 | class ValidationException(Exception): 61 | """Represents a path template validation error.""" 62 | pass 63 | 64 | 65 | class PathTemplate(object): 66 | """Represents a path template.""" 67 | 68 | segments = None 69 | segment_count = 0 70 | 71 | def __init__(self, data): 72 | parser = _Parser() 73 | self.segments = parser.parse(data) 74 | self.segment_count = parser.segment_count 75 | 76 | def __len__(self): 77 | return self.segment_count 78 | 79 | def __repr__(self): 80 | return _format(self.segments) 81 | 82 | def render(self, bindings): 83 | """Renders a string from a path template using the provided bindings. 84 | 85 | Args: 86 | bindings (dict): A dictionary of var names to binding strings. 87 | 88 | Returns: 89 | str: The rendered instantiation of this path template. 90 | 91 | Raises: 92 | ValidationError: If a key isn't provided or if a sub-template can't 93 | be parsed. 94 | """ 95 | out = [] 96 | binding = False 97 | for segment in self.segments: 98 | if segment.kind == _BINDING: 99 | if segment.literal not in bindings: 100 | raise ValidationException( 101 | ('rendering error: value for key \'{}\' ' 102 | 'not provided').format(segment.literal)) 103 | out.extend(PathTemplate(bindings[segment.literal]).segments) 104 | binding = True 105 | elif segment.kind == _END_BINDING: 106 | binding = False 107 | else: 108 | if binding: 109 | continue 110 | out.append(segment) 111 | path = _format(out) 112 | self.match(path) 113 | return path 114 | 115 | def match(self, path): 116 | """Matches a fully qualified path template string. 117 | 118 | Args: 119 | path (str): A fully qualified path template string. 120 | 121 | Returns: 122 | dict: Var names to matched binding values. 123 | 124 | Raises: 125 | ValidationException: If path can't be matched to the template. 126 | """ 127 | this = self.segments 128 | that = path.split('/') 129 | current_var = None 130 | bindings = {} 131 | segment_count = self.segment_count 132 | j = 0 133 | for i in range(0, len(this)): 134 | if j >= len(that): 135 | break 136 | if this[i].kind == _TERMINAL: 137 | if this[i].literal == '*': 138 | bindings[current_var] = that[j] 139 | j += 1 140 | elif this[i].literal == '**': 141 | until = j + len(that) - segment_count + 1 142 | segment_count += len(that) - segment_count 143 | bindings[current_var] = '/'.join(that[j:until]) 144 | j = until 145 | elif this[i].literal != that[j]: 146 | raise ValidationException( 147 | 'mismatched literal: \'%s\' != \'%s\'' % ( 148 | this[i].literal, that[j])) 149 | else: 150 | j += 1 151 | elif this[i].kind == _BINDING: 152 | current_var = this[i].literal 153 | if j != len(that) or j != segment_count: 154 | raise ValidationException( 155 | 'match error: could not render from the path template: {}' 156 | .format(path)) 157 | return bindings 158 | 159 | 160 | # pylint: disable=C0103 161 | # pylint: disable=R0201 162 | class _Parser(object): 163 | tokens = ( 164 | 'FORWARD_SLASH', 165 | 'LEFT_BRACE', 166 | 'RIGHT_BRACE', 167 | 'EQUALS', 168 | 'WILDCARD', 169 | 'PATH_WILDCARD', 170 | 'LITERAL', 171 | ) 172 | 173 | t_FORWARD_SLASH = r'/' 174 | t_LEFT_BRACE = r'\{' 175 | t_RIGHT_BRACE = r'\}' 176 | t_EQUALS = r'=' 177 | t_WILDCARD = r'\*' 178 | t_PATH_WILDCARD = r'\*\*' 179 | t_LITERAL = r'[^*=}{}\/]+' 180 | 181 | t_ignore = ' \t' 182 | 183 | binding_var_count = 0 184 | segment_count = 0 185 | 186 | def __init__(self): 187 | self.lexer = lex.lex(module=self) 188 | self.parser = yacc.yacc(module=self, debug=False, write_tables=False) 189 | 190 | def parse(self, data): 191 | """Returns a list of path template segments parsed from data. 192 | 193 | Args: 194 | data: A path template string. 195 | Returns: 196 | A list of _Segment. 197 | """ 198 | self.binding_var_count = 0 199 | self.segment_count = 0 200 | 201 | segments = self.parser.parse(data) 202 | # Validation step: checks that there are no nested bindings. 203 | path_wildcard = False 204 | for segment in segments: 205 | if segment.kind == _TERMINAL and segment.literal == '**': 206 | if path_wildcard: 207 | raise ValidationException( 208 | 'validation error: path template cannot contain more ' 209 | 'than one path wildcard') 210 | path_wildcard = True 211 | return segments 212 | 213 | def p_template(self, p): 214 | """template : FORWARD_SLASH bound_segments 215 | | bound_segments""" 216 | # ply fails on a negative index. 217 | p[0] = p[len(p) - 1] 218 | 219 | def p_bound_segments(self, p): 220 | """bound_segments : bound_segment FORWARD_SLASH bound_segments 221 | | bound_segment""" 222 | p[0] = p[1] 223 | if len(p) > 2: 224 | p[0].extend(p[3]) 225 | 226 | def p_unbound_segments(self, p): 227 | """unbound_segments : unbound_terminal FORWARD_SLASH unbound_segments 228 | | unbound_terminal""" 229 | p[0] = p[1] 230 | if len(p) > 2: 231 | p[0].extend(p[3]) 232 | 233 | def p_bound_segment(self, p): 234 | """bound_segment : bound_terminal 235 | | variable""" 236 | p[0] = p[1] 237 | 238 | def p_unbound_terminal(self, p): 239 | """unbound_terminal : WILDCARD 240 | | PATH_WILDCARD 241 | | LITERAL""" 242 | p[0] = [_Segment(_TERMINAL, p[1])] 243 | self.segment_count += 1 244 | 245 | def p_bound_terminal(self, p): 246 | """bound_terminal : unbound_terminal""" 247 | if p[1][0].literal in ['*', '**']: 248 | p[0] = [_Segment(_BINDING, '$%d' % self.binding_var_count), 249 | p[1][0], 250 | _Segment(_END_BINDING, '')] 251 | self.binding_var_count += 1 252 | else: 253 | p[0] = p[1] 254 | 255 | def p_variable(self, p): 256 | """variable : LEFT_BRACE LITERAL EQUALS unbound_segments RIGHT_BRACE 257 | | LEFT_BRACE LITERAL RIGHT_BRACE""" 258 | p[0] = [_Segment(_BINDING, p[2])] 259 | if len(p) > 4: 260 | p[0].extend(p[4]) 261 | else: 262 | p[0].append(_Segment(_TERMINAL, '*')) 263 | self.segment_count += 1 264 | p[0].append(_Segment(_END_BINDING, '')) 265 | 266 | def p_error(self, p): 267 | """Raises a parser error.""" 268 | if p: 269 | raise ValidationException( 270 | 'parser error: unexpected token \'%s\'' % p.type) 271 | else: 272 | raise ValidationException('parser error: unexpected EOF') 273 | 274 | def t_error(self, t): 275 | """Raises a lexer error.""" 276 | raise ValidationException( 277 | 'lexer error: illegal character \'%s\'' % t.value[0]) 278 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Copyright 2015, Google Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are 8 | # met: 9 | # 10 | # * Redistributions of source code must retain the above copyright 11 | # notice, this list of conditions and the following disclaimer. 12 | # * Redistributions in binary form must reproduce the above 13 | # copyright notice, this list of conditions and the following disclaimer 14 | # in the documentation and/or other materials provided with the 15 | # distribution. 16 | # * Neither the name of Google Inc. nor the names of its 17 | # contributors may be used to endorse or promote products derived from 18 | # this software without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | # 33 | # google-gax documentation build configuration file, created by 34 | # sphinx-quickstart on Sun Nov 29 14:22:46 2015. 35 | # 36 | # This file is execfile()d with the current directory set to its 37 | # containing dir. 38 | # 39 | # Note that not all possible configuration values are present in this 40 | # autogenerated file. 41 | # 42 | # All configuration values have a default; values that are commented out 43 | # serve to show the default. 44 | 45 | import sys 46 | import os 47 | import shlex 48 | 49 | # If extensions (or modules to document with autodoc) are in another directory, 50 | # add these directories to sys.path here. If the directory is relative to the 51 | # documentation root, use os.path.abspath to make it absolute, like shown here. 52 | sys.path.insert(0, os.path.abspath('..')) 53 | 54 | from google.gax import __version__ 55 | 56 | # -- General configuration ------------------------------------------------ 57 | 58 | # If your documentation needs a minimal Sphinx version, state it here. 59 | #needs_sphinx = '1.0' 60 | 61 | # Add any Sphinx extension module names here, as strings. They can be 62 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 63 | # ones. 64 | extensions = [ 65 | 'sphinx.ext.autodoc', 66 | 'sphinx.ext.autosummary', 67 | 'sphinx.ext.intersphinx', 68 | 'sphinx.ext.coverage', 69 | 'sphinx.ext.napoleon', 70 | 'sphinx.ext.viewcode', 71 | ] 72 | 73 | # autodoc/autosummary flags 74 | autoclass_content = 'both' 75 | autodoc_default_flags = ['members'] 76 | autosummary_generate = True 77 | 78 | # Add any paths that contain templates here, relative to this directory. 79 | templates_path = ['_templates'] 80 | 81 | # The suffix(es) of source filenames. 82 | # You can specify multiple suffix as a list of string: 83 | # source_suffix = ['.rst', '.md'] 84 | source_suffix = '.rst' 85 | 86 | # The encoding of source files. 87 | #source_encoding = 'utf-8-sig' 88 | 89 | # The master toctree document. 90 | master_doc = 'index' 91 | 92 | # General information about the project. 93 | project = u'google-gax' 94 | copyright = u'2015, Google' 95 | author = u'Google Inc' 96 | 97 | # The version info for the project you're documenting, acts as replacement for 98 | # |version| and |release|, also used in various other places throughout the 99 | # built documents. 100 | # 101 | # The full version, including alpha/beta/rc tags. 102 | release = __version__ 103 | # The short X.Y version. 104 | version = '.'.join(release.split('.')[0:2]) 105 | 106 | # The language for content autogenerated by Sphinx. Refer to documentation 107 | # for a list of supported languages. 108 | # 109 | # This is also used if you do content translation via gettext catalogs. 110 | # Usually you set "language" from the command line for these cases. 111 | language = None 112 | 113 | # There are two options for replacing |today|: either, you set today to some 114 | # non-false value, then it is used: 115 | #today = '' 116 | # Else, today_fmt is used as the format for a strftime call. 117 | #today_fmt = '%B %d, %Y' 118 | 119 | # List of patterns, relative to source directory, that match files and 120 | # directories to ignore when looking for source files. 121 | exclude_patterns = ['_build'] 122 | 123 | # The reST default role (used for this markup: `text`) to use for all 124 | # documents. 125 | #default_role = None 126 | 127 | # If true, '()' will be appended to :func: etc. cross-reference text. 128 | #add_function_parentheses = True 129 | 130 | # If true, the current module name will be prepended to all description 131 | # unit titles (such as .. function::). 132 | #add_module_names = True 133 | 134 | # If true, sectionauthor and moduleauthor directives will be shown in the 135 | # output. They are ignored by default. 136 | #show_authors = False 137 | 138 | # The name of the Pygments (syntax highlighting) style to use. 139 | pygments_style = 'sphinx' 140 | 141 | # A list of ignored prefixes for module index sorting. 142 | #modindex_common_prefix = [] 143 | 144 | # If true, keep warnings as "system message" paragraphs in the built documents. 145 | #keep_warnings = False 146 | 147 | # If true, `todo` and `todoList` produce output, else they produce nothing. 148 | todo_include_todos = True 149 | 150 | 151 | # -- Options for HTML output ---------------------------------------------- 152 | 153 | # The theme to use for HTML and HTML Help pages. See the documentation for 154 | # a list of builtin themes. 155 | html_theme = 'sphinx_rtd_theme' 156 | 157 | # Theme options are theme-specific and customize the look and feel of a theme 158 | # further. For a list of options available for each theme, see the 159 | # documentation. 160 | #html_theme_options = {} 161 | 162 | # Add any paths that contain custom themes here, relative to this directory. 163 | #html_theme_path = [] 164 | 165 | # The name for this set of Sphinx documents. If None, it defaults to 166 | # " v documentation". 167 | #html_title = None 168 | 169 | # A shorter title for the navigation bar. Default is the same as html_title. 170 | #html_short_title = None 171 | 172 | # The name of an image file (relative to this directory) to place at the top 173 | # of the sidebar. 174 | #html_logo = None 175 | 176 | # The name of an image file (within the static path) to use as favicon of the 177 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 178 | # pixels large. 179 | #html_favicon = None 180 | 181 | # Add any paths that contain custom static files (such as style sheets) here, 182 | # relative to this directory. They are copied after the builtin static files, 183 | # so a file named "default.css" will overwrite the builtin "default.css". 184 | html_static_path = ['_static'] 185 | 186 | # Add any extra paths that contain custom files (such as robots.txt or 187 | # .htaccess) here, relative to this directory. These files are copied 188 | # directly to the root of the documentation. 189 | #html_extra_path = [] 190 | 191 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 192 | # using the given strftime format. 193 | #html_last_updated_fmt = '%b %d, %Y' 194 | 195 | # If true, SmartyPants will be used to convert quotes and dashes to 196 | # typographically correct entities. 197 | #html_use_smartypants = True 198 | 199 | # Custom sidebar templates, maps document names to template names. 200 | #html_sidebars = {} 201 | 202 | # Additional templates that should be rendered to pages, maps page names to 203 | # template names. 204 | #html_additional_pages = {} 205 | 206 | # If false, no module index is generated. 207 | #html_domain_indices = True 208 | 209 | # If false, no index is generated. 210 | #html_use_index = True 211 | 212 | # If true, the index is split into individual pages for each letter. 213 | #html_split_index = False 214 | 215 | # If true, links to the reST sources are added to the pages. 216 | #html_show_sourcelink = True 217 | 218 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 219 | #html_show_sphinx = True 220 | 221 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 222 | #html_show_copyright = True 223 | 224 | # If true, an OpenSearch description file will be output, and all pages will 225 | # contain a tag referring to it. The value of this option must be the 226 | # base URL from which the finished HTML is served. 227 | #html_use_opensearch = '' 228 | 229 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 230 | #html_file_suffix = None 231 | 232 | # Language to be used for generating the HTML full-text search index. 233 | # Sphinx supports the following languages: 234 | # 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' 235 | # 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' 236 | #html_search_language = 'en' 237 | 238 | # A dictionary with options for the search language support, empty by default. 239 | # Now only 'ja' uses this config value 240 | #html_search_options = {'type': 'default'} 241 | 242 | # The name of a javascript file (relative to the configuration directory) that 243 | # implements a search results scorer. If empty, the default will be used. 244 | #html_search_scorer = 'scorer.js' 245 | 246 | # Output file base name for HTML help builder. 247 | htmlhelp_basename = 'google-gaxdoc' 248 | 249 | # -- Options for LaTeX output --------------------------------------------- 250 | 251 | latex_elements = { 252 | # The paper size ('letterpaper' or 'a4paper'). 253 | #'papersize': 'letterpaper', 254 | 255 | # The font size ('10pt', '11pt' or '12pt'). 256 | #'pointsize': '10pt', 257 | 258 | # Additional stuff for the LaTeX preamble. 259 | #'preamble': '', 260 | 261 | # Latex figure (float) alignment 262 | #'figure_align': 'htbp', 263 | } 264 | 265 | # Grouping the document tree into LaTeX files. List of tuples 266 | # (source start file, target name, title, 267 | # author, documentclass [howto, manual, or own class]). 268 | latex_documents = [ 269 | (master_doc, 'google-gax.tex', u'Google API eXtension Documentation', 270 | u'Tim Emiola', 'manual'), 271 | ] 272 | 273 | # The name of an image file (relative to this directory) to place at the top of 274 | # the title page. 275 | #latex_logo = None 276 | 277 | # For "manual" documents, if this is true, then toplevel headings are parts, 278 | # not chapters. 279 | #latex_use_parts = False 280 | 281 | # If true, show page references after internal links. 282 | #latex_show_pagerefs = False 283 | 284 | # If true, show URL addresses after external links. 285 | #latex_show_urls = False 286 | 287 | # Documents to append as an appendix to all manuals. 288 | #latex_appendices = [] 289 | 290 | # If false, no module index is generated. 291 | #latex_domain_indices = True 292 | 293 | 294 | # -- Options for manual page output --------------------------------------- 295 | 296 | # One entry per manual page. List of tuples 297 | # (source start file, name, description, authors, manual section). 298 | man_pages = [ 299 | (master_doc, 'google-gax', u'Google API eXtension Documentation', 300 | [author], 1) 301 | ] 302 | 303 | # If true, show URL addresses after external links. 304 | #man_show_urls = False 305 | 306 | 307 | # -- Options for Texinfo output ------------------------------------------- 308 | 309 | # Grouping the document tree into Texinfo files. List of tuples 310 | # (source start file, target name, title, author, 311 | # dir menu entry, description, category) 312 | texinfo_documents = [ 313 | (master_doc, 'google-gax', u'Google API eXtension Documentation', 314 | author, 'google-gax', 'Extends Google APIs', 315 | 'Miscellaneous'), 316 | ] 317 | 318 | # Documents to append as an appendix to all manuals. 319 | #texinfo_appendices = [] 320 | 321 | # If false, no module index is generated. 322 | #texinfo_domain_indices = True 323 | 324 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 325 | #texinfo_show_urls = 'footnote' 326 | 327 | # If true, do not generate a @detailmenu in the "Top" node's menu. 328 | #texinfo_no_detailmenu = False 329 | 330 | 331 | # Example configuration for intersphinx: refer to the Python standard library. 332 | intersphinx_mapping = { 333 | 'python': ('http://python.readthedocs.org/en/latest/', None), 334 | } 335 | 336 | 337 | # Napoleon settings 338 | napoleon_google_docstring = True 339 | napoleon_numpy_docstring = True 340 | napoleon_include_private_with_doc = False 341 | napoleon_include_special_with_doc = True 342 | napoleon_use_admonition_for_examples = False 343 | napoleon_use_admonition_for_notes = False 344 | napoleon_use_admonition_for_references = False 345 | napoleon_use_ivar = False 346 | napoleon_use_param = True 347 | napoleon_use_rtype = True 348 | -------------------------------------------------------------------------------- /google/gax/bundling.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016, Google Inc. 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are 6 | # met: 7 | # 8 | # * Redistributions of source code must retain the above copyright 9 | # notice, this list of conditions and the following disclaimer. 10 | # * Redistributions in binary form must reproduce the above 11 | # copyright notice, this list of conditions and the following disclaimer 12 | # in the documentation and/or other materials provided with the 13 | # distribution. 14 | # * Neither the name of Google Inc. nor the names of its 15 | # contributors may be used to endorse or promote products derived from 16 | # this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | """Provides behavior that supports request bundling. 31 | 32 | :func:`compute_bundle_id` is used generate ids linking API requests to the 33 | appropriate bundles. 34 | 35 | :class:`Event` is the result of scheduling a bundled api call. It is a 36 | decorated :class:`threading.Event`; its ``wait`` and ``is_set`` methods 37 | are used wait for the bundle request to complete or determine if it has 38 | been completed respectively. 39 | 40 | :class:`Task` manages the sending of all the requests in a specific bundle. 41 | 42 | :class:`Executor` has a ``schedule`` method that is used add bundled api calls 43 | to a new or existing :class:`Task`. 44 | 45 | """ 46 | 47 | from __future__ import absolute_import 48 | 49 | import collections 50 | import copy 51 | import logging 52 | import threading 53 | 54 | _LOG = logging.getLogger(__name__) 55 | 56 | 57 | def _str_dotted_getattr(obj, name): 58 | """Expands extends getattr to allow dots in x to indicate nested objects. 59 | 60 | Args: 61 | obj: an object. 62 | name: a name for a field in the object. 63 | 64 | Returns: 65 | the value of named attribute. 66 | 67 | Raises: 68 | AttributeError if the named attribute does not exist. 69 | """ 70 | if name.find('.') == -1: 71 | return getattr(obj, name) 72 | for part in name.split('.'): 73 | obj = getattr(obj, part) 74 | return str(obj) if obj is not None else None 75 | 76 | 77 | def compute_bundle_id(obj, discriminator_fields): 78 | """Computes a bundle id from the discriminator fields of `obj`. 79 | 80 | discriminator_fields may include '.' as a separator, which is used to 81 | indicate object traversal. This is meant to allow fields in the 82 | computed bundle_id. 83 | 84 | the id is a tuple computed by going through the discriminator fields in 85 | order and obtaining the str(value) object field (or nested object field) 86 | 87 | if any discriminator field cannot be found, ValueError is raised. 88 | 89 | Args: 90 | obj: an object. 91 | discriminator_fields: a list of discriminator fields in the order to be 92 | to be used in the id. 93 | 94 | Returns: 95 | tuple: computed as described above. 96 | 97 | Raises: 98 | AttributeError: if any discriminator fields attribute does not exist. 99 | """ 100 | return tuple(_str_dotted_getattr(obj, x) for x in discriminator_fields) 101 | 102 | 103 | _WARN_DEMUX_MISMATCH = ('cannot demultiplex the bundled response, got' 104 | ' %d subresponses; want %d, each bundled request will' 105 | ' receive all responses') 106 | 107 | 108 | class Task(object): 109 | """Coordinates the execution of a single bundle.""" 110 | # pylint: disable=too-many-instance-attributes 111 | 112 | def __init__(self, api_call, bundle_id, bundled_field, bundling_request, 113 | kwargs, subresponse_field=None): 114 | """Constructor. 115 | 116 | Args: 117 | api_call (callable[[object], object]): the func that is this tasks's 118 | API call. 119 | bundle_id (tuple): the id of this bundle. 120 | bundle_field (str): the field used to create the bundled request. 121 | bundling_request (object): the request to pass as the arg to api_call. 122 | kwargs (dict): keyword arguments passed to api_call. 123 | subresponse_field (str): optional field used to demultiplex responses. 124 | 125 | """ 126 | self._api_call = api_call 127 | self._bundling_request = bundling_request 128 | self._kwargs = kwargs 129 | self.bundle_id = bundle_id 130 | self.bundled_field = bundled_field 131 | self.subresponse_field = subresponse_field 132 | self._in_deque = collections.deque() 133 | self._event_deque = collections.deque() 134 | 135 | @property 136 | def element_count(self): 137 | """The number of bundled elements in the repeated field.""" 138 | return sum(len(elts) for elts in self._in_deque) 139 | 140 | @property 141 | def request_bytesize(self): 142 | """The size of in bytes of the bundled field elements.""" 143 | return sum(len(str(e)) for elts in self._in_deque for e in elts) 144 | 145 | def run(self): 146 | """Call the task's func. 147 | 148 | The task's func will be called with the bundling requests func 149 | """ 150 | if len(self._in_deque) == 0: 151 | return 152 | req = self._bundling_request 153 | del getattr(req, self.bundled_field)[:] 154 | getattr(req, self.bundled_field).extend( 155 | [e for elts in self._in_deque for e in elts]) 156 | 157 | subresponse_field = self.subresponse_field 158 | if subresponse_field: 159 | self._run_with_subresponses(req, subresponse_field, self._kwargs) 160 | else: 161 | self._run_with_no_subresponse(req, self._kwargs) 162 | 163 | def _run_with_no_subresponse(self, req, kwargs): 164 | try: 165 | resp = self._api_call(req, **kwargs) 166 | for event in self._event_deque: 167 | event.result = resp 168 | event.set() 169 | except Exception as exc: # pylint: disable=broad-except 170 | for event in self._event_deque: 171 | event.result = exc 172 | event.set() 173 | finally: 174 | self._in_deque.clear() 175 | self._event_deque.clear() 176 | 177 | def _run_with_subresponses(self, req, subresponse_field, kwargs): 178 | try: 179 | resp = self._api_call(req, **kwargs) 180 | in_sizes = [len(elts) for elts in self._in_deque] 181 | all_subresponses = getattr(resp, subresponse_field) 182 | if len(all_subresponses) != sum(in_sizes): 183 | _LOG.warn(_WARN_DEMUX_MISMATCH, len(all_subresponses), 184 | sum(in_sizes)) 185 | for event in self._event_deque: 186 | event.result = resp 187 | event.set() 188 | else: 189 | start = 0 190 | for i, event in zip(in_sizes, self._event_deque): 191 | next_copy = copy.copy(resp) 192 | subresponses = all_subresponses[start:start + i] 193 | setattr(next_copy, subresponse_field, subresponses) 194 | start += i 195 | event.result = next_copy 196 | event.set() 197 | except Exception as exc: # pylint: disable=broad-except 198 | for event in self._event_deque: 199 | event.result = exc 200 | event.set() 201 | finally: 202 | self._in_deque.clear() 203 | self._event_deque.clear() 204 | 205 | def extend(self, elts): 206 | """Adds elts to the tasks. 207 | 208 | Args: 209 | elts: a iterable of elements that can be appended to the task's 210 | bundle_field. 211 | 212 | Returns: 213 | an :class:`Event` that can be used to wait on the response. 214 | """ 215 | # Use a copy, not a reference, as it is later necessary to mutate 216 | # the proto field from which elts are drawn in order to construct 217 | # the bundled request. 218 | elts = elts[:] 219 | self._in_deque.append(elts) 220 | event = self._event_for(elts) 221 | self._event_deque.append(event) 222 | return event 223 | 224 | def _event_for(self, elts): 225 | """Creates an Event that is set when the bundle with elts is sent.""" 226 | event = Event() 227 | event.canceller = self._canceller_for(elts, event) 228 | return event 229 | 230 | def _canceller_for(self, elts, event): 231 | """Obtains a cancellation function that removes elts. 232 | 233 | The returned cancellation function returns ``True`` if all elements 234 | was removed successfully from the _in_deque, and false if it was not. 235 | """ 236 | def canceller(): 237 | """Cancels submission of ``elts`` as part of this bundle. 238 | 239 | Returns: 240 | ``False`` if any of elements had already been sent, otherwise 241 | ``True``. 242 | """ 243 | try: 244 | self._event_deque.remove(event) 245 | self._in_deque.remove(elts) 246 | return True 247 | except ValueError: 248 | return False 249 | 250 | return canceller 251 | 252 | 253 | TIMER_FACTORY = threading.Timer 254 | """A class with an interface similar to threading.Timer. 255 | 256 | Defaults to threading.Timer. This makes it easy to plug-in alternate 257 | timer implementations.""" 258 | 259 | 260 | class Executor(object): 261 | """Organizes bundling for an api service that requires it.""" 262 | # pylint: disable=too-few-public-methods 263 | 264 | def __init__(self, options): 265 | """Constructor. 266 | 267 | Args: 268 | options (gax.BundleOptions): configures strategy this instance 269 | uses when executing bundled functions. 270 | 271 | """ 272 | self._options = options 273 | self._tasks = {} 274 | self._task_lock = threading.RLock() 275 | self._timer = None 276 | 277 | def schedule(self, api_call, bundle_id, bundle_desc, bundling_request, 278 | kwargs=None): 279 | """Schedules bundle_desc of bundling_request as part of bundle_id. 280 | 281 | The returned value an :class:`Event` that 282 | 283 | * has a ``result`` attribute that will eventually be set to the result 284 | the api call 285 | * will be used to wait for the response 286 | * holds the canceller function for canceling this part of the bundle 287 | 288 | Args: 289 | api_call (callable[[object], object]): the scheduled API call. 290 | bundle_id (str): identifies the bundle on which the API call should be 291 | made. 292 | bundle_desc (gax.BundleDescriptor): describes the structure of the 293 | bundled call. 294 | bundling_request (object): the request instance to use in the API 295 | call. 296 | kwargs (dict): optional, the keyword arguments passed to the API call. 297 | 298 | Returns: 299 | an :class:`Event`. 300 | """ 301 | kwargs = kwargs or dict() 302 | bundle = self._bundle_for(api_call, bundle_id, bundle_desc, 303 | bundling_request, kwargs) 304 | elts = getattr(bundling_request, bundle_desc.bundled_field) 305 | event = bundle.extend(elts) 306 | 307 | # Run the bundle if the count threshold was reached. 308 | count_threshold = self._options.element_count_threshold 309 | if count_threshold > 0 and bundle.element_count >= count_threshold: 310 | self._run_now(bundle.bundle_id) 311 | 312 | # Run the bundle if the size threshold was reached. 313 | size_threshold = self._options.request_byte_threshold 314 | if size_threshold > 0 and bundle.request_bytesize >= size_threshold: 315 | self._run_now(bundle.bundle_id) 316 | 317 | return event 318 | 319 | def _bundle_for(self, api_call, bundle_id, bundle_desc, bundling_request, 320 | kwargs): 321 | with self._task_lock: 322 | bundle = self._tasks.get(bundle_id) 323 | if bundle is None: 324 | bundle = Task(api_call, bundle_id, bundle_desc.bundled_field, 325 | bundling_request, kwargs, 326 | subresponse_field=bundle_desc.subresponse_field) 327 | delay_threshold = self._options.delay_threshold 328 | if delay_threshold > 0: 329 | self._run_later(bundle, delay_threshold) 330 | self._tasks[bundle_id] = bundle 331 | return bundle 332 | 333 | def _run_later(self, bundle, delay_threshold): 334 | with self._task_lock: 335 | if self._timer is None: 336 | the_timer = TIMER_FACTORY( 337 | delay_threshold, 338 | self._run_now, 339 | args=[bundle.bundle_id]) 340 | the_timer.start() 341 | self._timer = the_timer 342 | 343 | def _run_now(self, bundle_id): 344 | with self._task_lock: 345 | if bundle_id in self._tasks: 346 | a_task = self._tasks.pop(bundle_id) 347 | a_task.run() 348 | 349 | 350 | class Event(object): 351 | """Wraps a threading.Event, adding, canceller and result attributes.""" 352 | 353 | def __init__(self): 354 | """Constructor. 355 | 356 | """ 357 | self._event = threading.Event() 358 | self.result = None 359 | self.canceller = None 360 | 361 | def is_set(self): 362 | """Calls ``is_set`` on the decorated :class:`threading.Event`.""" 363 | return self._event.is_set() 364 | 365 | def set(self): 366 | """Calls ``set`` on the decorated :class:`threading.Event`.""" 367 | return self._event.set() 368 | 369 | def clear(self): 370 | """Calls ``clear`` on the decorated :class:`threading.Event`. 371 | 372 | Also resets the result if one has been set. 373 | """ 374 | self.result = None 375 | return self._event.clear() 376 | 377 | def wait(self, timeout=None): 378 | """Calls ``wait`` on the decorated :class:`threading.Event`.""" 379 | return self._event.wait(timeout=timeout) 380 | 381 | def cancel(self): 382 | """Invokes the cancellation function provided on construction.""" 383 | if self.canceller: 384 | return self.canceller() 385 | else: 386 | return False 387 | -------------------------------------------------------------------------------- /google/gax/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015, Google Inc. 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are 6 | # met: 7 | # 8 | # * Redistributions of source code must retain the above copyright 9 | # notice, this list of conditions and the following disclaimer. 10 | # * Redistributions in binary form must reproduce the above 11 | # copyright notice, this list of conditions and the following disclaimer 12 | # in the documentation and/or other materials provided with the 13 | # distribution. 14 | # * Neither the name of Google Inc. nor the names of its 15 | # contributors may be used to endorse or promote products derived from 16 | # this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | """Google API Extensions""" 31 | 32 | from __future__ import absolute_import 33 | import collections 34 | 35 | 36 | __version__ = '0.10.1' 37 | 38 | 39 | OPTION_INHERIT = object() 40 | """Global constant. 41 | 42 | If a CallOptions field is set to OPTION_INHERIT, the call to which that 43 | CallOptions belongs will attempt to inherit that field from its default 44 | settings.""" 45 | 46 | 47 | class CallSettings(object): 48 | """Encapsulates the call settings for an API call.""" 49 | # pylint: disable=too-few-public-methods 50 | def __init__(self, timeout=30, retry=None, page_descriptor=None, 51 | bundler=None, bundle_descriptor=None): 52 | """Constructor. 53 | 54 | Args: 55 | timeout (int): The client-side timeout for API calls. This 56 | parameter is ignored for retrying calls. 57 | retry (:class:`RetryOptions`): The configuration for retrying upon 58 | transient error. If set to None, this call will not retry. 59 | page_descriptor (:class:`PageDescriptor`): indicates the structure 60 | of page streaming to be performed. If set to None, page streaming 61 | is disabled. 62 | bundler (:class:`gax.bundling.Executor`): orchestrates bundling. If 63 | None, bundling is not performed. 64 | bundle_descriptor (:class:`BundleDescriptor`): indicates the 65 | structure of of the bundle. If None, bundling is disabled. 66 | """ 67 | self.timeout = timeout 68 | self.retry = retry 69 | self.page_descriptor = page_descriptor 70 | self.bundler = bundler 71 | self.bundle_descriptor = bundle_descriptor 72 | 73 | def merge(self, options): 74 | """Returns a new CallSettings merged from this and a CallOptions object. 75 | 76 | Args: 77 | options (:class:`CallOptions`): an instance whose values override 78 | those in this object. If None, ``merge`` returns a copy of this 79 | object 80 | 81 | Returns: 82 | A :class:`CallSettings` object. 83 | """ 84 | if not options: 85 | return CallSettings( 86 | timeout=self.timeout, retry=self.retry, 87 | page_descriptor=self.page_descriptor, bundler=self.bundler, 88 | bundle_descriptor=self.bundle_descriptor) 89 | else: 90 | if options.timeout == OPTION_INHERIT: 91 | timeout = self.timeout 92 | else: 93 | timeout = options.timeout 94 | 95 | if options.retry == OPTION_INHERIT: 96 | retry = self.retry 97 | else: 98 | retry = options.retry 99 | 100 | if options.is_page_streaming: 101 | page_descriptor = self.page_descriptor 102 | else: 103 | page_descriptor = None 104 | 105 | if options.is_bundling: 106 | bundler = self.bundler 107 | else: 108 | bundler = None 109 | 110 | return CallSettings( 111 | timeout=timeout, retry=retry, 112 | page_descriptor=page_descriptor, bundler=bundler, 113 | bundle_descriptor=self.bundle_descriptor) 114 | 115 | 116 | class CallOptions(object): 117 | """Encapsulates the overridable settings for a particular API call. 118 | 119 | ``CallOptions`` is an optional arg for all GAX API calls. It is used to 120 | configure the settings of a specific API call. 121 | 122 | When provided, its values override the GAX service defaults for that 123 | particular call. 124 | """ 125 | # pylint: disable=too-few-public-methods 126 | def __init__(self, timeout=OPTION_INHERIT, retry=OPTION_INHERIT, 127 | is_page_streaming=OPTION_INHERIT, is_bundling=False): 128 | """Constructor. 129 | 130 | Example: 131 | >>> # change an api call's timeout 132 | >>> o1 = CallOptions(timeout=30) # make the timeout 30 seconds 133 | >>> 134 | >>> # disable page streaming on an api call that normally supports it 135 | >>> o2 = CallOptions(page_streaming=False) 136 | >>> 137 | >>> # disable retrying on an api call that normally retries 138 | >>> o3 = CallOptions(retry=None) 139 | >>> 140 | >>> # enable bundling on a call that supports it 141 | >>> o4 = CallOptions(is_bundling=True) 142 | 143 | Args: 144 | timeout (int): The client-side timeout for API calls. 145 | retry (:class:`RetryOptions`): determines whether and how to retry 146 | on transient errors. When set to None, the call will not retry. 147 | is_page_streaming (bool): If set and the call is configured for page 148 | streaming, page streaming is performed. 149 | is_bundling (bool): If set and the call is configured for bundling, 150 | bundling is performed. Bundling is always disabled by default. 151 | """ 152 | self.timeout = timeout 153 | self.retry = retry 154 | self.is_page_streaming = is_page_streaming 155 | self.is_bundling = is_bundling 156 | 157 | 158 | class PageDescriptor( 159 | collections.namedtuple( 160 | 'PageDescriptor', 161 | ['request_page_token_field', 162 | 'response_page_token_field', 163 | 'resource_field'])): 164 | """Describes the structure of a page-streaming call.""" 165 | pass 166 | 167 | 168 | class RetryOptions( 169 | collections.namedtuple( 170 | 'RetryOptions', 171 | ['retry_codes', 172 | 'backoff_settings'])): 173 | """Per-call configurable settings for retrying upon transient failure. 174 | 175 | Attributes: 176 | retry_codes (list[string]): a list of Google API canonical error codes 177 | upon which a retry should be attempted. 178 | backoff_settings (:class:`BackoffSettings`): configures the retry 179 | exponential backoff algorithm. 180 | """ 181 | pass 182 | 183 | 184 | class BackoffSettings( 185 | collections.namedtuple( 186 | 'BackoffSettings', 187 | ['initial_retry_delay_millis', 188 | 'retry_delay_multiplier', 189 | 'max_retry_delay_millis', 190 | 'initial_rpc_timeout_millis', 191 | 'rpc_timeout_multiplier', 192 | 'max_rpc_timeout_millis', 193 | 'total_timeout_millis'])): 194 | """Parameters to the exponential backoff algorithm for retrying. 195 | 196 | Attributes: 197 | initial_retry_delay_millis: the initial delay time, in milliseconds, 198 | between the completion of the first failed request and the initiation of 199 | the first retrying request. 200 | retry_delay_multiplier: the multiplier by which to increase the delay time 201 | between the completion of failed requests, and the initiation of the 202 | subsequent retrying request. 203 | max_retry_delay_millis: the maximum delay time, in milliseconds, between 204 | requests. When this value is reached, ``retry_delay_multiplier`` will no 205 | longer be used to increase delay time. 206 | initial_rpc_timeout_millis: the initial timeout parameter to the request. 207 | rpc_timeout_multiplier: the multiplier by which to increase the timeout 208 | parameter between failed requests. 209 | max_rpc_timeout_millis: the maximum timeout parameter, in milliseconds, 210 | for a request. When this value is reached, ``rpc_timeout_multiplier`` 211 | will no longer be used to increase the timeout. 212 | total_timeout_millis: the total time, in milliseconds, starting from when 213 | the initial request is sent, after which an error will be returned, 214 | regardless of the retrying attempts made meanwhile. 215 | """ 216 | pass 217 | 218 | 219 | class BundleDescriptor( 220 | collections.namedtuple( 221 | 'BundleDescriptor', 222 | ['bundled_field', 223 | 'request_discriminator_fields', 224 | 'subresponse_field'])): 225 | """Describes the structure of bundled call. 226 | 227 | request_discriminator_fields may include '.' as a separator, which is used 228 | to indicate object traversal. This allows fields in nested objects to be 229 | used to determine what requests to bundle. 230 | 231 | Attributes: 232 | bundled_field: the repeated field in the request message that 233 | will have its elements aggregated by bundling 234 | request_discriminator_fields: a list of fields in the 235 | target request message class that are used to determine 236 | which messages should be bundled together. 237 | subresponse_field: an optional field, when present it indicates the field 238 | in the response message that should be used to demultiplex the response 239 | into multiple response messages. 240 | """ 241 | def __new__(cls, 242 | bundled_field, 243 | request_discriminator_fields, 244 | subresponse_field=None): 245 | return super(cls, BundleDescriptor).__new__( 246 | cls, 247 | bundled_field, 248 | request_discriminator_fields, 249 | subresponse_field) 250 | 251 | 252 | class BundleOptions( 253 | collections.namedtuple( 254 | 'BundleOptions', 255 | ['element_count_threshold', 256 | 'element_count_limit', 257 | 'request_byte_threshold', 258 | 'request_byte_limit', 259 | 'delay_threshold'])): 260 | """Holds values used to configure bundling. 261 | 262 | The xxx_threshold attributes are used to configure when the bundled request 263 | should be made. 264 | 265 | Attributes: 266 | element_count_threshold: the bundled request will be sent once the 267 | count of outstanding elements in the repeated field reaches this 268 | value. 269 | element_count_limit: represents a hard limit on the number of elements 270 | in the repeated field of the bundle; if adding a request to a bundle 271 | would exceed this value, the bundle is sent and the new request is 272 | added to a fresh bundle. It is invalid for a single request to exceed 273 | this limit. 274 | request_byte_threshold: the bundled request will be sent once the count 275 | of bytes in the request reaches this value. Note that this value is 276 | pessimistically approximated by summing the bytesizes of the elements 277 | in the repeated field, and therefore may be an under-approximation. 278 | request_byte_limit: represents a hard limit on the size of the bundled 279 | request; if adding a request to a bundle would exceed this value, the 280 | bundle is sent and the new request is added to a fresh bundle. It is 281 | invalid for a single request to exceed this limit. Note that this 282 | value is pessimistically approximated by summing the bytesizes of the 283 | elements in the repeated field, with a buffer applied to correspond to 284 | the resulting under-approximation. 285 | delay_threshold: the bundled request will be sent this amount of 286 | time after the first element in the bundle was added to it. 287 | 288 | """ 289 | # pylint: disable=too-few-public-methods 290 | 291 | def __new__(cls, 292 | element_count_threshold=0, 293 | element_count_limit=0, 294 | request_byte_threshold=0, 295 | request_byte_limit=0, 296 | delay_threshold=0): 297 | """Invokes the base constructor with default values. 298 | 299 | The default values are zero for all attributes and it's necessary to 300 | specify at least one valid threshold value during construction. 301 | 302 | Args: 303 | element_count_threshold: the bundled request will be sent once the 304 | count of outstanding elements in the repeated field reaches this 305 | value. 306 | element_count_limit: represents a hard limit on the number of 307 | elements in the repeated field of the bundle; if adding a request 308 | to a bundle would exceed this value, the bundle is sent and the new 309 | request is added to a fresh bundle. It is invalid for a single 310 | request to exceed this limit. 311 | request_byte_threshold: the bundled request will be sent once the 312 | count of bytes in the request reaches this value. Note that this 313 | value is pessimistically approximated by summing the bytesizes of 314 | the elements in the repeated field, with a buffer applied to 315 | compensate for the corresponding under-approximation. 316 | request_byte_limit: represents a hard limit on the size of the 317 | bundled request; if adding a request to a bundle would exceed this 318 | value, the bundle is sent and the new request is added to a fresh 319 | bundle. It is invalid for a single request to exceed this 320 | limit. Note that this value is pessimistically approximated by 321 | summing the bytesizes of the elements in the repeated field, with a 322 | buffer applied to correspond to the resulting under-approximation. 323 | delay_threshold: the bundled request will be sent this amount of 324 | time after the first element in the bundle was added to it. 325 | 326 | """ 327 | assert isinstance(element_count_threshold, int), 'should be an int' 328 | assert isinstance(element_count_limit, int), 'should be an int' 329 | assert isinstance(request_byte_threshold, int), 'should be an int' 330 | assert isinstance(request_byte_limit, int), 'should be an int' 331 | assert isinstance(delay_threshold, int), 'should be an int' 332 | assert (element_count_threshold > 0 or 333 | request_byte_threshold > 0 or 334 | delay_threshold > 0), 'one threshold should be > 0' 335 | 336 | return super(cls, BundleOptions).__new__( 337 | cls, 338 | element_count_threshold, 339 | element_count_limit, 340 | request_byte_threshold, 341 | request_byte_limit, 342 | delay_threshold) 343 | -------------------------------------------------------------------------------- /test/test_api_callable.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015, Google Inc. 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are 6 | # met: 7 | # 8 | # * Redistributions of source code must retain the above copyright 9 | # notice, this list of conditions and the following disclaimer. 10 | # * Redistributions in binary form must reproduce the above 11 | # copyright notice, this list of conditions and the following disclaimer 12 | # in the documentation and/or other materials provided with the 13 | # distribution. 14 | # * Neither the name of Google Inc. nor the names of its 15 | # contributors may be used to endorse or promote products derived from 16 | # this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | # pylint: disable=missing-docstring,no-self-use,no-init,invalid-name,protected-access 31 | """Unit tests for api_callable""" 32 | 33 | from __future__ import absolute_import, division 34 | import mock 35 | import unittest2 36 | 37 | from google.gax import ( 38 | api_callable, bundling, BackoffSettings, BundleDescriptor, BundleOptions, 39 | CallSettings, PageDescriptor, RetryOptions) 40 | from google.gax.errors import GaxError, RetryError 41 | 42 | 43 | _SERVICE_NAME = 'test.interface.v1.api' 44 | 45 | 46 | _A_CONFIG = { 47 | 'interfaces': { 48 | _SERVICE_NAME: { 49 | 'retry_codes': { 50 | 'foo_retry': ['code_a', 'code_b'], 51 | 'bar_retry': ['code_c'] 52 | }, 53 | 'retry_params': { 54 | 'default': { 55 | 'initial_retry_delay_millis': 100, 56 | 'retry_delay_multiplier': 1.2, 57 | 'max_retry_delay_millis': 1000, 58 | 'initial_rpc_timeout_millis': 300, 59 | 'rpc_timeout_multiplier': 1.3, 60 | 'max_rpc_timeout_millis': 3000, 61 | 'total_timeout_millis': 30000 62 | } 63 | }, 64 | 'methods': { 65 | # Note that GAX should normalize this to snake case 66 | 'BundlingMethod': { 67 | 'retry_codes_name': 'foo_retry', 68 | 'retry_params_name': 'default', 69 | 'bundling': { 70 | 'element_count_threshold': 6, 71 | 'element_count_limit': 10 72 | } 73 | }, 74 | 'page_streaming_method': { 75 | 'retry_codes_name': 'bar_retry', 76 | 'retry_params_name': 'default' 77 | } 78 | } 79 | } 80 | } 81 | } 82 | 83 | 84 | _PAGE_DESCRIPTORS = { 85 | 'page_streaming_method': PageDescriptor( 86 | 'page_token', 'next_page_token', 'page_streams') 87 | } 88 | 89 | 90 | _BUNDLE_DESCRIPTORS = {'bundling_method': BundleDescriptor('bundled_field', [])} 91 | 92 | 93 | _RETRY_DICT = {'code_a': Exception, 94 | 'code_b': Exception, 95 | 'code_c': Exception} 96 | 97 | 98 | _FAKE_STATUS_CODE_1 = object() 99 | 100 | 101 | _FAKE_STATUS_CODE_2 = object() 102 | 103 | 104 | class CustomException(Exception): 105 | def __init__(self, msg, code): 106 | super(CustomException, self).__init__(msg) 107 | self.code = code 108 | 109 | 110 | class AnotherException(Exception): 111 | pass 112 | 113 | 114 | class TestCreateApiCallable(unittest2.TestCase): 115 | 116 | def test_call_api_call(self): 117 | settings = CallSettings() 118 | my_callable = api_callable.create_api_call( 119 | lambda _req, _timeout: 42, settings) 120 | self.assertEqual(my_callable(None), 42) 121 | 122 | @mock.patch('time.time') 123 | @mock.patch('google.gax.config.exc_to_code') 124 | def test_retry(self, mock_exc_to_code, mock_time): 125 | mock_exc_to_code.side_effect = lambda e: e.code 126 | to_attempt = 3 127 | retry = RetryOptions( 128 | [_FAKE_STATUS_CODE_1], 129 | BackoffSettings(0, 0, 0, 0, 0, 0, 1)) 130 | 131 | # Succeeds on the to_attempt'th call, and never again afterward 132 | mock_call = mock.Mock() 133 | mock_call.side_effect = ([CustomException('', _FAKE_STATUS_CODE_1)] * 134 | (to_attempt - 1) + [mock.DEFAULT]) 135 | mock_call.return_value = 1729 136 | mock_time.return_value = 0 137 | settings = CallSettings(timeout=0, retry=retry) 138 | my_callable = api_callable.create_api_call(mock_call, settings) 139 | self.assertEqual(my_callable(None), 1729) 140 | self.assertEqual(mock_call.call_count, to_attempt) 141 | 142 | @mock.patch('time.time') 143 | def test_no_retry_if_no_codes(self, mock_time): 144 | retry = RetryOptions([], BackoffSettings(1, 2, 3, 4, 5, 6, 7)) 145 | 146 | mock_call = mock.Mock() 147 | mock_call.side_effect = CustomException('', _FAKE_STATUS_CODE_1) 148 | mock_time.return_value = 0 149 | 150 | settings = CallSettings(timeout=0, retry=retry) 151 | my_callable = api_callable.create_api_call(mock_call, settings) 152 | self.assertRaises(CustomException, my_callable, None) 153 | self.assertEqual(mock_call.call_count, 1) 154 | 155 | @mock.patch('time.time') 156 | @mock.patch('google.gax.config.exc_to_code') 157 | def test_retry_aborts_simple(self, mock_exc_to_code, mock_time): 158 | def fake_call(dummy_request, dummy_timeout): 159 | raise CustomException('', _FAKE_STATUS_CODE_1) 160 | 161 | retry = RetryOptions( 162 | [_FAKE_STATUS_CODE_1], 163 | BackoffSettings(0, 0, 0, 0, 0, 0, 1)) 164 | mock_time.side_effect = [0, 2] 165 | mock_exc_to_code.side_effect = lambda e: e.code 166 | settings = CallSettings(timeout=0, retry=retry) 167 | my_callable = api_callable.create_api_call(fake_call, settings) 168 | 169 | try: 170 | my_callable(None) 171 | except RetryError as exc: 172 | self.assertIsInstance(exc.cause, CustomException) 173 | 174 | @mock.patch('time.time') 175 | @mock.patch('google.gax.config.exc_to_code') 176 | def test_retry_times_out_simple(self, mock_exc_to_code, mock_time): 177 | mock_exc_to_code.side_effect = lambda e: e.code 178 | to_attempt = 3 179 | retry = RetryOptions( 180 | [_FAKE_STATUS_CODE_1], 181 | BackoffSettings(0, 0, 0, 0, 0, 0, 1)) 182 | mock_call = mock.Mock() 183 | mock_call.side_effect = CustomException('', _FAKE_STATUS_CODE_1) 184 | mock_time.side_effect = ([0] * to_attempt + [2]) 185 | settings = CallSettings(timeout=0, retry=retry) 186 | my_callable = api_callable.create_api_call(mock_call, settings) 187 | 188 | try: 189 | my_callable(None) 190 | except RetryError as exc: 191 | self.assertIsInstance(exc.cause, CustomException) 192 | 193 | self.assertEqual(mock_call.call_count, to_attempt) 194 | 195 | @mock.patch('time.time') 196 | @mock.patch('google.gax.config.exc_to_code') 197 | def test_retry_aborts_on_unexpected_exception( 198 | self, mock_exc_to_code, mock_time): 199 | mock_exc_to_code.side_effect = lambda e: e.code 200 | retry = RetryOptions( 201 | [_FAKE_STATUS_CODE_1], 202 | BackoffSettings(0, 0, 0, 0, 0, 0, 1)) 203 | mock_call = mock.Mock() 204 | mock_call.side_effect = CustomException('', _FAKE_STATUS_CODE_2) 205 | mock_time.return_value = 0 206 | settings = CallSettings(timeout=0, retry=retry) 207 | my_callable = api_callable.create_api_call(mock_call, settings) 208 | self.assertRaises(Exception, my_callable, None) 209 | self.assertEqual(mock_call.call_count, 1) 210 | 211 | @mock.patch('time.time') 212 | def test_retry_times_out_no_response(self, mock_time): 213 | mock_time.return_value = 1 214 | retry = RetryOptions( 215 | [_FAKE_STATUS_CODE_1], 216 | BackoffSettings(0, 0, 0, 0, 0, 0, 0)) 217 | settings = CallSettings(timeout=0, retry=retry) 218 | my_callable = api_callable.create_api_call(lambda: None, settings) 219 | 220 | self.assertRaises(RetryError, my_callable, None) 221 | 222 | @mock.patch('time.sleep') 223 | @mock.patch('time.time') 224 | @mock.patch('google.gax.config.exc_to_code') 225 | def test_retry_exponential_backoff(self, mock_exc_to_code, mock_time, 226 | mock_sleep): 227 | # pylint: disable=too-many-locals 228 | mock_exc_to_code.side_effect = lambda e: e.code 229 | MILLIS_PER_SEC = 1000 230 | mock_time.return_value = 0 231 | 232 | def incr_time(secs): 233 | mock_time.return_value += secs 234 | 235 | def api_call(dummy_request, timeout, **dummy_kwargs): 236 | incr_time(timeout) 237 | raise CustomException(str(timeout), _FAKE_STATUS_CODE_1) 238 | 239 | mock_call = mock.Mock() 240 | mock_sleep.side_effect = incr_time 241 | mock_call.side_effect = api_call 242 | 243 | params = BackoffSettings(3, 2, 24, 5, 2, 80, 2500) 244 | retry = RetryOptions([_FAKE_STATUS_CODE_1], params) 245 | settings = CallSettings(timeout=0, retry=retry) 246 | my_callable = api_callable.create_api_call(mock_call, settings) 247 | 248 | try: 249 | my_callable(None) 250 | except RetryError as exc: 251 | self.assertIsInstance(exc.cause, CustomException) 252 | 253 | self.assertGreaterEqual(mock_time(), 254 | params.total_timeout_millis / MILLIS_PER_SEC) 255 | 256 | # Very rough bounds 257 | calls_lower_bound = params.total_timeout_millis / ( 258 | params.max_retry_delay_millis + params.max_rpc_timeout_millis) 259 | self.assertGreater(mock_call.call_count, calls_lower_bound) 260 | 261 | calls_upper_bound = (params.total_timeout_millis / 262 | params.initial_retry_delay_millis) 263 | self.assertLess(mock_call.call_count, calls_upper_bound) 264 | 265 | def test_page_streaming(self): 266 | # A mock grpc function that page streams a list of consecutive 267 | # integers, returning `page_size` integers with each call and using 268 | # the next integer to return as the page token, until `pages_to_stream` 269 | # pages have been returned. 270 | page_size = 3 271 | pages_to_stream = 5 272 | 273 | # pylint: disable=abstract-method, too-few-public-methods 274 | class PageStreamingRequest(object): 275 | def __init__(self, page_token=0): 276 | self.page_token = page_token 277 | 278 | class PageStreamingResponse(object): 279 | def __init__(self, nums=(), next_page_token=0): 280 | self.nums = nums 281 | self.next_page_token = next_page_token 282 | 283 | fake_grpc_func_descriptor = PageDescriptor( 284 | 'page_token', 'next_page_token', 'nums') 285 | 286 | def grpc_return_value(request, *dummy_args, **dummy_kwargs): 287 | if (request.page_token > 0 and 288 | request.page_token < page_size * pages_to_stream): 289 | return PageStreamingResponse( 290 | nums=iter(range(request.page_token, 291 | request.page_token + page_size)), 292 | next_page_token=request.page_token + page_size) 293 | elif request.page_token >= page_size * pages_to_stream: 294 | return PageStreamingResponse() 295 | else: 296 | return PageStreamingResponse(nums=iter(range(page_size)), 297 | next_page_token=page_size) 298 | 299 | with mock.patch('grpc.framework.crust.implementations.' 300 | '_UnaryUnaryMultiCallable') as mock_grpc: 301 | mock_grpc.side_effect = grpc_return_value 302 | settings = CallSettings( 303 | page_descriptor=fake_grpc_func_descriptor, timeout=0) 304 | my_callable = api_callable.create_api_call(mock_grpc, settings=settings) 305 | self.assertEqual(list(my_callable(PageStreamingRequest())), 306 | list(range(page_size * pages_to_stream))) 307 | 308 | def test_bundling_page_streaming_error(self): 309 | settings = CallSettings( 310 | page_descriptor=object(), bundle_descriptor=object(), 311 | bundler=object()) 312 | with self.assertRaises(ValueError): 313 | api_callable.create_api_call(lambda _req, _timeout: 42, settings) 314 | 315 | def test_bundling(self): 316 | # pylint: disable=abstract-method, too-few-public-methods 317 | class BundlingRequest(object): 318 | def __init__(self, elements=None): 319 | self.elements = elements 320 | 321 | fake_grpc_func_descriptor = BundleDescriptor('elements', []) 322 | bundler = bundling.Executor(BundleOptions(element_count_threshold=8)) 323 | 324 | def my_func(request, dummy_timeout): 325 | return len(request.elements) 326 | 327 | settings = CallSettings( 328 | bundler=bundler, bundle_descriptor=fake_grpc_func_descriptor, 329 | timeout=0) 330 | my_callable = api_callable.create_api_call(my_func, settings) 331 | first = my_callable(BundlingRequest([0] * 3)) 332 | self.assertIsInstance(first, bundling.Event) 333 | self.assertIsNone(first.result) # pylint: disable=no-member 334 | second = my_callable(BundlingRequest([0] * 5)) 335 | self.assertEquals(second.result, 8) # pylint: disable=no-member 336 | 337 | def test_construct_settings(self): 338 | defaults = api_callable.construct_settings( 339 | _SERVICE_NAME, _A_CONFIG, dict(), dict(), _RETRY_DICT, 30, 340 | bundle_descriptors=_BUNDLE_DESCRIPTORS, 341 | page_descriptors=_PAGE_DESCRIPTORS) 342 | settings = defaults['bundling_method'] 343 | self.assertEquals(settings.timeout, 30) 344 | self.assertIsInstance(settings.bundler, bundling.Executor) 345 | self.assertIsInstance(settings.bundle_descriptor, BundleDescriptor) 346 | self.assertIsNone(settings.page_descriptor) 347 | self.assertIsInstance(settings.retry, RetryOptions) 348 | settings = defaults['page_streaming_method'] 349 | self.assertEquals(settings.timeout, 30) 350 | self.assertIsNone(settings.bundler) 351 | self.assertIsNone(settings.bundle_descriptor) 352 | self.assertIsInstance(settings.page_descriptor, PageDescriptor) 353 | self.assertIsInstance(settings.retry, RetryOptions) 354 | 355 | def test_construct_settings_override(self): 356 | _bundling_override = {'bundling_method': None} 357 | _retry_override = {'page_streaming_method': None} 358 | defaults = api_callable.construct_settings( 359 | _SERVICE_NAME, _A_CONFIG, _bundling_override, _retry_override, 360 | _RETRY_DICT, 30, bundle_descriptors=_BUNDLE_DESCRIPTORS, 361 | page_descriptors=_PAGE_DESCRIPTORS) 362 | settings = defaults['bundling_method'] 363 | self.assertEquals(settings.timeout, 30) 364 | self.assertIsNone(settings.bundler) 365 | self.assertIsNone(settings.page_descriptor) 366 | settings = defaults['page_streaming_method'] 367 | self.assertEquals(settings.timeout, 30) 368 | self.assertIsInstance(settings.page_descriptor, PageDescriptor) 369 | self.assertIsNone(settings.retry) 370 | 371 | @mock.patch('google.gax.config.API_ERRORS', (CustomException, )) 372 | def test_catch_error(self): 373 | def abortion_error_func(*dummy_args, **dummy_kwargs): 374 | raise CustomException(None, None) 375 | 376 | def other_error_func(*dummy_args, **dummy_kwargs): 377 | raise AnotherException 378 | 379 | gax_error_callable = api_callable.create_api_call( 380 | abortion_error_func, CallSettings()) 381 | self.assertRaises(GaxError, gax_error_callable, None) 382 | 383 | other_error_callable = api_callable.create_api_call( 384 | other_error_func, CallSettings()) 385 | self.assertRaises(AnotherException, other_error_callable, None) 386 | -------------------------------------------------------------------------------- /google/gax/api_callable.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016, Google Inc. 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are 6 | # met: 7 | # 8 | # * Redistributions of source code must retain the above copyright 9 | # notice, this list of conditions and the following disclaimer. 10 | # * Redistributions in binary form must reproduce the above 11 | # copyright notice, this list of conditions and the following disclaimer 12 | # in the documentation and/or other materials provided with the 13 | # distribution. 14 | # * Neither the name of Google Inc. nor the names of its 15 | # contributors may be used to endorse or promote products derived from 16 | # this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | """Provides function wrappers that implement page streaming and retrying.""" 31 | 32 | from __future__ import absolute_import, division 33 | import random 34 | import sys 35 | import time 36 | 37 | from . import (BackoffSettings, BundleOptions, bundling, CallSettings, config, 38 | OPTION_INHERIT, RetryOptions) 39 | from .errors import GaxError, RetryError 40 | 41 | _MILLIS_PER_SECOND = 1000 42 | 43 | 44 | def _add_timeout_arg(a_func, timeout): 45 | """Updates a_func so that it gets called with the timeout as its final arg. 46 | 47 | This converts a callable, a_func, into another callable with an additional 48 | positional arg. 49 | 50 | Args: 51 | a_func (callable): a callable to be updated 52 | timeout (int): to be added to the original callable as it final positional 53 | arg. 54 | 55 | 56 | Returns: 57 | callable: the original callable updated to the timeout arg 58 | """ 59 | 60 | def inner(*args, **kw): 61 | """Updates args with the timeout.""" 62 | updated_args = args + (timeout,) 63 | return a_func(*updated_args, **kw) 64 | 65 | return inner 66 | 67 | 68 | def _retryable(a_func, retry): 69 | """Creates a function equivalent to a_func, but that retries on certain 70 | exceptions. 71 | 72 | Args: 73 | a_func (callable): A callable. 74 | retry (RetryOptions): Configures the exceptions upon which the callable 75 | should retry, and the parameters to the exponential backoff retry 76 | algorithm. 77 | 78 | Returns: 79 | A function that will retry on exception. 80 | """ 81 | 82 | delay_mult = retry.backoff_settings.retry_delay_multiplier 83 | max_delay = (retry.backoff_settings.max_retry_delay_millis / 84 | _MILLIS_PER_SECOND) 85 | timeout_mult = retry.backoff_settings.rpc_timeout_multiplier 86 | max_timeout = (retry.backoff_settings.max_rpc_timeout_millis / 87 | _MILLIS_PER_SECOND) 88 | total_timeout = (retry.backoff_settings.total_timeout_millis / 89 | _MILLIS_PER_SECOND) 90 | 91 | def inner(*args, **kwargs): 92 | """Equivalent to ``a_func``, but retries upon transient failure. 93 | 94 | Retrying is done through an exponential backoff algorithm configured 95 | by the options in ``retry``. 96 | """ 97 | delay = retry.backoff_settings.initial_retry_delay_millis 98 | timeout = (retry.backoff_settings.initial_rpc_timeout_millis / 99 | _MILLIS_PER_SECOND) 100 | exc = RetryError('Retry total timeout exceeded before any' 101 | 'response was received') 102 | now = time.time() 103 | deadline = now + total_timeout 104 | 105 | while now < deadline: 106 | try: 107 | to_call = _add_timeout_arg(a_func, timeout) 108 | return to_call(*args, **kwargs) 109 | 110 | # pylint: disable=broad-except 111 | except Exception as exception: 112 | if config.exc_to_code(exception) not in retry.retry_codes: 113 | raise RetryError( 114 | 'Exception occurred in retry method that was not' 115 | ' classified as transient', exception) 116 | 117 | # pylint: disable=redefined-variable-type 118 | exc = RetryError('Retry total timeout exceeded with exception', 119 | exception) 120 | to_sleep = random.uniform(0, delay) 121 | time.sleep(to_sleep / _MILLIS_PER_SECOND) 122 | now = time.time() 123 | delay = min(delay * delay_mult, max_delay) 124 | timeout = min( 125 | timeout * timeout_mult, max_timeout, deadline - now) 126 | continue 127 | 128 | raise exc 129 | 130 | return inner 131 | 132 | 133 | def _bundleable(a_func, desc, bundler): 134 | """Creates a function that transforms an API call into a bundling call. 135 | 136 | It transform a_func from an API call that receives the requests and returns 137 | the response into a callable that receives the same request, and 138 | returns a :class:`bundling.Event`. 139 | 140 | The returned Event object can be used to obtain the eventual result of the 141 | bundled call. 142 | 143 | Args: 144 | a_func (callable[[req], resp]): an API call that supports bundling. 145 | desc (gax.BundleDescriptor): describes the bundling that a_func 146 | supports. 147 | bundler (gax.bundling.Executor): orchestrates bundling. 148 | 149 | Returns: 150 | callable: takes the API call's request and keyword args and returns a 151 | bundling.Event object. 152 | 153 | """ 154 | def inner(*args, **kwargs): 155 | """Schedules execution of a bundling task.""" 156 | request = args[0] 157 | the_id = bundling.compute_bundle_id( 158 | request, desc.request_discriminator_fields) 159 | return bundler.schedule(a_func, the_id, desc, request, kwargs) 160 | 161 | return inner 162 | 163 | 164 | def _page_streamable(a_func, 165 | request_page_token_field, 166 | response_page_token_field, 167 | resource_field): 168 | """Creates a function that yields an iterable to performs page-streaming. 169 | 170 | Args: 171 | a_func: an API call that is page streaming. 172 | request_page_token_field: The field of the page token in the request. 173 | response_page_token_field: The field of the next page token in the 174 | response. 175 | resource_field: The field to be streamed. 176 | 177 | Returns: 178 | A function that returns an iterable over the specified field. 179 | """ 180 | 181 | def inner(*args, **kwargs): 182 | """A generator that yields all the paged responses.""" 183 | request = args[0] 184 | while True: 185 | response = a_func(request, **kwargs) 186 | for obj in getattr(response, resource_field): 187 | yield obj 188 | next_page_token = getattr(response, response_page_token_field) 189 | if not next_page_token: 190 | break 191 | setattr(request, request_page_token_field, next_page_token) 192 | 193 | return inner 194 | 195 | 196 | def _construct_bundling(method_config, method_bundling_override, 197 | bundle_descriptor): 198 | """Helper for ``construct_settings()``. 199 | 200 | Args: 201 | method_config: A dictionary representing a single ``methods`` entry of the 202 | standard API client config file. (See ``construct_settings()`` for 203 | information on this yaml.) 204 | method_retry_override: A BundleOptions object, OPTION_INHERIT, or None. 205 | If set to OPTION_INHERIT, the retry settings are derived from method 206 | config. Otherwise, this parameter overrides ``method_config``. 207 | bundle_descriptor: A BundleDescriptor object describing the structure of 208 | bundling for this method. If not set, this method will not bundle. 209 | 210 | Returns: 211 | A tuple (bundling.Executor, BundleDescriptor) that configures bundling. 212 | The bundling.Executor may be None if this method should not bundle. 213 | """ 214 | if 'bundling' in method_config and bundle_descriptor: 215 | 216 | if method_bundling_override == OPTION_INHERIT: 217 | params = method_config['bundling'] 218 | bundler = bundling.Executor(BundleOptions( 219 | element_count_threshold=params.get( 220 | 'element_count_threshold', 0), 221 | element_count_limit=params.get('element_count_limit', 0), 222 | request_byte_threshold=params.get('request_byte_threshold', 0), 223 | request_byte_limit=params.get('request_byte_limit', 0), 224 | delay_threshold=params.get('delay_threshold_millis', 0))) 225 | elif method_bundling_override: 226 | bundler = bundling.Executor(method_bundling_override) 227 | else: 228 | bundler = None 229 | 230 | else: 231 | bundler = None 232 | 233 | return bundler 234 | 235 | 236 | def _construct_retry( 237 | method_config, method_retry_override, retry_codes, retry_params, 238 | retry_names): 239 | """Helper for ``construct_settings()``. 240 | 241 | Args: 242 | method_config: A dictionary representing a single ``methods`` entry of the 243 | standard API client config file. (See ``construct_settings()`` for 244 | information on this yaml.) 245 | method_retry_override: A RetryOptions object, OPTION_INHERIT, or None. 246 | If set to OPTION_INHERIT, the retry settings are derived from method 247 | config. Otherwise, this parameter overrides ``method_config``. 248 | retry_codes_def: A dictionary parsed from the ``retry_codes_def`` entry 249 | of the standard API client config file. (See ``construct_settings()`` 250 | for information on this yaml.) 251 | retry_params: A dictionary parsed from the ``retry_params`` entry 252 | of the standard API client config file. (See ``construct_settings()`` 253 | for information on this yaml.) 254 | retry_names: A dictionary mapping the string names used in the 255 | standard API client config file to API response status codes. 256 | 257 | Returns: 258 | A RetryOptions object, or None. 259 | """ 260 | if method_retry_override != OPTION_INHERIT: 261 | return method_retry_override 262 | 263 | codes = [] 264 | if retry_codes: 265 | for codes_name in retry_codes: 266 | if (codes_name == method_config['retry_codes_name'] and 267 | retry_codes[codes_name]): 268 | codes = [ 269 | retry_names[name] for name in retry_codes[codes_name]] 270 | break 271 | 272 | params_struct = None 273 | if method_config.get('retry_params_name'): 274 | for params_name in retry_params: 275 | if params_name == method_config['retry_params_name']: 276 | params_struct = retry_params[params_name].copy() 277 | break 278 | backoff_settings = BackoffSettings(**params_struct) 279 | else: 280 | backoff_settings = None 281 | 282 | retry = RetryOptions(retry_codes=codes, backoff_settings=backoff_settings) 283 | return retry 284 | 285 | 286 | def _upper_camel_to_lower_under(string): 287 | if not string: 288 | return '' 289 | out = '' 290 | out += string[0].lower() 291 | for char in string[1:]: 292 | if char.isupper(): 293 | out += '_' + char.lower() 294 | else: 295 | out += char 296 | return out 297 | 298 | 299 | def construct_settings( 300 | service_name, client_config, bundling_override, retry_override, 301 | retry_names, timeout, bundle_descriptors=None, page_descriptors=None): 302 | """Constructs a dictionary mapping method names to CallSettings. 303 | 304 | The ``client_config`` parameter is parsed from a client configuration JSON 305 | file of the form: 306 | 307 | .. code-block:: json 308 | 309 | { 310 | "interfaces": { 311 | "google.fake.v1.ServiceName": { 312 | "retry_codes": { 313 | "idempotent": ["UNAVAILABLE", "DEADLINE_EXCEEDED"], 314 | "non_idempotent": [] 315 | }, 316 | "retry_params": { 317 | "default": { 318 | "initial_retry_delay_millis": 100, 319 | "retry_delay_multiplier": 1.2, 320 | "max_retry_delay_millis": 1000, 321 | "initial_rpc_timeout_millis": 2000, 322 | "rpc_timeout_multiplier": 1.5, 323 | "max_rpc_timeout_millis": 30000, 324 | "total_timeout_millis": 45000 325 | } 326 | }, 327 | "methods": { 328 | "CreateFoo": { 329 | "retry_codes_name": "idempotent", 330 | "retry_params_name": "default" 331 | }, 332 | "Publish": { 333 | "retry_codes_name": "non_idempotent", 334 | "retry_params_name": "default", 335 | "bundling": { 336 | "element_count_threshold": 40, 337 | "element_count_limit": 200, 338 | "request_byte_threshold": 90000, 339 | "request_byte_limit": 100000, 340 | "delay_threshold_millis": 100 341 | } 342 | } 343 | } 344 | } 345 | } 346 | } 347 | 348 | Args: 349 | service_name: The fully-qualified name of this service, used as a key into 350 | the client config file (in the example above, this value should be 351 | ``google.fake.v1.ServiceName``). 352 | client_config: A dictionary parsed from the standard API client config 353 | file. 354 | bundle_descriptors: A dictionary of method names to BundleDescriptor 355 | objects for methods that are bundling-enabled. 356 | page_descriptors: A dictionary of method names to PageDescriptor objects 357 | for methods that are page streaming-enabled. 358 | bundling_override: A dictionary of method names to BundleOptions 359 | override those specified in ``client_config``. 360 | retry_override: A dictionary of method names to RetryOptions that 361 | override those specified in ``client_config``. 362 | retry_names: A dictionary mapping the strings referring to response status 363 | codes to the Python objects representing those codes. 364 | timeout: The timeout parameter for all API calls in this dictionary. 365 | 366 | Raises: 367 | KeyError: If the configuration for the service in question cannot be 368 | located in the provided ``client_config``. 369 | """ 370 | # pylint: disable=too-many-locals 371 | defaults = dict() 372 | bundle_descriptors = bundle_descriptors or {} 373 | page_descriptors = page_descriptors or {} 374 | 375 | try: 376 | service_config = client_config['interfaces'][service_name] 377 | except KeyError: 378 | raise KeyError('Client configuration not found for service: {}' 379 | .format(service_name)) 380 | 381 | for method in service_config.get('methods'): 382 | method_config = service_config['methods'][method] 383 | snake_name = _upper_camel_to_lower_under(method) 384 | 385 | bundle_descriptor = bundle_descriptors.get(snake_name) 386 | bundler = _construct_bundling( 387 | method_config, bundling_override.get(snake_name, OPTION_INHERIT), 388 | bundle_descriptor) 389 | 390 | retry = _construct_retry( 391 | method_config, retry_override.get(snake_name, OPTION_INHERIT), 392 | service_config['retry_codes'], service_config['retry_params'], 393 | retry_names) 394 | 395 | defaults[snake_name] = CallSettings( 396 | timeout=timeout, retry=retry, 397 | page_descriptor=page_descriptors.get(snake_name), 398 | bundler=bundler, bundle_descriptor=bundle_descriptor) 399 | 400 | return defaults 401 | 402 | 403 | def _catch_errors(a_func, errors): 404 | """Updates a_func to wrap exceptions with GaxError 405 | 406 | Args: 407 | a_func (callable): A callable. 408 | retry (list[Exception]): Configures the exceptions to wrap. 409 | 410 | Returns: 411 | A function that will wrap certain exceptions with GaxError 412 | """ 413 | def inner(*args, **kwargs): 414 | """Wraps specified exceptions""" 415 | try: 416 | return a_func(*args, **kwargs) 417 | # pylint: disable=catching-non-exception 418 | except tuple(errors) as exception: 419 | raise (GaxError('RPC failed', cause=exception), None, 420 | sys.exc_info()[2]) 421 | 422 | return inner 423 | 424 | 425 | def create_api_call(func, settings): 426 | """Converts an rpc call into an API call governed by the settings. 427 | 428 | In typical usage, ``func`` will be a callable used to make an rpc request. 429 | This will mostly likely be a bound method from a request stub used to make 430 | an rpc call. 431 | 432 | The result is created by applying a series of function decorators defined 433 | in this module to ``func``. ``settings`` is used to determine which 434 | function decorators to apply. 435 | 436 | The result is another callable which for most values of ``settings`` has 437 | has the same signature as the original. Only when ``settings`` configures 438 | bundling does the signature change. 439 | 440 | Args: 441 | func (callable[[object], object]): is used to make a bare rpc call 442 | settings (:class:`CallSettings`): provides the settings for this call 443 | 444 | Returns: 445 | func (callable[[object], object]): a bound method on a request stub used 446 | to make an rpc call 447 | 448 | Raises: 449 | ValueError: if ``settings`` has incompatible values, e.g, if bundling 450 | and page_streaming are both configured 451 | 452 | """ 453 | if settings.retry and settings.retry.retry_codes: 454 | api_call = _retryable(func, settings.retry) 455 | else: 456 | api_call = _add_timeout_arg(func, settings.timeout) 457 | 458 | if settings.page_descriptor: 459 | if settings.bundler and settings.bundle_descriptor: 460 | raise ValueError('The API call has incompatible settings: ' 461 | 'bundling and page streaming') 462 | return _page_streamable( 463 | api_call, 464 | settings.page_descriptor.request_page_token_field, 465 | settings.page_descriptor.response_page_token_field, 466 | settings.page_descriptor.resource_field) 467 | 468 | if settings.bundler and settings.bundle_descriptor: 469 | return _bundleable(api_call, settings.bundle_descriptor, 470 | settings.bundler) 471 | 472 | return _catch_errors(api_call, config.API_ERRORS) 473 | -------------------------------------------------------------------------------- /test/test_bundling.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016, Google Inc. 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are 6 | # met: 7 | # 8 | # * Redistributions of source code must retain the above copyright 9 | # notice, this list of conditions and the following disclaimer. 10 | # * Redistributions in binary form must reproduce the above 11 | # copyright notice, this list of conditions and the following disclaimer 12 | # in the documentation and/or other materials provided with the 13 | # distribution. 14 | # * Neither the name of Google Inc. nor the names of its 15 | # contributors may be used to endorse or promote products derived from 16 | # this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | # pylint: disable=missing-docstring,no-self-use,no-init,invalid-name 31 | """Unit tests for bundling.""" 32 | 33 | from __future__ import absolute_import 34 | 35 | import mock 36 | import unittest2 37 | 38 | from google.gax import bundling, BundleOptions, BundleDescriptor 39 | 40 | 41 | # pylint: disable=too-few-public-methods 42 | class _Simple(object): 43 | def __init__(self, value, other_value=None): 44 | self.field1 = value 45 | self.field2 = other_value 46 | 47 | def __eq__(self, other): 48 | return (self.field1 == other.field1 and 49 | self.field2 == other.field2) 50 | 51 | def __str__(self): 52 | return 'field1={0}, field2={1}'.format(self.field1, self.field2) 53 | 54 | 55 | class _Outer(object): 56 | def __init__(self, value): 57 | self.inner = _Simple(value, other_value=value) 58 | self.field1 = value 59 | 60 | 61 | class TestComputeBundleId(unittest2.TestCase): 62 | 63 | def test_computes_bundle_ids_ok(self): 64 | tests = [ 65 | { 66 | 'message': 'single field value', 67 | 'object': _Simple('dummy_value'), 68 | 'fields': ['field1'], 69 | 'want': ('dummy_value',) 70 | }, { 71 | 'message': 'composite value with None', 72 | 'object': _Simple('dummy_value'), 73 | 'fields': ['field1', 'field2'], 74 | 'want': ('dummy_value', None) 75 | }, { 76 | 'message': 'a composite value', 77 | 'object': _Simple('dummy_value', 'other_value'), 78 | 'fields': ['field1', 'field2'], 79 | 'want': ('dummy_value', 'other_value') 80 | }, { 81 | 'message': 'a simple dotted value', 82 | 'object': _Outer('this is dotty'), 83 | 'fields': ['inner.field1'], 84 | 'want': ('this is dotty',) 85 | }, { 86 | 'message': 'a complex case', 87 | 'object': _Outer('what!?'), 88 | 'fields': ['inner.field1', 'inner.field2', 'field1'], 89 | 'want': ('what!?', 'what!?', 'what!?') 90 | } 91 | ] 92 | for t in tests: 93 | got = bundling.compute_bundle_id(t['object'], t['fields']) 94 | message = 'failed while making an id for {}'.format(t['message']) 95 | self.assertEquals(got, t['want'], message) 96 | 97 | def test_should_raise_if_fields_are_missing(self): 98 | tests = [ 99 | { 100 | 'object': _Simple('dummy_value'), 101 | 'fields': ['field3'], 102 | }, { 103 | 'object': _Simple('dummy_value'), 104 | 'fields': ['field1', 'field3'], 105 | }, { 106 | 'object': _Simple('dummy_value', 'other_value'), 107 | 'fields': ['field1', 'field3'], 108 | }, { 109 | 'object': _Outer('this is dotty'), 110 | 'fields': ['inner.field3'], 111 | }, { 112 | 'object': _Outer('what!?'), 113 | 'fields': ['inner.field4'], 114 | } 115 | ] 116 | for t in tests: 117 | self.assertRaises(AttributeError, 118 | bundling.compute_bundle_id, 119 | t['object'], 120 | t['fields']) 121 | 122 | 123 | def _return_request(req): 124 | """A dummy api call that simply returns the request.""" 125 | return req 126 | 127 | 128 | def _return_kwargs(dummy_req, **kwargs): 129 | """A dummy api call that simply returns its keyword arguments.""" 130 | return kwargs 131 | 132 | 133 | def _make_a_test_task(api_call=_return_request): 134 | return bundling.Task( 135 | api_call, 136 | 'an_id', 137 | 'field1', 138 | _Simple([]), 139 | dict()) 140 | 141 | 142 | def _extend_with_n_elts(a_task, elt, n): 143 | return a_task.extend([elt] * n) 144 | 145 | 146 | def _raise_exc(dummy_req): 147 | """A dummy api call that raises an exception""" 148 | raise ValueError('Raised in a test') 149 | 150 | 151 | class TestTask(unittest2.TestCase): 152 | 153 | def test_extend_increases_the_element_count(self): 154 | simple_msg = 'a simple msg' 155 | tests = [ 156 | { 157 | 'update': (lambda t: None), 158 | 'message': 'no messages added', 159 | 'want': 0 160 | }, { 161 | 'update': (lambda t: t.extend([simple_msg])), 162 | 'message': 'a single message added', 163 | 'want': 1 164 | }, { 165 | 'update': (lambda t: _extend_with_n_elts(t, simple_msg, 5)), 166 | 'message': 'a 5 messages added', 167 | 'want': 5 168 | } 169 | ] 170 | for t in tests: 171 | test_task = _make_a_test_task() 172 | t['update'](test_task) 173 | got = test_task.element_count 174 | message = 'bad message count when {}'.format(t['message']) 175 | self.assertEquals(got, t['want'], message) 176 | 177 | def test_extend_increases_the_request_byte_count(self): 178 | simple_msg = 'a simple msg' 179 | tests = [ 180 | { 181 | 'update': (lambda t: None), 182 | 'message': 'no messages added', 183 | 'want': 0 184 | }, { 185 | 'update': (lambda t: t.extend([simple_msg])), 186 | 'message': 'a single bundle message', 187 | 'want': len(simple_msg) 188 | }, { 189 | 'update': (lambda t: _extend_with_n_elts(t, simple_msg, 5)), 190 | 'message': '5 bundled messages', 191 | 'want': 5 * len(simple_msg) 192 | } 193 | ] 194 | for t in tests: 195 | test_task = _make_a_test_task() 196 | t['update'](test_task) 197 | got = test_task.request_bytesize 198 | message = 'bad message count when {}'.format(t['message']) 199 | self.assertEquals(got, t['want'], message) 200 | 201 | def test_run_sends_the_bundle_elements(self): 202 | simple_msg = 'a simple msg' 203 | tests = [ 204 | { 205 | 'update': (lambda t: None), 206 | 'message': 'no messages added', 207 | 'has_event': False, 208 | 'count_before_run': 0, 209 | 'want': [] 210 | }, { 211 | 'update': (lambda t: t.extend([simple_msg])), 212 | 'message': 'a single bundled message', 213 | 'has_event': True, 214 | 'count_before_run': 1, 215 | 'want': _Simple([simple_msg]) 216 | }, { 217 | 'update': (lambda t: _extend_with_n_elts(t, simple_msg, 5)), 218 | 'message': '5 bundle messages', 219 | 'has_event': True, 220 | 'count_before_run': 5, 221 | 'want': _Simple([simple_msg] * 5) 222 | } 223 | ] 224 | for t in tests: 225 | test_task = _make_a_test_task() 226 | event = t['update'](test_task) 227 | self.assertEquals(test_task.element_count, t['count_before_run']) 228 | test_task.run() 229 | self.assertEquals(test_task.element_count, 0) 230 | self.assertEquals(test_task.request_bytesize, 0) 231 | if t['has_event']: 232 | self.assertIsNotNone( 233 | event, 234 | 'expected event for {}'.format(t['message'])) 235 | got = event.result 236 | message = 'bad output when run with {}'.format(t['message']) 237 | self.assertEquals(got, t['want'], message) 238 | 239 | def test_run_adds_an_error_if_execution_fails(self): 240 | simple_msg = 'a simple msg' 241 | test_task = _make_a_test_task(api_call=_raise_exc) 242 | event = test_task.extend([simple_msg]) 243 | self.assertEquals(test_task.element_count, 1) 244 | test_task.run() 245 | self.assertEquals(test_task.element_count, 0) 246 | self.assertEquals(test_task.request_bytesize, 0) 247 | self.assertTrue(isinstance(event.result, ValueError)) 248 | 249 | def test_calling_the_canceller_stops_the_element_from_getting_sent(self): 250 | an_elt = 'a simple msg' 251 | another_msg = 'another msg' 252 | test_task = _make_a_test_task() 253 | an_event = test_task.extend([an_elt]) 254 | another_event = test_task.extend([another_msg]) 255 | self.assertEquals(test_task.element_count, 2) 256 | self.assertTrue(an_event.cancel()) 257 | self.assertEquals(test_task.element_count, 1) 258 | self.assertFalse(an_event.cancel()) 259 | self.assertEquals(test_task.element_count, 1) 260 | test_task.run() 261 | self.assertEquals(test_task.element_count, 0) 262 | self.assertEquals(_Simple([another_msg]), another_event.result) 263 | self.assertFalse(an_event.is_set()) 264 | self.assertIsNone(an_event.result) 265 | 266 | 267 | SIMPLE_DESCRIPTOR = BundleDescriptor('field1', []) 268 | DEMUX_DESCRIPTOR = BundleDescriptor('field1', [], subresponse_field='field1') 269 | 270 | 271 | class TestExecutor(unittest2.TestCase): 272 | 273 | def test_api_calls_are_grouped_by_bundle_id(self): 274 | an_elt = 'dummy message' 275 | api_call = _return_request 276 | bundle_ids = ['id1', 'id2'] 277 | threshold = 5 # arbitrary 278 | options = BundleOptions(element_count_threshold=threshold) 279 | bundler = bundling.Executor(options) 280 | for an_id in bundle_ids: 281 | for i in range(threshold - 1): 282 | got_event = bundler.schedule( 283 | api_call, 284 | an_id, 285 | SIMPLE_DESCRIPTOR, 286 | _Simple([an_elt]) 287 | ) 288 | self.assertIsNotNone( 289 | got_event.canceller, 290 | 'missing canceller after element #{}'.format(i)) 291 | self.assertFalse( 292 | got_event.is_set(), 293 | 'event unexpectedly set after element #{}'.format(i)) 294 | self.assertIsNone(got_event.result) 295 | for an_id in bundle_ids: 296 | got_event = bundler.schedule( 297 | api_call, 298 | an_id, 299 | SIMPLE_DESCRIPTOR, 300 | _Simple([an_elt]) 301 | ) 302 | self.assertIsNotNone(got_event.canceller, 303 | 'missing expected canceller') 304 | self.assertTrue( 305 | got_event.is_set(), 306 | 'event is not set after triggering element') 307 | self.assertEquals(_Simple([an_elt] * threshold), 308 | got_event.result) 309 | 310 | def test_each_event_has_exception_when_demuxed_api_call_fails(self): 311 | an_elt = 'dummy message' 312 | api_call = _raise_exc 313 | bundle_id = 'an_id' 314 | threshold = 5 # arbitrary, greater than 1 315 | options = BundleOptions(element_count_threshold=threshold) 316 | bundler = bundling.Executor(options) 317 | events = [] 318 | for i in range(threshold - 1): 319 | got_event = bundler.schedule( 320 | api_call, 321 | bundle_id, 322 | DEMUX_DESCRIPTOR, 323 | _Simple(['%s%d' % (an_elt, i)]) 324 | ) 325 | self.assertFalse( 326 | got_event.is_set(), 327 | 'event unexpectedly set after element #{}'.format(i)) 328 | self.assertIsNone(got_event.result) 329 | events.append(got_event) 330 | last_event = bundler.schedule( 331 | api_call, 332 | bundle_id, 333 | DEMUX_DESCRIPTOR, 334 | _Simple(['%s%d' % (an_elt, threshold - 1)]) 335 | ) 336 | events.append(last_event) 337 | 338 | previous_event = None 339 | for event in events: 340 | if previous_event: 341 | self.assertTrue(previous_event != event) 342 | self.assertTrue(event.is_set(), 343 | 'event is not set after triggering element') 344 | self.assertTrue(isinstance(event.result, ValueError)) 345 | previous_event = event 346 | 347 | def test_each_event_has_its_result_from_a_demuxed_api_call(self): 348 | an_elt = 'dummy message' 349 | api_call = _return_request 350 | bundle_id = 'an_id' 351 | threshold = 5 # arbitrary, greater than 1 352 | options = BundleOptions(element_count_threshold=threshold) 353 | bundler = bundling.Executor(options) 354 | events = [] 355 | 356 | # send 3 groups of elements of different sizes in the bundle 357 | for i in range(1, 4): 358 | got_event = bundler.schedule( 359 | api_call, 360 | bundle_id, 361 | DEMUX_DESCRIPTOR, 362 | _Simple(['%s%d' % (an_elt, i)] * i) 363 | ) 364 | events.append(got_event) 365 | previous_event = None 366 | for i, event in enumerate(events): 367 | index = i + 1 368 | if previous_event: 369 | self.assertTrue(previous_event != event) 370 | self.assertTrue(event.is_set(), 371 | 'event is not set after triggering element') 372 | self.assertEquals(event.result, 373 | _Simple(['%s%d' % (an_elt, index)] * index)) 374 | previous_event = event 375 | 376 | def test_each_event_has_same_result_from_mismatched_demuxed_api_call(self): 377 | an_elt = 'dummy message' 378 | mismatched_result = _Simple([an_elt, an_elt]) 379 | bundle_id = 'an_id' 380 | threshold = 5 # arbitrary, greater than 1 381 | options = BundleOptions(element_count_threshold=threshold) 382 | bundler = bundling.Executor(options) 383 | events = [] 384 | 385 | # send 3 groups of elements of different sizes in the bundle 386 | for i in range(1, 4): 387 | got_event = bundler.schedule( 388 | lambda x: mismatched_result, 389 | bundle_id, 390 | DEMUX_DESCRIPTOR, 391 | _Simple(['%s%d' % (an_elt, i)] * i) 392 | ) 393 | events.append(got_event) 394 | previous_event = None 395 | for i, event in enumerate(events): 396 | if previous_event: 397 | self.assertTrue(previous_event != event) 398 | self.assertTrue(event.is_set(), 399 | 'event is not set after triggering element') 400 | self.assertEquals(event.result, mismatched_result) 401 | previous_event = event 402 | 403 | def test_schedule_passes_kwargs(self): 404 | an_elt = 'dummy_msg' 405 | options = BundleOptions(element_count_threshold=1) 406 | bundle_id = 'an_id' 407 | bundler = bundling.Executor(options) 408 | event = bundler.schedule( 409 | _return_kwargs, 410 | bundle_id, 411 | SIMPLE_DESCRIPTOR, 412 | _Simple([an_elt]), 413 | {'an_option': 'a_value'} 414 | ) 415 | self.assertEquals('a_value', 416 | event.result['an_option']) 417 | 418 | 419 | class TestExecutor_ElementCountTrigger(unittest2.TestCase): 420 | 421 | def test_api_call_not_invoked_until_threshold(self): 422 | an_elt = 'dummy message' 423 | an_id = 'bundle_id' 424 | api_call = _return_request 425 | threshold = 3 # arbitrary 426 | options = BundleOptions(element_count_threshold=threshold) 427 | bundler = bundling.Executor(options) 428 | for i in range(threshold): 429 | got_event = bundler.schedule( 430 | api_call, 431 | an_id, 432 | SIMPLE_DESCRIPTOR, 433 | _Simple([an_elt]) 434 | ) 435 | self.assertIsNotNone( 436 | got_event.canceller, 437 | 'missing canceller after element #{}'.format(i)) 438 | if i + 1 < threshold: 439 | self.assertFalse(got_event.is_set()) 440 | self.assertIsNone(got_event.result) 441 | else: 442 | self.assertTrue(got_event.is_set()) 443 | self.assertEquals(_Simple([an_elt] * threshold), 444 | got_event.result) 445 | 446 | 447 | class TestExecutor_RequestByteTrigger(unittest2.TestCase): 448 | 449 | def test_api_call_not_invoked_until_threshold(self): 450 | an_elt = 'dummy message' 451 | an_id = 'bundle_id' 452 | api_call = _return_request 453 | elts_for_threshold = 3 454 | threshold = elts_for_threshold * len(an_elt) # arbitrary 455 | options = BundleOptions(request_byte_threshold=threshold) 456 | bundler = bundling.Executor(options) 457 | for i in range(elts_for_threshold): 458 | got_event = bundler.schedule( 459 | api_call, 460 | an_id, 461 | SIMPLE_DESCRIPTOR, 462 | _Simple([an_elt]) 463 | ) 464 | self.assertIsNotNone( 465 | got_event.canceller, 466 | 'missing canceller after element #{}'.format(i)) 467 | if i + 1 < elts_for_threshold: 468 | self.assertFalse(got_event.is_set()) 469 | self.assertIsNone(got_event.result) 470 | else: 471 | self.assertTrue(got_event.is_set()) 472 | self.assertEquals(_Simple([an_elt] * elts_for_threshold), 473 | got_event.result) 474 | 475 | 476 | class TestExecutor_DelayThreshold(unittest2.TestCase): 477 | 478 | @mock.patch('google.gax.bundling.TIMER_FACTORY') 479 | def test_api_call_is_scheduled_on_timer(self, timer_class): 480 | an_elt = 'dummy message' 481 | an_id = 'bundle_id' 482 | api_call = _return_request 483 | delay_threshold = 3 484 | options = BundleOptions(delay_threshold=delay_threshold) 485 | bundler = bundling.Executor(options) 486 | got_event = bundler.schedule( 487 | api_call, 488 | an_id, 489 | SIMPLE_DESCRIPTOR, 490 | _Simple([an_elt]) 491 | ) 492 | self.assertIsNotNone(got_event, 'missing event after first request') 493 | self.assertIsNone(got_event.result) 494 | self.assertTrue(timer_class.called) 495 | timer_args, timer_kwargs = timer_class.call_args_list[0] 496 | self.assertEquals(delay_threshold, timer_args[0]) 497 | self.assertEquals({'args': [an_id]}, timer_kwargs) 498 | timer_class.return_value.start.assert_called_once_with() 499 | 500 | 501 | class TestEvent(unittest2.TestCase): 502 | 503 | def test_can_be_set(self): 504 | ev = bundling.Event() 505 | self.assertFalse(ev.is_set()) 506 | ev.set() 507 | self.assertTrue(ev.is_set()) 508 | 509 | def test_can_be_cleared(self): 510 | ev = bundling.Event() 511 | ev.result = object() 512 | ev.set() 513 | self.assertTrue(ev.is_set()) 514 | self.assertIsNotNone(ev.result) 515 | ev.clear() 516 | self.assertFalse(ev.is_set()) 517 | self.assertIsNone(ev.result) 518 | 519 | def test_cancel_returns_false_without_canceller(self): 520 | ev = bundling.Event() 521 | self.assertFalse(ev.cancel()) 522 | 523 | def test_cancel_returns_canceller_result(self): 524 | ev = bundling.Event() 525 | ev.canceller = lambda: True 526 | self.assertTrue(ev.cancel()) 527 | ev.canceller = lambda: False 528 | self.assertFalse(ev.cancel()) 529 | 530 | def test_wait_does_not_block_if_event_is_set(self): 531 | ev = bundling.Event() 532 | ev.set() 533 | self.assertTrue(ev.wait()) 534 | --------------------------------------------------------------------------------