├── .codeclimate.yml ├── .gitignore ├── .travis.yml ├── LICENSE ├── MANIFEST.in ├── README.rst ├── docs ├── code │ ├── client.rst │ ├── exceptions.rst │ ├── helpers.rst │ ├── modules │ │ ├── client.rst │ │ ├── connection.rst │ │ ├── exc.rst │ │ ├── features.rst │ │ ├── iterables.rst │ │ ├── protocol.acl.rst │ │ ├── protocol.auth.rst │ │ ├── protocol.check.rst │ │ ├── protocol.children.rst │ │ ├── protocol.close.rst │ │ ├── protocol.connect.rst │ │ ├── protocol.create.rst │ │ ├── protocol.data.rst │ │ ├── protocol.delete.rst │ │ ├── protocol.exists.rst │ │ ├── protocol.part.rst │ │ ├── protocol.ping.rst │ │ ├── protocol.primitives.rst │ │ ├── protocol.reconfig.rst │ │ ├── protocol.request.rst │ │ ├── protocol.response.rst │ │ ├── protocol.sasl.rst │ │ ├── protocol.stat.rst │ │ ├── protocol.transaction.rst │ │ ├── protocol.watches.rst │ │ ├── recipes.allocator.rst │ │ ├── recipes.barrier.rst │ │ ├── recipes.base_lock.rst │ │ ├── recipes.base_watcher.rst │ │ ├── recipes.children_watcher.rst │ │ ├── recipes.counter.rst │ │ ├── recipes.data_watcher.rst │ │ ├── recipes.double_barrier.rst │ │ ├── recipes.election.rst │ │ ├── recipes.lease.rst │ │ ├── recipes.lock.rst │ │ ├── recipes.party.rst │ │ ├── recipes.proxy.rst │ │ ├── recipes.recipe.rst │ │ ├── recipes.sequential.rst │ │ ├── recipes.shared_lock.rst │ │ ├── recipes.tree_cache.rst │ │ ├── session.rst │ │ ├── states.rst │ │ └── transaction.rst │ ├── protocol.rst │ ├── protocol_basics.rst │ ├── recipe_basics.rst │ ├── recipes.rst │ └── sessions.rst ├── conf.py ├── index.rst ├── releases.rst ├── releases │ ├── 0.8.0.rst │ ├── 0.8.1.rst │ ├── 0.9.0.rst │ ├── 0.9.1.rst │ └── 0.9.2.rst ├── requirements.txt ├── source_docs.rst ├── static │ └── custom.css └── templates │ └── page.html ├── examples ├── __init__.py ├── allocator.py ├── barrier.py ├── child_watcher.py ├── counter.py ├── data_watcher.py ├── double_barrier.py ├── election.py ├── lease.py ├── locking.py ├── party.py ├── run ├── runtime_config.py ├── shared_locking.py └── transactions.py ├── setup.cfg ├── setup.py ├── tests ├── __init__.py ├── protocol │ ├── __init__.py │ ├── test_create.py │ ├── test_part.py │ ├── test_primitives.py │ ├── test_request.py │ ├── test_response.py │ └── test_transaction.py ├── test_client.py ├── test_connection.py ├── test_exc.py ├── test_features.py ├── test_iterables.py ├── test_retry.py ├── test_states.py └── test_transaction.py ├── tox.ini └── zoonado ├── __init__.py ├── client.py ├── compat.py ├── connection.py ├── encoding.py ├── exc.py ├── features.py ├── iterables.py ├── protocol ├── __init__.py ├── acl.py ├── auth.py ├── check.py ├── children.py ├── close.py ├── connect.py ├── create.py ├── data.py ├── delete.py ├── exists.py ├── part.py ├── ping.py ├── primitives.py ├── reconfig.py ├── request.py ├── response.py ├── sasl.py ├── stat.py ├── sync.py ├── transaction.py └── watches.py ├── recipes ├── __init__.py ├── allocator.py ├── barrier.py ├── base_lock.py ├── base_watcher.py ├── children_watcher.py ├── counter.py ├── data_watcher.py ├── double_barrier.py ├── election.py ├── lease.py ├── lock.py ├── party.py ├── proxy.py ├── recipe.py ├── sequential.py ├── shared_lock.py └── tree_cache.py ├── retry.py ├── session.py ├── states.py └── transaction.py /.codeclimate.yml: -------------------------------------------------------------------------------- 1 | engines: 2 | pep8: 3 | enabled: true 4 | duplication: 5 | enabled: true 6 | config: 7 | languages: 8 | - python 9 | radon: 10 | enabled: true 11 | config: 12 | threshold: "C" 13 | fixme: 14 | enabled: true 15 | ratings: 16 | paths: 17 | - "zoonado" 18 | exclude_paths: 19 | - "tests/*" 20 | - "docs/*" 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.egg-info 3 | .coverage 4 | .tox 5 | .docbuild 6 | *.egg 7 | build 8 | dist 9 | __pycache__ 10 | .cache 11 | htmlcov 12 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3.5 3 | envs: 4 | - TOXENV=py27 5 | - TOXENV=py35 6 | - TOXENV=pypy 7 | before_install: 8 | - pip install codeclimate-test-reporter 9 | install: pip install tox 10 | script: tox 11 | after_success: 12 | - codeclimate-test-reporter 13 | deploy: 14 | - provider: pypi 15 | user: wglass 16 | password: 17 | secure: xCWMjudi0Gu4ROcWnX3UCXTDz4ESSEEUonbX60DH/iqQuHK0k7LVBaSPLKhd0LMGQeD+V18vRDAB/O2vj0k+PW/I7fCHLsGiTSWbp1aAn4JJ/qv1KVPrXUCnivFWCv6oXNT5f8mqy88c25PJRBebxkYq+p9OT9wzpANZHMIs8+5a6BNW/WC/WcRuuvk3S6gSB5+36CgQ4jMjPh7qVuIT0SbSDRxAMTAr7jov5gBHcv2PsBItvMNDuCmde1o2ydI0vfHDlow9RdWz16T27AfNFH02QtA/lbFQ17kwEsB3gWR7aZk7BKfIoOllcRJU341VlZPyPSORxMqJ80uH74JncfqcQmnUfAu+oflmewBldDAAMIvQRL+6kPQjcWbgAoiHC4wzwI/Bc0U5SeqUkZyzefM4gbixc64mPajN8OUhslGF/BnKaMo+8FxIFVl+sAMobXXACUiNvypkzMt+A6wz5w41fge3gfwXFRBIH8iBN+EJjkIxQI3p1Jggvjoww+BLivAMSDgji19v0Bm91aoTV+JUMkA5irxSszNaCKxORD4gmuodONuHUJinNFLXImZS0bFdcB00aImgnm2NkRJFkdmVubS1YpU848M5tJu/c/OOu7BcbD5XjblpZiEWXJ0bX+c43oYVPM2J1HPwL5rynP48B2oN14AfhwyBRo5QK4g= 18 | on: 19 | tags: true 20 | distributions: sdist bdist_wheel 21 | repo: wglass/zoonado 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE README.rst 2 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ======================================= 2 | Zoonado: Async Tornado Zookeeper Client 3 | ======================================= 4 | 5 | .. image:: 6 | https://img.shields.io/pypi/v/zoonado.svg 7 | :alt: Python Package Version 8 | :target: http://pypi.python.org/pypi/zoonado 9 | .. image:: 10 | https://readthedocs.org/projects/zoonado/badge/?version=latest 11 | :alt: Documentation Status 12 | :target: http://zoonado.readthedocs.org/en/latest/ 13 | .. image:: 14 | https://travis-ci.org/wglass/zoonado.svg?branch=master 15 | :alt: Build Status 16 | :target: https://travis-ci.org/wglass/zoonado 17 | .. image:: 18 | https://codeclimate.com/github/wglass/zoonado/badges/gpa.svg 19 | :alt: Code Climate 20 | :target: https://codeclimate.com/github/wglass/zoonado 21 | .. image:: 22 | https://codeclimate.com/github/wglass/zoonado/badges/coverage.svg 23 | :alt: Test Coverage 24 | :target: https://codeclimate.com/github/wglass/zoonado/coverage 25 | 26 | .. 27 | 28 | Zoonado is a Zookeeper_ python client using Tornado_ to achieve async I/O. 29 | 30 | 31 | .. contents:: :local: 32 | 33 | 34 | Installation 35 | ~~~~~~~~~~~~ 36 | 37 | Zoonado is available via PyPI_, installation is as easy as:: 38 | 39 | pip install zoonado 40 | 41 | 42 | Quick Example 43 | ~~~~~~~~~~~~~ 44 | 45 | :: 46 | 47 | from tornado import gen 48 | from zoonado import Zoonado 49 | 50 | @gen.coroutine 51 | def run(): 52 | zk = Zoonado("zk01,zk02,zk03", chroot="/shared/namespace") 53 | 54 | yield zk.start() 55 | 56 | yield zk.create("/foo/bar", data="bazz", ephemeral=True) 57 | 58 | yield zk.set_data("/foo/bar", "bwee") 59 | 60 | yield zk.close() 61 | 62 | 63 | Development 64 | ~~~~~~~~~~~ 65 | 66 | The code is hosted on GitHub_ 67 | 68 | 69 | To file a bug or possible enhancement see the `Issue Tracker`_, also found 70 | on GitHub. 71 | 72 | 73 | License 74 | ~~~~~~~ 75 | 76 | Zoonado is licensed under the terms of the Apache license (2.0). See the 77 | LICENSE_ file for more details. 78 | 79 | 80 | .. _Zookeeper: https://zookeeper.apache.org 81 | .. _Tornado: http://tornadoweb.org 82 | .. _PyPI: https://pypi.python.org/pypi/zoonado 83 | .. _GitHub: https://github.com/wglass/zoonado 84 | .. _`Issue Tracker`: https://github.com/wglass/zoonado/issues 85 | .. _LICENSE: https://github.com/wglass/zoonado/blob/master/LICENSE 86 | -------------------------------------------------------------------------------- /docs/code/client.rst: -------------------------------------------------------------------------------- 1 | Client 2 | ====== 3 | 4 | .. toctree:: 5 | 6 | modules/client 7 | modules/features 8 | modules/transaction 9 | -------------------------------------------------------------------------------- /docs/code/exceptions.rst: -------------------------------------------------------------------------------- 1 | Exceptions 2 | ========== 3 | 4 | .. toctree:: 5 | 6 | modules/exc 7 | -------------------------------------------------------------------------------- /docs/code/helpers.rst: -------------------------------------------------------------------------------- 1 | Helper Modules 2 | ============== 3 | 4 | .. toctree:: 5 | 6 | modules/iterables 7 | -------------------------------------------------------------------------------- /docs/code/modules/client.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.client`` 2 | ================== 3 | 4 | .. automodule:: zoonado.client 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/connection.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.connection`` 2 | ====================== 3 | 4 | .. automodule:: zoonado.connection 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/exc.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.exc`` 2 | =============== 3 | 4 | .. automodule:: zoonado.exc 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/features.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.features`` 2 | ==================== 3 | 4 | .. automodule:: zoonado.features 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/iterables.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.iterables`` 2 | ===================== 3 | 4 | .. automodule:: zoonado.iterables 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/protocol.acl.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.protocol.acl`` 2 | ======================== 3 | 4 | .. automodule:: zoonado.protocol.acl 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/protocol.auth.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.protocol.auth`` 2 | ========================= 3 | 4 | .. automodule:: zoonado.protocol.auth 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/protocol.check.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.protocol.check`` 2 | ========================== 3 | 4 | .. automodule:: zoonado.protocol.check 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/protocol.children.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.protocol.children`` 2 | ============================= 3 | 4 | .. automodule:: zoonado.protocol.children 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/protocol.close.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.protocol.close`` 2 | ========================== 3 | 4 | .. automodule:: zoonado.protocol.close 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/protocol.connect.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.protocol.connect`` 2 | ============================ 3 | 4 | .. automodule:: zoonado.protocol.connect 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/protocol.create.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.protocol.create`` 2 | =========================== 3 | 4 | .. automodule:: zoonado.protocol.create 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/protocol.data.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.protocol.data`` 2 | ========================= 3 | 4 | .. automodule:: zoonado.protocol.data 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/protocol.delete.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.protocol.delete`` 2 | =========================== 3 | 4 | .. automodule:: zoonado.protocol.delete 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/protocol.exists.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.protocol.exists`` 2 | =========================== 3 | 4 | .. automodule:: zoonado.protocol.exists 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/protocol.part.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.protocol.part`` 2 | ========================= 3 | 4 | .. automodule:: zoonado.protocol.part 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/protocol.ping.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.protocol.ping`` 2 | ========================= 3 | 4 | .. automodule:: zoonado.protocol.ping 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/protocol.primitives.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.protocol.primitives`` 2 | =============================== 3 | 4 | .. automodule:: zoonado.protocol.primitives 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/protocol.reconfig.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.protocol.reconfig`` 2 | ============================= 3 | 4 | .. automodule:: zoonado.protocol.reconfig 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/protocol.request.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.protocol.request`` 2 | ============================ 3 | 4 | .. automodule:: zoonado.protocol.request 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/protocol.response.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.protocol.response`` 2 | ============================= 3 | 4 | .. automodule:: zoonado.protocol.response 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/protocol.sasl.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.protocol.sasl`` 2 | ========================= 3 | 4 | .. automodule:: zoonado.protocol.sasl 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/protocol.stat.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.protocol.stat`` 2 | ========================= 3 | 4 | .. automodule:: zoonado.protocol.stat 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/protocol.transaction.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.protocol.transaction`` 2 | ================================ 3 | 4 | .. automodule:: zoonado.protocol.transaction 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/protocol.watches.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.protocol.watches`` 2 | ============================ 3 | 4 | .. automodule:: zoonado.protocol.watches 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/recipes.allocator.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.recipes.allocator`` 2 | ============================= 3 | 4 | .. automodule:: zoonado.recipes.allocator 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/recipes.barrier.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.recipes.barrier`` 2 | =========================== 3 | 4 | .. automodule:: zoonado.recipes.barrier 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/recipes.base_lock.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.recipes.base_lock`` 2 | ============================= 3 | 4 | .. automodule:: zoonado.recipes.base_lock 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/recipes.base_watcher.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.recipes.base_watcher`` 2 | ================================ 3 | 4 | .. automodule:: zoonado.recipes.base_watcher 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/recipes.children_watcher.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.recipes.children_watcher`` 2 | ==================================== 3 | 4 | .. automodule:: zoonado.recipes.children_watcher 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/recipes.counter.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.recipes.counter`` 2 | =========================== 3 | 4 | .. automodule:: zoonado.recipes.counter 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/recipes.data_watcher.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.recipes.data_watcher`` 2 | ================================ 3 | 4 | .. automodule:: zoonado.recipes.data_watcher 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/recipes.double_barrier.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.recipes.double_barrier`` 2 | ================================== 3 | 4 | .. automodule:: zoonado.recipes.double_barrier 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/recipes.election.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.recipes.election`` 2 | ============================ 3 | 4 | .. automodule:: zoonado.recipes.election 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/recipes.lease.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.recipes.lease`` 2 | ========================= 3 | 4 | .. automodule:: zoonado.recipes.lease 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/recipes.lock.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.recipes.lock`` 2 | ======================== 3 | 4 | .. automodule:: zoonado.recipes.lock 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/recipes.party.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.recipes.party`` 2 | ========================= 3 | 4 | .. automodule:: zoonado.recipes.party 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/recipes.proxy.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.recipes.proxy`` 2 | ========================= 3 | 4 | .. automodule:: zoonado.recipes.proxy 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/recipes.recipe.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.recipes.recipe`` 2 | ========================== 3 | 4 | .. automodule:: zoonado.recipes.recipe 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/recipes.sequential.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.recipes.sequential`` 2 | ============================== 3 | 4 | .. automodule:: zoonado.recipes.sequential 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/recipes.shared_lock.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.recipes.shared_lock`` 2 | =============================== 3 | 4 | .. automodule:: zoonado.recipes.shared_lock 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/recipes.tree_cache.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.recipes.tree_cache`` 2 | ============================== 3 | 4 | .. automodule:: zoonado.recipes.tree_cache 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/session.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.session`` 2 | =================== 3 | 4 | .. automodule:: zoonado.session 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/states.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.states`` 2 | ================== 3 | 4 | .. automodule:: zoonado.states 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/modules/transaction.rst: -------------------------------------------------------------------------------- 1 | ``zoonado.transaction`` 2 | ======================= 3 | 4 | .. automodule:: zoonado.transaction 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/code/protocol.rst: -------------------------------------------------------------------------------- 1 | The Protocol 2 | ============ 3 | 4 | .. toctree:: 5 | 6 | modules/protocol.connect 7 | modules/protocol.close 8 | modules/protocol.ping 9 | modules/protocol.exists 10 | modules/protocol.stat 11 | modules/protocol.create 12 | modules/protocol.data 13 | modules/protocol.children 14 | modules/protocol.delete 15 | modules/protocol.watches 16 | modules/protocol.transaction 17 | modules/protocol.check 18 | modules/protocol.acl 19 | modules/protocol.sasl 20 | modules/protocol.auth 21 | modules/protocol.reconfig 22 | -------------------------------------------------------------------------------- /docs/code/protocol_basics.rst: -------------------------------------------------------------------------------- 1 | Protocol Basics 2 | =============== 3 | 4 | .. toctree:: 5 | 6 | modules/protocol.primitives 7 | modules/protocol.part 8 | modules/protocol.request 9 | modules/protocol.response 10 | -------------------------------------------------------------------------------- /docs/code/recipe_basics.rst: -------------------------------------------------------------------------------- 1 | Recipe Basics 2 | ============= 3 | 4 | .. toctree:: 5 | 6 | modules/recipes.recipe 7 | modules/recipes.sequential 8 | modules/recipes.proxy 9 | modules/recipes.base_watcher 10 | modules/recipes.base_lock 11 | -------------------------------------------------------------------------------- /docs/code/recipes.rst: -------------------------------------------------------------------------------- 1 | Recipes 2 | ======= 3 | 4 | .. toctree:: 5 | 6 | modules/recipes.lock 7 | modules/recipes.shared_lock 8 | modules/recipes.data_watcher 9 | modules/recipes.children_watcher 10 | modules/recipes.election 11 | modules/recipes.counter 12 | modules/recipes.barrier 13 | modules/recipes.double_barrier 14 | modules/recipes.lease 15 | modules/recipes.party 16 | modules/recipes.tree_cache 17 | modules/recipes.allocator 18 | -------------------------------------------------------------------------------- /docs/code/sessions.rst: -------------------------------------------------------------------------------- 1 | Sessions 2 | ======== 3 | 4 | .. toctree:: 5 | 6 | modules/session 7 | modules/connection 8 | modules/states 9 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. title:: Zoonado: Async Tornado Zookeeper Client 2 | 3 | Async Tornado Zookeeper Client 4 | ============================== 5 | 6 | 7 | .. 8 | 9 | Zoonado is a Zookeeper_ python client using Tornado_ to achieve async I/O. 10 | 11 | 12 | Installation 13 | ------------ 14 | 15 | Pip 16 | ~~~ 17 | 18 | Zoonado is available via PyPI_, installation is as easy as:: 19 | 20 | pip install zoonado 21 | 22 | 23 | Manual 24 | ~~~~~~ 25 | 26 | To install manually, first download and unzip the :current_tarball:`z`, then: 27 | 28 | .. parsed-literal:: 29 | 30 | tar -zxvf zoonado-|version|.tar.gz 31 | cd zoonado-|version| 32 | python setup.py install 33 | 34 | 35 | Development 36 | ----------- 37 | 38 | The code is hosted on GitHub_ 39 | 40 | 41 | To file a bug or possible enhancement see the `Issue Tracker`_, also found 42 | on GitHub. 43 | 44 | 45 | License 46 | ------- 47 | 48 | Zoonado is licensed under the terms of the Apache license (2.0). See the 49 | LICENSE_ file for more details. 50 | 51 | 52 | .. _Zookeeper: https://zookeeper.apache.org 53 | .. _Tornado: http://tornadoweb.org 54 | .. _PyPI: https://pypi.pyton.org/pypi/zoonado 55 | .. _GitHub: https://github.com/wglass/zoonado 56 | .. _`Issue Tracker`: https://github.com/wglass/zoonado/issues 57 | .. _LICENSE: https://github.com/wglass/zoonado/blob/master/LICENSE 58 | 59 | 60 | .. toctree:: 61 | :hidden: 62 | :titlesonly: 63 | :maxdepth: 2 64 | 65 | source_docs 66 | releases 67 | -------------------------------------------------------------------------------- /docs/releases.rst: -------------------------------------------------------------------------------- 1 | ================ 2 | Release Notes 3 | ================ 4 | 5 | .. toctree:: 6 | :maxdepth: 2 7 | :glob: 8 | 9 | releases/* 10 | -------------------------------------------------------------------------------- /docs/releases/0.8.0.rst: -------------------------------------------------------------------------------- 1 | 0.8.0 2 | ~~~~~ 3 | 4 | * Initial public release 5 | -------------------------------------------------------------------------------- /docs/releases/0.8.1.rst: -------------------------------------------------------------------------------- 1 | 0.8.1 2 | ~~~~~ 3 | 4 | Bugfixes 5 | 6 | * Fixed RetryPolicy.n_times to actually limit 7 | * Fixed RetryPolicy.until_elapsed to honor the given timeout 8 | * Fixed last_zxid to store the zxid and not the xid 9 | 10 | Improvements 11 | 12 | * Minimized shadowing of builtins 13 | * Eliminated unused named variables 14 | -------------------------------------------------------------------------------- /docs/releases/0.9.0.rst: -------------------------------------------------------------------------------- 1 | 0.9.0 2 | ~~~~~ 3 | 4 | API Changes 5 | 6 | * The Zoonado class now takes optional arguments data_serializer 7 | and data_deserializer, used for (de)serializing data payloads. 8 | 9 | Bugfixes 10 | 11 | * Many compatibility issues with python 3.5 fixed 12 | * Nested sub-recipes now have their client set correctly 13 | * Fixups to examples so they run more smoothly. 14 | 15 | Improvements 16 | 17 | * Test coverage up past 70% 18 | * Python 3.5 compatibility fixed up 19 | -------------------------------------------------------------------------------- /docs/releases/0.9.1.rst: -------------------------------------------------------------------------------- 1 | 0.9.1 2 | ~~~~~ 3 | 4 | Bugfixes 5 | ======== 6 | 7 | * Fixes issue where the Party recipe wouldn't set its members initially. 8 | -------------------------------------------------------------------------------- /docs/releases/0.9.2.rst: -------------------------------------------------------------------------------- 1 | 0.9.2 2 | ~~~~~ 3 | 4 | Bugfixes 5 | ======== 6 | 7 | * Fix arguments to a call to `connection.close(timeout)` (jleibund) 8 | * Fix various calls to `gen.with_timeout()` (jleibund) 9 | * Update Connection and Session to recover from lost connections (jleibund) 10 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | Sphinx 2 | sphinx-bootstrap-theme 3 | sphinxcontrib-spelling 4 | pyenchant 5 | -------------------------------------------------------------------------------- /docs/source_docs.rst: -------------------------------------------------------------------------------- 1 | Source Docs 2 | =========== 3 | 4 | .. toctree:: 5 | :titlesonly: 6 | 7 | code/client 8 | code/sessions 9 | code/recipe_basics 10 | code/recipes 11 | code/protocol_basics 12 | code/protocol 13 | code/exceptions 14 | code/helpers 15 | -------------------------------------------------------------------------------- /docs/static/custom.css: -------------------------------------------------------------------------------- 1 | .navbar-default { 2 | background-color: #476c9b; 3 | } 4 | .navbar-default a { 5 | text-decoration: none; 6 | } 7 | .navbar-default .navbar-brand { 8 | color: #ffffff; 9 | } 10 | .navbar-default .navbar-brand:hover { 11 | color: #d6e3f8; 12 | } 13 | .navbar-default .navbar-nav>li>a { 14 | color: #ffffff; 15 | } 16 | .navbar-default .navbar-nav>li>a:hover { 17 | color: #d6e3f8; 18 | } 19 | .navbar-form .form-control { 20 | color: #ffffff; 21 | } 22 | .navbar-form .form-control:focus { 23 | box-shadow: inset 0 -2px 0 #468c98; 24 | } 25 | .alert-info { 26 | background-color: #476c9b; 27 | } 28 | .alert-info code { 29 | color: #ffffff; 30 | background-color: #476c9b; 31 | } 32 | .alert-warning { 33 | background-color: #984447; 34 | } 35 | .alert-warning code { 36 | color: #ffffff; 37 | background-color: #984447; 38 | } 39 | .alert { 40 | color: #ffffff; 41 | } 42 | .alert a:not(.close) { 43 | color: #ffffff; 44 | font-weight: italic; 45 | text-decoration: underline; 46 | } 47 | .alert-warning a:not(.close) { 48 | color: #ffffff; 49 | font-style: italic; 50 | } 51 | a { 52 | color: #476c9b; 53 | font-weight: bold; 54 | } 55 | a:hover { 56 | color: #476c9b; 57 | text-decoration: underline; 58 | } 59 | code { 60 | color: #92140c; 61 | } 62 | dt:target, .highlighted { 63 | background-color: #c1b4ae; 64 | } 65 | -------------------------------------------------------------------------------- /docs/templates/page.html: -------------------------------------------------------------------------------- 1 | {# Import the theme's layout. #} 2 | {% extends "!page.html" %} 3 | 4 | {# Custom CSS overrides #} 5 | {% set bootswatch_css_custom = ['_static/custom.css'] %} 6 | -------------------------------------------------------------------------------- /examples/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wglass/zoonado/8f143b3dd26be88131356f731e7ca51809bc69cb/examples/__init__.py -------------------------------------------------------------------------------- /examples/allocator.py: -------------------------------------------------------------------------------- 1 | import collections 2 | import itertools 3 | import logging 4 | import random 5 | 6 | from tornado import gen, ioloop 7 | from zoonado.recipes.allocator import round_robin 8 | 9 | 10 | log = logging.getLogger(__name__) 11 | 12 | 13 | ANIMALS = ["cat", "dog", "mouse", "human"] 14 | 15 | 16 | def arguments(parser): 17 | parser.add_argument( 18 | "znode", type=str, 19 | help="Path of the base znode to use." 20 | ) 21 | parser.add_argument( 22 | "--workers", "-w", type=int, default=5, 23 | help="Number of worker coroutines to launch." 24 | ) 25 | parser.add_argument( 26 | "--items", "-n", type=int, default=17, 27 | help="Number of items to allocate amongst the workers." 28 | ) 29 | parser.add_argument( 30 | "--alloc-func", "-a", default="round_robin", 31 | choices=allocation_functions.keys(), 32 | help="Which allocation function to use." 33 | ) 34 | 35 | 36 | @gen.coroutine 37 | def run(client, args): 38 | items = set([ 39 | "%s::%s" % (i, random.choice(ANIMALS)) 40 | for i in range(args.items) 41 | ]) 42 | allocation_function = allocation_functions[args.alloc_func] 43 | 44 | yield client.start() 45 | 46 | for i in range(args.workers): 47 | ioloop.IOLoop.current().add_callback( 48 | worker, i, client, args.znode, allocation_function, items 49 | ) 50 | 51 | yield gen.sleep(10) 52 | 53 | 54 | @gen.coroutine 55 | def worker(number, client, znode_path, allocation_fn, items): 56 | name = "worker_%s" % number 57 | 58 | allocator = client.recipes.Allocator(znode_path, name, allocation_fn) 59 | 60 | yield allocator.start() 61 | 62 | yield allocator.update(items) 63 | 64 | while True: 65 | log.info("[WORKER %d] My set: %s", number, allocator.allocation) 66 | yield gen.sleep(2) 67 | 68 | 69 | def animal_buckets(members, items): 70 | animal_assignment = {} 71 | for member, animal in zip(itertools.cycle(members), ANIMALS): 72 | animal_assignment[animal] = member 73 | 74 | allocation = collections.defaultdict(set) 75 | for member, item in zip(itertools.cycle(members), items): 76 | animal = item.split("::")[1] 77 | allocation[animal_assignment[animal]].add(item) 78 | 79 | return allocation 80 | 81 | 82 | allocation_functions = { 83 | "round_robin": round_robin, 84 | "buckets": animal_buckets, 85 | } 86 | -------------------------------------------------------------------------------- /examples/barrier.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from tornado import gen, ioloop 4 | 5 | 6 | log = logging.getLogger() 7 | 8 | 9 | def arguments(parser): 10 | parser.add_argument( 11 | "--path", "-b", type=str, default="/example-barrier", 12 | help="ZNode path to use for the barrier." 13 | ) 14 | parser.add_argument( 15 | "--workers", "-w", type=int, default=5, 16 | help="Number of worker coroutines." 17 | ) 18 | 19 | 20 | @gen.coroutine 21 | def run(client, args): 22 | yield client.start() 23 | 24 | barrier = client.recipes.Barrier(args.path) 25 | 26 | yield barrier.create() 27 | 28 | for i in range(args.workers): 29 | yield gen.sleep(1) 30 | ioloop.IOLoop.current().add_callback(worker, i, client, args.path) 31 | 32 | yield gen.sleep(2) 33 | 34 | yield barrier.lift() 35 | 36 | yield gen.sleep(3) 37 | 38 | yield client.close() 39 | 40 | 41 | @gen.coroutine 42 | def worker(number, client, barrier_path): 43 | log.info("[WORKER #%d] Starting up", number) 44 | 45 | barrier = client.recipes.Barrier(barrier_path) 46 | 47 | log.info("[WORKER #%d] Waiting on barrier...", number) 48 | yield barrier.wait() 49 | 50 | while True: 51 | log.info("[WORKER #%d] Doing work!", number) 52 | yield gen.sleep(.3) 53 | -------------------------------------------------------------------------------- /examples/child_watcher.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import random 3 | from tornado import gen 4 | from zoonado import exc 5 | 6 | log = logging.getLogger() 7 | 8 | 9 | def arguments(parser): 10 | parser.add_argument( 11 | "--path", "-p", type=str, default="/examplewatcher", 12 | help="ZNode path to use for the example." 13 | ) 14 | 15 | 16 | def watcher_callback(children): 17 | children.sort() 18 | log.info("There are %d items now: %s", len(children), ", ".join(children)) 19 | 20 | 21 | @gen.coroutine 22 | def run(client, args): 23 | yield client.start() 24 | 25 | try: 26 | yield client.create(args.path) 27 | except exc.NodeExists: 28 | pass 29 | 30 | watcher = client.recipes.ChildrenWatcher() 31 | 32 | watcher.add_callback(args.path, watcher_callback) 33 | 34 | to_make = ["cat", "dog", "mouse", "human"] 35 | random.shuffle(to_make) 36 | 37 | for item in to_make: 38 | yield client.create(args.path + "/" + item, ephemeral=True) 39 | yield gen.sleep(1) 40 | 41 | for item in to_make: 42 | yield client.delete(args.path + "/" + item) 43 | yield gen.sleep(1) 44 | -------------------------------------------------------------------------------- /examples/counter.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import random 3 | 4 | from tornado import gen 5 | from zoonado import exc 6 | 7 | 8 | log = logging.getLogger(__name__) 9 | 10 | 11 | def arguments(parser): 12 | parser.add_argument( 13 | "--path", "-b", type=str, default="/example-counter", 14 | help="ZNode path to use for the barrier." 15 | ) 16 | parser.add_argument( 17 | "--workers", "-w", type=int, default=5, 18 | help="Number of worker coroutines." 19 | ) 20 | 21 | 22 | @gen.coroutine 23 | def run(client, args): 24 | yield client.start() 25 | 26 | value_history = [] 27 | 28 | def callback(new_value): 29 | value_history.append(new_value) 30 | 31 | watcher = client.recipes.DataWatcher() 32 | watcher.add_callback(args.path, callback) 33 | 34 | try: 35 | data = yield client.get_data(args.path) 36 | log.info("Initial value is %s", data) 37 | except exc.NoNode: 38 | log.info("Initial value is blank") 39 | 40 | yield [ 41 | worker(number, client, args) 42 | for number in range(args.workers) 43 | ] 44 | 45 | log.info("Value history: %s", " -> ".join(value_history)) 46 | 47 | yield client.close() 48 | 49 | 50 | @gen.coroutine 51 | def worker(number, client, args): 52 | log.info("[WORKER #%d] Starting up", number) 53 | 54 | counter = client.recipes.Counter(args.path) 55 | 56 | yield counter.start() 57 | 58 | for _ in range(2): 59 | op = random.choice(["incr", "incr", "decr", "decr", "set"]) 60 | if op == "incr": 61 | log.info("[WORKER #%d] Incrementing count", number) 62 | yield counter.incr() 63 | elif op == "decr": 64 | log.info("[WORKER #%d] Decrementing count", number) 65 | yield counter.decr() 66 | else: 67 | new_value = random.choice([4, 8, 12, 16]) 68 | log.info("[WORKER #%d] Setting count to %d", number, new_value) 69 | yield counter.set_value(new_value) 70 | 71 | yield counter.stop() 72 | -------------------------------------------------------------------------------- /examples/data_watcher.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import random 3 | from tornado import gen 4 | from zoonado import exc 5 | 6 | log = logging.getLogger() 7 | 8 | 9 | def arguments(parser): 10 | parser.add_argument( 11 | "--path", "-p", type=str, default="/examplewatcher", 12 | help="ZNode path to use for the example." 13 | ) 14 | 15 | 16 | def watcher_callback(new_data): 17 | left, right = new_data.split(":") 18 | log.info("Left: %s, Right: %s", left, right) 19 | 20 | 21 | @gen.coroutine 22 | def run(client, args): 23 | yield client.start() 24 | 25 | try: 26 | yield client.create(args.path) 27 | except exc.NodeExists: 28 | pass 29 | 30 | watcher = client.recipes.DataWatcher() 31 | 32 | watcher.add_callback(args.path, watcher_callback) 33 | 34 | choices = ["foo:bar", "bwee:bwoo", "derp:hork"] 35 | 36 | for _ in range(5): 37 | yield client.set_data(args.path, data=random.choice(choices)) 38 | yield gen.sleep(1) 39 | 40 | yield client.delete(args.path) 41 | -------------------------------------------------------------------------------- /examples/double_barrier.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import random 3 | 4 | from tornado import gen 5 | 6 | 7 | log = logging.getLogger() 8 | 9 | 10 | def arguments(parser): 11 | parser.add_argument( 12 | "--path", "-b", type=str, default="/example-barrier", 13 | help="ZNode path to use for the double barrier." 14 | ) 15 | parser.add_argument( 16 | "--workers", "-w", type=int, default=5, 17 | help="Number of worker coroutines." 18 | ) 19 | parser.add_argument( 20 | "--min-workers", "-m", type=int, default=3, 21 | help="Minimum number of workers required to lift the barrier." 22 | ) 23 | 24 | 25 | @gen.coroutine 26 | def run(client, args): 27 | yield client.start() 28 | 29 | workers = [] 30 | 31 | for i in range(args.workers): 32 | workers.append(worker(i, client, args)) 33 | yield gen.sleep(1) 34 | 35 | yield workers 36 | 37 | yield client.close() 38 | 39 | 40 | @gen.coroutine 41 | def worker(number, client, args): 42 | log.info("[WORKER #%d] Starting up", number) 43 | 44 | barrier = client.recipes.DoubleBarrier(args.path, args.min_workers) 45 | 46 | log.info("[WORKER #%d] Entering barrier...", number) 47 | yield barrier.enter() 48 | 49 | workload = random.choice([3, 4, 5]) 50 | 51 | log.info("[WORKER #%d] My workload is %d", number, workload) 52 | 53 | for i in range(workload): 54 | log.info("[WORKER #%d] Doing task %d", number, i + 1) 55 | yield gen.sleep(1) 56 | 57 | log.info("[WORKER #%d] Workload complete, leaving barrier", number) 58 | yield barrier.leave() 59 | 60 | log.info("[WORKER #%d] All done!", number) 61 | -------------------------------------------------------------------------------- /examples/election.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import random 3 | 4 | from tornado import gen 5 | 6 | 7 | log = logging.getLogger() 8 | 9 | 10 | def arguments(parser): 11 | parser.add_argument( 12 | "--workers", "-w", type=int, default=5, 13 | help="Number of workers to launch." 14 | ) 15 | parser.add_argument( 16 | "--znode-path", "-p", type=str, default="examplelock", 17 | help="ZNode path to use for the election." 18 | ) 19 | 20 | 21 | @gen.coroutine 22 | def run(client, args): 23 | log.info("Launching %d workers.", args.workers) 24 | yield client.start() 25 | 26 | order = list(range(args.workers)) 27 | random.shuffle(order) 28 | 29 | yield [worker(i, client, args) for i in order] 30 | 31 | yield client.close() 32 | 33 | 34 | @gen.coroutine 35 | def worker(number, client, args): 36 | election = client.recipes.LeaderElection(args.znode_path) 37 | 38 | yield election.join() 39 | 40 | if election.has_leadership: 41 | log.info("[WORKER #%d] I am the leader!", number) 42 | else: 43 | log.info("[WORKER #%d] not the leader.", number) 44 | 45 | yield election.resign() 46 | -------------------------------------------------------------------------------- /examples/lease.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import logging 3 | import random 4 | 5 | from tornado import gen, ioloop 6 | from zoonado import exc 7 | 8 | 9 | log = logging.getLogger() 10 | 11 | 12 | def arguments(parser): 13 | parser.add_argument( 14 | "--path", "-p", type=str, default="/example-task-lease" 15 | ) 16 | parser.add_argument( 17 | "--limit", "-l", type=int, default=1, 18 | help="Max number of simultaneous leases." 19 | ) 20 | 21 | 22 | @gen.coroutine 23 | def run(client, args): 24 | yield client.start() 25 | 26 | try: 27 | yield client.create(args.path) 28 | except exc.NodeExists: 29 | pass 30 | 31 | # simulate a cron job, same task fired at an interval 32 | for i in range(8): 33 | ioloop.IOLoop.current().add_callback(limited_task, i, client, args) 34 | yield gen.sleep(1) 35 | 36 | 37 | @gen.coroutine 38 | def limited_task(number, client, args): 39 | lease = client.recipes.Lease(args.path, args.limit) 40 | 41 | seconds = random.choice([1, 2, 3]) 42 | 43 | obtained = yield lease.obtain(duration=datetime.timedelta(seconds=seconds)) 44 | if obtained: 45 | log.info("[ITERATION #%d] Got lease for %d seconds", number, seconds) 46 | else: 47 | log.info("[ITERATION #%d] No lease available, can't work", number) 48 | -------------------------------------------------------------------------------- /examples/locking.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import random 3 | 4 | from tornado import gen 5 | 6 | 7 | log = logging.getLogger() 8 | 9 | 10 | def arguments(parser): 11 | parser.add_argument( 12 | "--workers", "-w", type=int, default=3, 13 | help="Number of workers to launch." 14 | ) 15 | parser.add_argument( 16 | "--lock-path", "-p", type=str, default="examplelock", 17 | help="ZNode path to use for the lock." 18 | ) 19 | 20 | 21 | @gen.coroutine 22 | def run(client, args): 23 | log.info("Launching %d workers.", args.workers) 24 | 25 | yield client.start() 26 | 27 | order = list(range(args.workers)) 28 | random.shuffle(order) 29 | 30 | yield [work(i, client, args) for i in order] 31 | 32 | yield client.close() 33 | 34 | 35 | @gen.coroutine 36 | def work(number, client, args): 37 | lock = client.recipes.Lock(args.lock_path) 38 | 39 | num_iterations = 3 40 | 41 | log.info("[WORKER #%d] Acquiring lock...", number) 42 | with (yield lock.acquire()) as check: 43 | log.info("[WORKER #%d] Got lock!", number) 44 | 45 | for _ in range(num_iterations): 46 | wait = random.choice([1, 2, 3]) 47 | if not check(): 48 | log.warn("[WORKER #%d] lost my lock!", number) 49 | break 50 | 51 | log.info("[WORKER #%d] working %d secs", number, wait) 52 | yield gen.sleep(wait) 53 | 54 | log.info("[WORKER #%d] Done!", number) 55 | -------------------------------------------------------------------------------- /examples/party.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import random 3 | 4 | from tornado import gen 5 | 6 | 7 | log = logging.getLogger() 8 | 9 | 10 | def arguments(parser): 11 | parser.add_argument( 12 | "--workers", "-w", type=int, default=5, 13 | help="Number of workers to launch." 14 | ) 15 | parser.add_argument( 16 | "--znode-path", "-p", type=str, default="examplelock", 17 | help="ZNode path to use for the election." 18 | ) 19 | 20 | 21 | @gen.coroutine 22 | def run(client, args): 23 | log.info("Launching %d workers.", args.workers) 24 | yield client.start() 25 | 26 | yield [ 27 | worker(i, client, args) 28 | for i in range(args.workers) 29 | ] 30 | 31 | yield client.close() 32 | 33 | 34 | @gen.coroutine 35 | def worker(number, client, args): 36 | party = client.recipes.Party(args.znode_path, "worker_%d" % number) 37 | 38 | log.info("[WORKER #%d] Joining the party", number) 39 | 40 | yield party.join() 41 | 42 | for _ in range(10): 43 | log.info("[WORKER #%d] Members I see: %s", number, party.members) 44 | yield gen.sleep(.5) 45 | should_leave = random.choice([False, False, True]) 46 | if should_leave: 47 | log.info("[WORKER #%d] Leaving the party temporarily", number) 48 | yield party.leave() 49 | yield gen.sleep(1) 50 | log.info("[WORKER #%d] Rejoining the party", number) 51 | yield party.join() 52 | 53 | yield party.leave() 54 | -------------------------------------------------------------------------------- /examples/run: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import argparse 3 | import logging 4 | 5 | from tornado import ioloop 6 | from zoonado import Zoonado 7 | 8 | 9 | examples_list = ( 10 | "runtime_config", 11 | "locking", 12 | "shared_locking", 13 | "election", 14 | "allocator", 15 | "barrier", 16 | "double_barrier", 17 | "child_watcher", 18 | "data_watcher", 19 | "party", 20 | "transactions", 21 | "lease", 22 | "counter", 23 | ) 24 | 25 | 26 | def get_target_module(example): 27 | try: 28 | return __import__( 29 | ".".join(["examples", example]), fromlist=["run", "arguments"] 30 | ) 31 | except (ImportError, AttributeError, ValueError) as e: 32 | print("Error loading example '%s': %s" % (example, str(e))) 33 | return None 34 | 35 | 36 | example_xref = { 37 | name: get_target_module(name) 38 | for name in examples_list 39 | } 40 | 41 | 42 | parser = argparse.ArgumentParser() 43 | parser.add_argument( 44 | "--verbose", "-v", action="count", 45 | help="Verbosity level. One for debug messages, two for protocol payloads." 46 | ) 47 | parser.add_argument( 48 | "--servers", default="localhost", 49 | help="Comma-delimited list of zookeeper hosts." 50 | ) 51 | parser.add_argument( 52 | "--chroot", default=None, 53 | help="Use a chroot path for the example." 54 | ) 55 | 56 | subparsers = parser.add_subparsers( 57 | title="examples", description="available examples", dest="example", 58 | help="Which available example to run." 59 | ) 60 | for name, module in example_xref.items(): 61 | if not module: 62 | continue 63 | subparser = subparsers.add_parser(name, help=module.__doc__) 64 | module.arguments(subparser) 65 | 66 | 67 | def main(): 68 | args = parser.parse_args() 69 | 70 | logging.basicConfig(level=logging.INFO) 71 | 72 | if not args.verbose: 73 | args.verbose = 0 74 | 75 | if args.verbose >= 1: 76 | logging.basicConfig(level=logging.DEBUG) 77 | logging.getLogger("zoonado").setLevel(logging.DEBUG) 78 | if args.verbose >= 2: 79 | logging.getLogger("zoonado.connection.payload").setLevel(logging.DEBUG) 80 | 81 | if not args.example: 82 | parser.print_help() 83 | return 84 | 85 | example = example_xref[args.example] 86 | 87 | client = Zoonado(args.servers, chroot=args.chroot) 88 | 89 | loop = ioloop.IOLoop.instance() 90 | 91 | def wind_down(f): 92 | try: 93 | f.result() 94 | finally: 95 | loop.stop() 96 | 97 | loop.add_future(example.run(client, args), wind_down) 98 | 99 | try: 100 | loop.start() 101 | except KeyboardInterrupt: 102 | loop.stop() 103 | 104 | if __name__ == "__main__": 105 | main() 106 | -------------------------------------------------------------------------------- /examples/runtime_config.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from tornado import ioloop, gen 4 | from zoonado import exc 5 | 6 | 7 | log = logging.getLogger() 8 | 9 | 10 | def arguments(_): 11 | pass 12 | 13 | 14 | @gen.coroutine 15 | def run(client, args): 16 | config_path = "/exampleconfig" 17 | loop = ioloop.IOLoop.current() 18 | 19 | yield client.start() 20 | 21 | config = client.recipes.TreeCache(config_path) 22 | 23 | yield config.start() 24 | 25 | try: 26 | yield client.create(config_path + "/running", data="yes") 27 | except exc.NodeExists: 28 | yield client.set_data(config_path + "/running", data="yes") 29 | 30 | for path in ["foo", "bar", "bazz", "bloo"]: 31 | try: 32 | yield client.create(config_path + "/" + path, data="1") 33 | except exc.NodeExists: 34 | yield client.set_data(config_path + "/" + path, data="1") 35 | 36 | loop.add_callback(foo, config) 37 | loop.add_callback(bar, config) 38 | loop.add_callback(bazz, config) 39 | loop.add_callback(bloo, config) 40 | 41 | yield gen.sleep(1) 42 | 43 | yield client.set_data(config_path + "/foo", "3") 44 | 45 | yield gen.sleep(1) 46 | 47 | yield client.set_data(config_path + "/bar", "2") 48 | 49 | yield client.set_data(config_path + "/bazz", "5") 50 | 51 | yield gen.sleep(6) 52 | 53 | yield client.set_data(config_path + "/running", data="no") 54 | 55 | yield gen.sleep(2) 56 | 57 | yield client.close() 58 | 59 | 60 | @gen.coroutine 61 | def foo(config): 62 | while config.running.value == "yes": 63 | log.info("[FOO] doing work for %s seconds!", config.foo.value) 64 | yield gen.sleep(int(config.foo.value)) 65 | 66 | log.info("[FOO] no longer working.") 67 | 68 | 69 | @gen.coroutine 70 | def bar(config): 71 | while config.running.value == "yes": 72 | log.info("[BAR] doing work for %s seconds!", config.bar.value) 73 | yield gen.sleep(int(config.bar.value)) 74 | 75 | log.info("[BAR] no longer working.") 76 | 77 | 78 | @gen.coroutine 79 | def bazz(config): 80 | while config.running.value == "yes": 81 | log.info("[BAZZ] doing work for %s seconds!", config.bazz.value) 82 | yield gen.sleep(int(config.bazz.value)) 83 | 84 | log.info("[BAZZ] no longer working.") 85 | 86 | 87 | @gen.coroutine 88 | def bloo(config): 89 | while config.running.value == "yes": 90 | log.info("[BLOO] doing work for %s seconds!", config.bloo.value) 91 | yield gen.sleep(int(config.bloo.value)) 92 | 93 | log.info("[BLOO] no longer working.") 94 | -------------------------------------------------------------------------------- /examples/shared_locking.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import random 3 | 4 | from tornado import gen 5 | 6 | log = logging.getLogger() 7 | 8 | 9 | def arguments(parser): 10 | parser.add_argument( 11 | "--reader-count", "-r", type=int, default=6, 12 | help="Number of readers to launch.", 13 | ) 14 | parser.add_argument( 15 | "--writer-count", "-w", type=int, default=2, 16 | help="Number of writers to launch.", 17 | ) 18 | 19 | 20 | @gen.coroutine 21 | def run(client, args): 22 | workers = ([reader] * args.reader_count) + ([writer] * args.writer_count) 23 | random.shuffle(workers) 24 | 25 | yield client.start() 26 | 27 | yield [ 28 | worker_func(i, client) 29 | for i, worker_func in enumerate(workers) 30 | ] 31 | 32 | yield client.close() 33 | 34 | 35 | @gen.coroutine 36 | def reader(number, client): 37 | lock = client.recipes.SharedLock("/examplelock") 38 | 39 | wait = random.choice([1, 2, 3]) 40 | 41 | log.info("[READER #%d] Acquiring lock...", number) 42 | with (yield lock.acquire_read()): 43 | log.info("[READER #%d] Got lock! Sleeping %d seconds", number, wait) 44 | yield gen.sleep(wait) 45 | log.info("[READER #%d] Done!", number) 46 | 47 | 48 | @gen.coroutine 49 | def writer(number, client): 50 | lock = client.recipes.SharedLock("/examplelock") 51 | 52 | wait = random.choice([2, 3]) 53 | 54 | log.info("[WRITER #%d] Acquiring lock...", number) 55 | with (yield lock.acquire_write()): 56 | log.info("[WRITER #%d] Got lock! Sleeping %d seconds", number, wait) 57 | yield gen.sleep(wait) 58 | log.info("[WRITER #%d] Done!", number) 59 | -------------------------------------------------------------------------------- /examples/transactions.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import random 3 | import threading 4 | 5 | from tornado import gen, ioloop 6 | from zoonado import Zoonado 7 | 8 | 9 | log = logging.getLogger() 10 | 11 | monitor_ioloop = None 12 | 13 | 14 | def arguments(_): 15 | pass 16 | 17 | 18 | @gen.coroutine 19 | def run(client, args): 20 | yield client.start() 21 | 22 | yield client.create("/shared-znode", ephemeral=True) 23 | 24 | monitor_thread = threading.Thread(target=monitor_data, args=(args,)) 25 | monitor_thread.start() 26 | 27 | threads = [ 28 | threading.Thread(name="A", target=launch_loop, args=(args,)), 29 | threading.Thread(name="B", target=launch_loop, args=(args,)), 30 | threading.Thread(name="C", target=launch_loop, args=(args,)), 31 | ] 32 | 33 | for thread in threads: 34 | thread.start() 35 | 36 | for thread in threads: 37 | thread.join() 38 | 39 | monitor_ioloop.stop() 40 | 41 | monitor_thread.join() 42 | 43 | yield client.close() 44 | 45 | 46 | def monitor_data(args): 47 | global monitor_ioloop 48 | 49 | name = threading.current_thread().name 50 | log.info("Launching loop in thread %s", name) 51 | 52 | io_loop = ioloop.IOLoop() 53 | 54 | io_loop.make_current() 55 | 56 | monitor_ioloop = io_loop 57 | 58 | @gen.coroutine 59 | def monitor(): 60 | client = Zoonado(args.servers, chroot=args.chroot) 61 | yield client.start() 62 | 63 | def data_callback(new_data): 64 | log.info("Shared data set to '%s'", new_data) 65 | 66 | watcher = client.recipes.DataWatcher() 67 | watcher.add_callback("/shared-znode", data_callback) 68 | 69 | yield gen.moment 70 | 71 | io_loop.add_callback(monitor) 72 | 73 | io_loop.start() 74 | 75 | 76 | def launch_loop(args): 77 | name = threading.current_thread().name 78 | log.info("Launching loop in thread %s", name) 79 | 80 | io_loop = ioloop.IOLoop() 81 | 82 | io_loop.make_current() 83 | 84 | io_loop.add_callback(update_loop, name, args, io_loop) 85 | 86 | io_loop.start() 87 | 88 | 89 | @gen.coroutine 90 | def update_loop(name, args, io_loop): 91 | log.info("[LOOP %s] starting up!", name) 92 | 93 | client = Zoonado(args.servers, chroot=args.chroot) 94 | 95 | try: 96 | yield client.start() 97 | 98 | for i in range(5): 99 | yield client.exists("/shared-znode") 100 | expected_version = client.stat_cache["/shared-znode"].version 101 | 102 | yield gen.sleep(random.choice([.2, .4, .5])) 103 | 104 | log.info( 105 | "[LOOP %s] I expect the shared znode to have version %s", 106 | name, client.stat_cache["/shared-znode"].version 107 | ) 108 | 109 | txn = client.begin_transaction() 110 | 111 | txn.create("/znode-" + name, ephemeral=True) 112 | txn.check_version("/shared-znode", expected_version) 113 | txn.set_data("/shared-znode", "altered by loop %s!" % name) 114 | txn.delete("/znode-" + name) 115 | 116 | log.info("[LOOP %s] committing...", name) 117 | result = yield txn.commit() 118 | if not result: 119 | log.info("[LOOP %s] rolled back!", name) 120 | 121 | yield client.close() 122 | finally: 123 | io_loop.stop() 124 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [build_sphinx] 2 | source_dir = docs 3 | build_dir = .docbuild 4 | all_files = 1 5 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | from zoonado import __version__ 4 | 5 | 6 | setup( 7 | name="zoonado", 8 | version=__version__, 9 | description="Async tornado client for Zookeeper.", 10 | author="William Glass", 11 | author_email="william.glass@gmail.com", 12 | url="http://github.com/wglass/zoonado", 13 | license="Apache", 14 | keywords=["zookeeper", "tornado", "async", "distributed"], 15 | packages=find_packages(exclude=["tests", "tests.*"]), 16 | install_requires=[ 17 | "tornado>=4.1", 18 | "six" 19 | ], 20 | entry_points={ 21 | "zoonado.recipes": [ 22 | "data_watcher = zoonado.recipes.data_watcher:DataWatcher", 23 | "children_watcher" + 24 | " = zoonado.recipes.children_watcher:ChildrenWatcher", 25 | "lock = zoonado.recipes.lock:Lock", 26 | "shared_lock = zoonado.recipes.shared_lock:SharedLock", 27 | "lease = zoonado.recipes.lease:Lease", 28 | "barrier = zoonado.recipes.barrier:Barrier", 29 | "double_barrier = zoonado.recipes.double_barrier:DoubleBarrier", 30 | "election = zoonado.recipes.election:LeaderElection", 31 | "party = zoonado.recipes.party:Party", 32 | "counter = zoonado.recipes.counter:Counter", 33 | "tree_cache = zoonado.recipes.tree_cache:TreeCache", 34 | "allocator = zoonado.recipes.allocator:Allocator", 35 | ], 36 | }, 37 | tests_require=[ 38 | "nose", 39 | "mock", 40 | "coverage", 41 | "flake8>=3.0.0", 42 | ], 43 | classifiers=[ 44 | "Development Status :: 4 - Beta", 45 | "Intended Audience :: Developers", 46 | "License :: OSI Approved :: Apache Software License", 47 | "Operating System :: MacOS", 48 | "Operating System :: MacOS :: MacOS X", 49 | "Operating System :: POSIX", 50 | "Operating System :: POSIX :: Linux", 51 | "Operating System :: Unix", 52 | "Programming Language :: Python", 53 | "Programming Language :: Python :: 2.7", 54 | "Programming Language :: Python :: 3.4", 55 | "Programming Language :: Python :: 3.5", 56 | "Programming Language :: Python :: Implementation", 57 | "Programming Language :: Python :: Implementation :: CPython", 58 | "Programming Language :: Python :: Implementation :: PyPy", 59 | "Topic :: Software Development", 60 | "Topic :: Software Development :: Libraries", 61 | ], 62 | ) 63 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wglass/zoonado/8f143b3dd26be88131356f731e7ca51809bc69cb/tests/__init__.py -------------------------------------------------------------------------------- /tests/protocol/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wglass/zoonado/8f143b3dd26be88131356f731e7ca51809bc69cb/tests/protocol/__init__.py -------------------------------------------------------------------------------- /tests/protocol/test_create.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from zoonado.protocol import create 4 | 5 | 6 | class CreateProtocolTests(unittest.TestCase): 7 | 8 | def test_set_flags_defaults_to_false(self): 9 | c = create.CreateRequest(path="/foo/bar", data=None, acl=[]) 10 | 11 | c.set_flags() 12 | 13 | self.assertEqual(c.flags, 0) 14 | 15 | def test_setting_all_flags(self): 16 | c = create.CreateRequest(path="/foo/bar", data=None, acl=[]) 17 | 18 | c.set_flags(ephemeral=True, sequential=True, container=True) 19 | 20 | self.assertEqual(c.flags, 7) 21 | 22 | def test_ephemeral_flag_is_first_bit(self): 23 | c = create.CreateRequest(path="/foo/bar", data=None, acl=[]) 24 | 25 | c.set_flags(ephemeral=True) 26 | 27 | self.assertEqual(c.flags, 1) 28 | 29 | def test_sequential_flag_is_second_bit(self): 30 | c = create.CreateRequest(path="/foo/bar", data=None, acl=[]) 31 | 32 | c.set_flags(sequential=True) 33 | 34 | self.assertEqual(c.flags, 2) 35 | 36 | def test_container_flag_is_third_bit(self): 37 | c = create.CreateRequest(path="/foo/bar", data=None, acl=[]) 38 | 39 | c.set_flags(container=True) 40 | 41 | self.assertEqual(c.flags, 4) 42 | -------------------------------------------------------------------------------- /tests/protocol/test_part.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import struct 3 | import unittest 4 | 5 | from zoonado.protocol import part, primitives 6 | 7 | 8 | class PartTests(unittest.TestCase): 9 | 10 | def test_instantiation(self): 11 | 12 | class FakePart(part.Part): 13 | parts = ( 14 | ("first", primitives.Int), 15 | ("second", primitives.UString), 16 | ("third", primitives.Float), 17 | ) 18 | 19 | p = FakePart(first=8, second=u"foobar") 20 | 21 | self.assertEqual(p.first, 8) 22 | self.assertEqual(p.second, u"foobar") 23 | 24 | def test_unset_fields_set_to_none(self): 25 | 26 | class FakePart(part.Part): 27 | parts = ( 28 | ("first", primitives.Int), 29 | ("second", primitives.UString), 30 | ) 31 | 32 | p = FakePart(second=u"bazz") 33 | 34 | self.assertEqual(p.first, None) 35 | 36 | def test_passing_unknown_named_field(self): 37 | 38 | class FakePart(part.Part): 39 | parts = ( 40 | ("first", primitives.Int), 41 | ("second", primitives.UString), 42 | ) 43 | 44 | with self.assertRaises(ValueError): 45 | FakePart(other=5.5) 46 | 47 | def test_simple_rendering(self): 48 | 49 | class FakePart(part.Part): 50 | parts = ( 51 | ("first", primitives.Int), 52 | ("second", primitives.UString), 53 | ("third", primitives.Float), 54 | ) 55 | 56 | p = FakePart(first=8, second=u"foobar") 57 | 58 | fmt, data = p.render() 59 | 60 | self.assertEqual(fmt, "ii6sf") 61 | self.assertEqual(data, [8, 6, b"foobar", None]) 62 | 63 | def test_render_some_field(self): 64 | 65 | class FakePart(part.Part): 66 | parts = ( 67 | ("first", primitives.Int), 68 | ("second", primitives.UString), 69 | ("third", primitives.Float), 70 | ) 71 | 72 | p = FakePart(first=8, second=u"foobar", third=3.3) 73 | 74 | fmt, data = p.render([FakePart.parts[0], FakePart.parts[2]]) 75 | 76 | self.assertEqual(fmt, "if") 77 | self.assertEqual(data, [8, 3.3]) 78 | 79 | def test_render_with_nested_parts(self): 80 | 81 | class NestedPart(part.Part): 82 | parts = ( 83 | ("left", primitives.UString), 84 | ("right", primitives.UString), 85 | ) 86 | 87 | class FakePart(part.Part): 88 | parts = ( 89 | ("first", primitives.Int), 90 | ("second", primitives.UString), 91 | ("third", NestedPart), 92 | ) 93 | 94 | p = FakePart( 95 | second=u"foobar", third=NestedPart(left="up", right="down") 96 | ) 97 | 98 | fmt, data = p.render() 99 | 100 | self.assertEqual(fmt, "ii6si2si4s") 101 | self.assertEqual(data, [None, 6, b"foobar", 2, b"up", 4, b"down"]) 102 | 103 | def test_parse(self): 104 | 105 | class NestedPart(part.Part): 106 | parts = ( 107 | ("left", primitives.UString), 108 | ("right", primitives.UString), 109 | ) 110 | 111 | class FakePart(part.Part): 112 | parts = ( 113 | ("first", primitives.Int), 114 | ("second", primitives.UString), 115 | ("third", NestedPart), 116 | ) 117 | 118 | raw = struct.pack( 119 | "!ii6si2si4s", 8, 6, b"foobar", 2, b"up", 4, b"down" 120 | ) 121 | 122 | p, new_offset = FakePart.parse(raw, offset=0) 123 | 124 | self.assertEqual(new_offset, struct.calcsize("!ii6si2si4s")) 125 | 126 | self.assertEqual(p.first, 8) 127 | self.assertEqual(p.second, u"foobar") 128 | self.assertEqual(p.third, NestedPart(left=u"up", right=u"down")) 129 | 130 | def test_parse_with_unicode(self): 131 | 132 | class NestedPart(part.Part): 133 | parts = ( 134 | ("left", primitives.UString), 135 | ("right", primitives.UString), 136 | ) 137 | 138 | class FakePart(part.Part): 139 | parts = ( 140 | ("first", primitives.Int), 141 | ("second", primitives.UString), 142 | ("third", NestedPart), 143 | ) 144 | 145 | raw = struct.pack( 146 | "!ii7si2si4s", 8, 7, b"fo\xc3\xb8bar", 2, b"up", 4, b"down" 147 | ) 148 | 149 | p, new_offset = FakePart.parse(raw, offset=0) 150 | 151 | self.assertEqual(new_offset, struct.calcsize("!ii7si2si4s")) 152 | 153 | self.assertEqual(p.first, 8) 154 | self.assertEqual(p.second, u"foøbar") 155 | self.assertEqual(p.third, NestedPart(left=u"up", right=u"down")) 156 | 157 | def test_equality(self): 158 | 159 | class NestedPart(part.Part): 160 | parts = ( 161 | ("left", primitives.UString), 162 | ("right", primitives.UString), 163 | ) 164 | 165 | class FakePart(part.Part): 166 | parts = ( 167 | ("first", primitives.Int), 168 | ("second", primitives.UString), 169 | ("third", NestedPart), 170 | ) 171 | 172 | p1 = FakePart( 173 | second=u"foobar", 174 | third=NestedPart(left="up", right="down"), 175 | ) 176 | 177 | p2 = FakePart( 178 | third=NestedPart(right="down", left="up"), 179 | second=u"foobar", 180 | ) 181 | 182 | self.assertEqual(p1, p2) 183 | 184 | def test_inequality(self): 185 | 186 | class NestedPart(part.Part): 187 | parts = ( 188 | ("left", primitives.UString), 189 | ("right", primitives.UString), 190 | ) 191 | 192 | class FakePart(part.Part): 193 | parts = ( 194 | ("first", primitives.Int), 195 | ("second", primitives.UString), 196 | ("third", NestedPart), 197 | ) 198 | 199 | p1 = FakePart( 200 | second=u"foobar", 201 | third=NestedPart(left="up", right="down"), 202 | ) 203 | 204 | p2 = FakePart( 205 | third=NestedPart(left="down", right="up"), 206 | second=u"foobar", 207 | ) 208 | 209 | self.assertNotEqual(p1, p2) 210 | 211 | def test_string_repr(self): 212 | 213 | class NestedPart(part.Part): 214 | parts = ( 215 | ("left", primitives.UString), 216 | ("right", primitives.UString), 217 | ) 218 | 219 | class FakePart(part.Part): 220 | parts = ( 221 | ("first", primitives.Int), 222 | ("second", primitives.Vector.of(primitives.UString)), 223 | ("third", NestedPart), 224 | ) 225 | 226 | p = FakePart( 227 | second=[u"foobar", u"bleebloo"], 228 | third=NestedPart(left="up", right="down"), 229 | ) 230 | 231 | self.assertEqual( 232 | str(p), 233 | "FakePart(" + 234 | "first=None, " + 235 | "second=[foobar, bleebloo], " + 236 | "third=NestedPart(left=up, right=down))" 237 | ) 238 | -------------------------------------------------------------------------------- /tests/protocol/test_primitives.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import struct 3 | import unittest 4 | 5 | from zoonado.protocol import primitives, part 6 | 7 | 8 | class PrimitivesTests(unittest.TestCase): 9 | 10 | def test_basic_equality(self): 11 | int1 = primitives.Int(8) 12 | int2 = primitives.Int(8) 13 | 14 | assert int1 == int2 15 | 16 | def test_basic_string_repr(self): 17 | self.assertEqual(str(primitives.Float(1.3)), "Float(1.3)") 18 | 19 | def test_basic_parse_return_tuple(self): 20 | a_long = primitives.Long(3.33) 21 | 22 | fmt, values = a_long.render() 23 | 24 | self.assertEqual(fmt, "q") 25 | self.assertEqual(values, [3.33]) 26 | 27 | def test_variable_render_value_must_be_defined(self): 28 | v = primitives.VariablePrimitive([]) 29 | 30 | with self.assertRaises(NotImplementedError): 31 | v.render_value(["foo"]) 32 | 33 | def test_variable_parse_value_must_be_defined(self): 34 | with self.assertRaises(NotImplementedError): 35 | primitives.VariablePrimitive.parse_value(["bar"]) 36 | 37 | def test_vector_rendering(self): 38 | v = primitives.Vector.of(primitives.Double)([1, 2.9, 3, 4.0]) 39 | 40 | fmt, values = v.render() 41 | 42 | self.assertEqual(fmt, "idddd") 43 | self.assertEqual(values, [4, 1, 2.9, 3, 4.0]) 44 | 45 | def test_vector_rendering_of_parts(self): 46 | 47 | class MyPart(part.Part): 48 | parts = ( 49 | ("foo", primitives.Long), 50 | ) 51 | 52 | part1 = MyPart(foo=3.3333) 53 | part2 = MyPart(foo=1.2344) 54 | 55 | v = primitives.Vector.of(MyPart)([part1, part2]) 56 | 57 | fmt, values = v.render() 58 | 59 | self.assertEqual(fmt, "iqq") 60 | self.assertEqual(values, [2, 3.3333, 1.2344]) 61 | 62 | def test_empty_vector_rendering(self): 63 | v = primitives.Vector.of(primitives.Double)([]) 64 | 65 | fmt, values = v.render() 66 | 67 | self.assertEqual(fmt, "i") 68 | self.assertEqual(values, [0]) 69 | 70 | def test_none_vector_rendering(self): 71 | v = primitives.Vector.of(primitives.Double)(None) 72 | 73 | fmt, values = v.render() 74 | 75 | self.assertEqual(fmt, "i") 76 | self.assertEqual(values, [0]) 77 | 78 | def test_vector_parsing(self): 79 | raw = struct.pack("!iddd", 3, 100, 1440, 1200 * 1200) 80 | 81 | print(struct.unpack("!iddd", raw)) 82 | values, new_offset = primitives.Vector.of(primitives.Double).parse( 83 | raw, offset=0 84 | ) 85 | 86 | self.assertEqual(values, [100.0, 1440.0, 1200.0 * 1200]) 87 | self.assertEqual(new_offset, struct.calcsize("!iddd")) 88 | 89 | def test_string_rendering(self): 90 | s = primitives.UString(u"foobar") 91 | 92 | fmt, values = s.render() 93 | 94 | self.assertEqual(fmt, "i6s") 95 | self.assertEqual(values, [6, b"foobar"]) 96 | 97 | def test_none_string_rendering(self): 98 | s = primitives.UString(None) 99 | 100 | fmt, values = s.render() 101 | 102 | self.assertEqual(fmt, "i") 103 | self.assertEqual(values, [-1]) 104 | 105 | def test_none_string_parsing(self): 106 | raw = struct.pack("!i", -1) 107 | 108 | value, new_offset = primitives.UString.parse(raw, offset=0) 109 | 110 | self.assertEqual(value, None) 111 | self.assertEqual(new_offset, 4) 112 | 113 | def test_blank_string_rendering(self): 114 | s = primitives.UString("") 115 | 116 | fmt, values = s.render() 117 | 118 | self.assertEqual(fmt, "i0s") 119 | self.assertEqual(values, [0, b""]) 120 | 121 | def test_buffer_rendering(self): 122 | b = primitives.Buffer(b'asdf') 123 | 124 | fmt, values = b.render() 125 | 126 | self.assertEqual(fmt, "i4s") 127 | self.assertEqual(values, [4, b"asdf"]) 128 | 129 | def test_empty_buffer_rendering(self): 130 | b = primitives.Buffer(None) 131 | 132 | fmt, values = b.render() 133 | 134 | self.assertEqual(fmt, "i") 135 | self.assertEqual(values, [-1]) 136 | 137 | def test_buffer_parsing(self): 138 | raw = struct.pack("!i3s", 3, b"fo\xc3") 139 | 140 | value, new_offset = primitives.Buffer.parse(raw, offset=0) 141 | 142 | self.assertEqual(value, b"fo\xc3") 143 | self.assertEqual(new_offset, 7) 144 | 145 | def test_ustring_string(self): 146 | s = primitives.UString(u"foobar") 147 | 148 | self.assertEqual(str(s), str(u"foobar")) 149 | 150 | def test_vector_string(self): 151 | a = primitives.Vector.of(primitives.Int)([1, 3, 6, 9]) 152 | 153 | self.assertEqual(str(a), "Int[1, 3, 6, 9]") 154 | 155 | def test_ustring_render_parse_is_stable(self): 156 | s = primitives.UString(u"foobar") 157 | 158 | fmt, values = s.render() 159 | 160 | raw = struct.pack("!" + fmt, *values) 161 | 162 | value, _ = primitives.UString.parse(raw, 0) 163 | 164 | self.assertEqual(value, u"foobar") 165 | 166 | def test_ustring_render_parse_handles_strings(self): 167 | s = primitives.UString("bazzzz") 168 | 169 | fmt, values = s.render() 170 | 171 | raw = struct.pack("!" + fmt, *values) 172 | 173 | value, _ = primitives.UString.parse(raw, 0) 174 | 175 | self.assertEqual(value, u"bazzzz") 176 | -------------------------------------------------------------------------------- /tests/protocol/test_request.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from mock import patch 3 | 4 | from zoonado.protocol import request, primitives 5 | 6 | 7 | class RequestTests(unittest.TestCase): 8 | 9 | @patch.object(request, "struct") 10 | def test_serialize_without_xid_or_opcode(self, mock_struct): 11 | mock_struct.pack.return_value = b"fake result" 12 | 13 | class FakeRequest(request.Request): 14 | parts = ( 15 | ("first", primitives.Int), 16 | ("second", primitives.UString), 17 | ) 18 | 19 | r = FakeRequest(first=3, second=u"foobar") 20 | 21 | result = r.serialize() 22 | 23 | self.assertEqual(result, b"fake result") 24 | 25 | mock_struct.pack.assert_called_once_with("!ii6s", 3, 6, b'foobar') 26 | 27 | @patch.object(request, "struct") 28 | def test_serialize_with_xid(self, mock_struct): 29 | mock_struct.pack.return_value = b"fake result" 30 | 31 | class FakeRequest(request.Request): 32 | parts = ( 33 | ("first", primitives.Int), 34 | ("second", primitives.UString), 35 | ) 36 | 37 | r = FakeRequest(first=3, second=u"foobar") 38 | 39 | result = r.serialize(xid=12) 40 | 41 | self.assertEqual(result, b"fake result") 42 | 43 | mock_struct.pack.assert_called_once_with("!iii6s", 12, 3, 6, b'foobar') 44 | 45 | @patch.object(request, "struct") 46 | def test_serialize_with_xid_and_opcode(self, mock_struct): 47 | mock_struct.pack.return_value = b"fake result" 48 | 49 | class FakeRequest(request.Request): 50 | opcode = 99 51 | 52 | parts = ( 53 | ("first", primitives.Int), 54 | ("second", primitives.UString), 55 | ) 56 | 57 | r = FakeRequest(first=3, second=u"foobar") 58 | 59 | result = r.serialize(xid=12) 60 | 61 | self.assertEqual(result, b"fake result") 62 | 63 | mock_struct.pack.assert_called_once_with( 64 | "!iiii6s", 12, 99, 3, 6, b'foobar' 65 | ) 66 | -------------------------------------------------------------------------------- /tests/protocol/test_response.py: -------------------------------------------------------------------------------- 1 | import struct 2 | import unittest 3 | 4 | from zoonado.protocol import response, primitives 5 | 6 | 7 | class ResponseTests(unittest.TestCase): 8 | 9 | def test_deserialize(self): 10 | 11 | class FakeResponse(response.Response): 12 | opcode = 99 13 | 14 | parts = ( 15 | ("first", primitives.Int), 16 | ("second", primitives.UString), 17 | ) 18 | 19 | # note that the xid and opcode are omitted, they're part of a preamble 20 | # that a connection would use to determine which Response to use 21 | # for deserializing 22 | raw = struct.pack("!ii6s", 3, 6, b"foobar") 23 | 24 | result = FakeResponse.deserialize(raw) 25 | 26 | self.assertEqual(result.first, 3) 27 | self.assertEqual(result.second, u"foobar") 28 | -------------------------------------------------------------------------------- /tests/protocol/test_transaction.py: -------------------------------------------------------------------------------- 1 | import struct 2 | import unittest 3 | 4 | from mock import patch 5 | 6 | from zoonado import exc 7 | from zoonado.protocol import transaction, request, response, primitives 8 | 9 | 10 | class FakeOpRequest(request.Request): 11 | opcode = 99 12 | parts = ( 13 | ("foo", primitives.UString), 14 | ) 15 | 16 | 17 | class FakeOpResponse(response.Response): 18 | opcode = 99 19 | parts = ( 20 | ("foo", primitives.UString), 21 | ) 22 | 23 | 24 | class FakeSpecialRequest(request.Request): 25 | opcode = 999 26 | special_xid = -99 27 | parts = () 28 | 29 | 30 | class FakeSpecialResponse(response.Response): 31 | opcode = 999 32 | parts = () 33 | 34 | 35 | fake_response_xref = { 36 | response_class.opcode: response_class 37 | for response_class in (FakeOpResponse, FakeSpecialResponse) 38 | } 39 | 40 | 41 | class TransactionTests(unittest.TestCase): 42 | 43 | def test_request_serialize(self): 44 | txn = transaction.TransactionRequest() 45 | 46 | txn.add(FakeOpRequest(foo="bar")) 47 | txn.add(FakeSpecialRequest()) 48 | 49 | result = txn.serialize(xid=20) 50 | 51 | expected = struct.pack( 52 | "!" + "".join([ 53 | 'i', # xid 54 | 'i', # txn opcode 55 | 'i?i', # multiheader: opcode, done, error 56 | 'i3s', # three-character string 57 | 'i?i', # multiheader 58 | '', # special request's body is blank 59 | 'i?i', # ending multiheader 60 | ]), 61 | *[ 62 | 20, 63 | 14, 64 | 99, False, -1, 65 | 3, b"bar", 66 | 999, False, -1, 67 | -1, True, -1, 68 | ] 69 | ) 70 | 71 | self.assertEqual(result, expected) 72 | 73 | def test_request_stringified(self): 74 | txn = transaction.TransactionRequest() 75 | 76 | txn.add(FakeOpRequest(foo="bar")) 77 | txn.add(FakeSpecialRequest()) 78 | 79 | self.assertEqual( 80 | str(txn), 81 | "Txn[FakeOpRequest(foo=bar), FakeSpecialRequest()]" 82 | ) 83 | 84 | @patch.object(transaction, "response_xref", fake_response_xref) 85 | def test_response_deserialization(self): 86 | 87 | payload = struct.pack( 88 | "!" + "".join([ 89 | 'i?i', # multiheader: opcode, done, error 90 | 'i3s', # three-character string 91 | 'i?i', # multiheader 92 | '', # special request's body is blank 93 | 'i?i', # ending multiheader 94 | ]), 95 | *[ 96 | 99, False, -1, # 99 = opcode for fake op 97 | 3, b"bar", 98 | 999, False, -1, # 999 = opcode for fake special response 99 | -1, True, -1, 100 | ] 101 | ) 102 | 103 | response = transaction.TransactionResponse.deserialize(payload) 104 | 105 | self.assertIsInstance(response.responses[0], FakeOpResponse) 106 | self.assertEqual(response.responses[0].foo, "bar") 107 | self.assertIsInstance(response.responses[1], FakeSpecialResponse) 108 | 109 | @patch.object(transaction, "response_xref", fake_response_xref) 110 | def test_response_error_deserialization(self): 111 | 112 | # since transactions are atomic, if one response is an error, they 113 | # will all be 114 | payload = struct.pack( 115 | "!" + "".join([ 116 | 'i?i', # multiheader: opcode, done, error 117 | 'i', # error code 118 | 'i?i', # multiheader 119 | 'i', # error code 120 | 'i?i', # ending multiheader 121 | ]), 122 | *[ 123 | -1, False, -1, 124 | 0, # 'rolled back' error 125 | -1, False, -1, 126 | -3, # 'data inconsistency' error 127 | -1, True, -1, 128 | ] 129 | ) 130 | 131 | response = transaction.TransactionResponse.deserialize(payload) 132 | 133 | self.assertIsInstance(response.responses[0], exc.RolledBack) 134 | self.assertIsInstance(response.responses[1], exc.DataInconsistency) 135 | 136 | @patch.object(transaction, "response_xref", fake_response_xref) 137 | def test_response_stringified(self): 138 | payload = struct.pack( 139 | "!" + "".join([ 140 | 'i?i', # multiheader: opcode, done, error 141 | 'i3s', # three-character string 142 | 'i?i', # multiheader 143 | '', # special request's body is blank 144 | 'i?i', # ending multiheader 145 | ]), 146 | *[ 147 | 99, False, -1, 148 | 3, b"bar", 149 | 999, False, -1, 150 | -1, True, -1, 151 | ] 152 | ) 153 | 154 | response = transaction.TransactionResponse.deserialize(payload) 155 | 156 | self.assertEqual( 157 | str(response), 158 | "Txn[FakeOpResponse(foo=bar), FakeSpecialResponse()]" 159 | ) 160 | -------------------------------------------------------------------------------- /tests/test_connection.py: -------------------------------------------------------------------------------- 1 | from mock import patch, Mock, call 2 | from tornado import testing, concurrent 3 | 4 | from zoonado.protocol.connect import ConnectRequest, ConnectResponse 5 | from zoonado import connection 6 | 7 | 8 | class ConnectionTests(testing.AsyncTestCase): 9 | 10 | def setUp(self): 11 | super(ConnectionTests, self).setUp() 12 | 13 | self.response_buffer = bytearray() 14 | 15 | tcpclient_patcher = patch.object(connection, "tcpclient") 16 | mock_tcpclient = tcpclient_patcher.start() 17 | self.addCleanup(tcpclient_patcher.stop) 18 | 19 | self.mock_client = mock_tcpclient.TCPClient.return_value 20 | 21 | stream = Mock() 22 | self.mock_client.connect.return_value = self.future_value(stream) 23 | 24 | def read_some(num_bytes): 25 | result = self.response_buffer[:num_bytes] 26 | del self.response_buffer[:num_bytes] 27 | 28 | return self.future_value(result) 29 | 30 | def read_all(): 31 | result = self.response_buffer[:] 32 | 33 | self.response_buffer = bytearray() 34 | 35 | return self.future_value(result) 36 | 37 | stream.write.return_value = self.future_value(None) 38 | stream.read_bytes.side_effect = read_some 39 | stream.read_until_close.side_effect = read_all 40 | 41 | def future_value(self, value): 42 | f = concurrent.Future() 43 | f.set_result(value) 44 | return f 45 | 46 | def future_error(self, exception): 47 | f = concurrent.Future() 48 | f.set_exception(exception) 49 | return f 50 | 51 | @testing.gen_test 52 | def test_connect_gets_version_info(self): 53 | self.response_buffer.extend( 54 | b"""Zookeeper version: 3.4.6-1569965, built on 02/20/2014 09:09 GMT 55 | Latency min/avg/max: 0/0/1137 56 | Received: 21462 57 | Sent: 21474 58 | Connections: 2 59 | Outstanding: 0 60 | Zxid: 0x11171 61 | Mode: standalone 62 | Node count: 232""" 63 | ) 64 | 65 | conn = connection.Connection("local", 9999, Mock()) 66 | 67 | yield conn.connect() 68 | 69 | self.assertEqual(conn.version_info, (3, 4, 6)) 70 | self.assertEqual(conn.start_read_only, False) 71 | 72 | self.mock_client.connect.assert_has_calls([ 73 | call("local", 9999), 74 | call("local", 9999), 75 | ]) 76 | 77 | @testing.gen_test 78 | def test_send_connect_returns_none_on_error(self): 79 | conn = connection.Connection("local", 9999, Mock()) 80 | 81 | conn.stream = Mock() 82 | conn.stream.write.return_value = self.future_value(None) 83 | conn.stream.read_bytes.return_value = self.future_error(Exception("!")) 84 | 85 | result = yield conn.send_connect( 86 | ConnectRequest( 87 | protocol_version=0, 88 | last_seen_zxid=0, 89 | timeout=8000, 90 | session_id=0, 91 | password=b'\x00', 92 | read_only=False, 93 | ) 94 | ) 95 | 96 | self.assertEqual(result, None) 97 | 98 | @patch.object(connection.Connection, "read_response") 99 | @testing.gen_test 100 | def test_send_connect(self, read_response): 101 | conn = connection.Connection("local", 9999, Mock()) 102 | conn.stream = Mock() 103 | conn.stream.write.return_value = self.future_value(None) 104 | 105 | response = ConnectResponse( 106 | protocol_version=0, 107 | timeout=7000, 108 | session_id=123456, 109 | password=b"\xc3" 110 | ) 111 | 112 | read_response.return_value = self.future_value( 113 | ( 114 | 23, # xid 115 | 123, # zxid 116 | response, # response 117 | ) 118 | ) 119 | 120 | result = yield conn.send_connect( 121 | ConnectRequest( 122 | protocol_version=0, 123 | last_seen_zxid=0, 124 | timeout=8000, 125 | session_id=0, 126 | password=b'\x00', 127 | read_only=False, 128 | ) 129 | ) 130 | 131 | self.assertEqual(result, (123, response)) 132 | -------------------------------------------------------------------------------- /tests/test_exc.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from zoonado import exc 4 | 5 | 6 | class ExceptionTests(unittest.TestCase): 7 | 8 | def test_connect_error_string(self): 9 | e = exc.ConnectError("broker01", 9091, server_id=8) 10 | 11 | self.assertEqual(str(e), "Error connecting to broker01:9091") 12 | 13 | def test_response_error_string(self): 14 | e = exc.DataInconsistency() 15 | 16 | self.assertEqual(str(e), "DataInconsistency") 17 | 18 | def test_unknown_error_string(self): 19 | e = exc.UnknownError(-1000) 20 | 21 | self.assertEqual(str(e), "Unknown error code: -1000") 22 | 23 | def test_get_response_error(self): 24 | e = exc.get_response_error(-8) 25 | 26 | self.assertIsInstance(e, exc.BadArguments) 27 | 28 | def test_get_response_error_unknown(self): 29 | e = exc.get_response_error(-999) 30 | 31 | self.assertIsInstance(e, exc.UnknownError) 32 | 33 | self.assertEqual(e.error_code, -999) 34 | -------------------------------------------------------------------------------- /tests/test_features.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from zoonado import features 4 | 5 | 6 | class FeaturesTests(unittest.TestCase): 7 | 8 | def test_three_four_version(self): 9 | available = features.Features((3, 4, 8)) 10 | 11 | assert available.create_with_stat is False 12 | assert available.containers is False 13 | assert available.reconfigure is False 14 | 15 | def test_three_five_version(self): 16 | available = features.Features((3, 5, 0)) 17 | 18 | assert available.create_with_stat is True 19 | assert available.containers is False 20 | assert available.reconfigure is True 21 | 22 | def test_three_five_point_version(self): 23 | available = features.Features((3, 5, 1)) 24 | 25 | assert available.create_with_stat is True 26 | assert available.containers is True 27 | assert available.reconfigure is True 28 | -------------------------------------------------------------------------------- /tests/test_iterables.py: -------------------------------------------------------------------------------- 1 | import collections 2 | import unittest 3 | 4 | from zoonado import iterables 5 | 6 | 7 | class IterablesTests(unittest.TestCase): 8 | 9 | def test_drain_on_list(self): 10 | data = ["foo", 1, "bar", 9] 11 | 12 | result = list(iterables.drain(data)) 13 | 14 | self.assertEqual(len(data), 0) 15 | self.assertEqual(result, [9, "bar", 1, "foo"]) 16 | 17 | def test_drain_on_deque(self): 18 | data = collections.deque(["foo", 1, "bar", 9]) 19 | 20 | result = list(iterables.drain(data)) 21 | 22 | self.assertEqual(len(data), 0) 23 | self.assertEqual(result, ["foo", 1, "bar", 9]) 24 | 25 | def test_drain_on_set(self): 26 | data = set(["foo", 1, "bar", 9]) 27 | 28 | result = list(iterables.drain(data)) 29 | 30 | self.assertEqual(len(data), 0) 31 | self.assertEqual(set(result), set(["foo", 1, "bar", 9])) 32 | 33 | def test_drain_on_dict(self): 34 | data = {"foo": 1, "bar": 9} 35 | 36 | result = {key: value for key, value in iterables.drain(data)} 37 | 38 | self.assertEqual(len(data), 0) 39 | self.assertEqual(result, {"foo": 1, "bar": 9}) 40 | -------------------------------------------------------------------------------- /tests/test_retry.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from mock import patch, Mock 4 | from tornado import testing, concurrent 5 | 6 | from zoonado import retry, exc 7 | 8 | 9 | class RetryTests(testing.AsyncTestCase): 10 | 11 | @testing.gen_test 12 | def test_clear_timings(self): 13 | policy = retry.RetryPolicy.forever() 14 | 15 | request1 = Mock() 16 | request2 = Mock() 17 | 18 | yield policy.enforce(request1) 19 | yield policy.enforce(request1) 20 | yield policy.enforce(request2) 21 | 22 | self.assertEqual(len(policy.timings[id(request1)]), 2) 23 | self.assertEqual(len(policy.timings[id(request2)]), 1) 24 | 25 | policy.clear(request1) 26 | 27 | self.assertTrue(id(request1) not in policy.timings) 28 | self.assertEqual(len(policy.timings[id(request2)]), 1) 29 | 30 | def test_once(self): 31 | policy = retry.RetryPolicy.once() 32 | 33 | self.assertEqual(policy.try_limit, 1) 34 | self.assertEqual(policy.sleep_func([1, 2, 3]), None) 35 | 36 | def test_n_times(self): 37 | policy = retry.RetryPolicy.n_times(3) 38 | 39 | self.assertEqual(policy.try_limit, 3) 40 | self.assertEqual(policy.sleep_func([1, 2, 3]), None) 41 | 42 | def test_forever(self): 43 | policy = retry.RetryPolicy.forever() 44 | 45 | self.assertEqual(policy.try_limit, None) 46 | self.assertEqual(policy.sleep_func([1, 2, 3]), None) 47 | 48 | def test_exponential_backoff(self): 49 | policy = retry.RetryPolicy.exponential_backoff() 50 | 51 | self.assertEqual(policy.try_limit, None) 52 | 53 | timings = [] 54 | 55 | wait = policy.sleep_func(timings) 56 | timings.append(wait) 57 | 58 | self.assertEqual(wait, 1) 59 | 60 | wait = policy.sleep_func(timings) 61 | timings.append(wait) 62 | 63 | self.assertEqual(wait, 2) 64 | 65 | wait = policy.sleep_func(timings) 66 | timings.append(wait) 67 | 68 | self.assertEqual(wait, 4) 69 | 70 | wait = policy.sleep_func(timings) 71 | timings.append(wait) 72 | 73 | self.assertEqual(wait, 8) 74 | 75 | wait = policy.sleep_func(timings) 76 | timings.append(wait) 77 | 78 | self.assertEqual(wait, 16) 79 | 80 | def test_exponential_backoff_with_max_and_base(self): 81 | policy = retry.RetryPolicy.exponential_backoff(base=3, maximum=25) 82 | 83 | self.assertEqual(policy.try_limit, None) 84 | 85 | timings = [] 86 | 87 | wait = policy.sleep_func(timings) 88 | timings.append(wait) 89 | 90 | self.assertEqual(wait, 1) 91 | 92 | wait = policy.sleep_func(timings) 93 | timings.append(wait) 94 | 95 | self.assertEqual(wait, 3) 96 | 97 | wait = policy.sleep_func(timings) 98 | timings.append(wait) 99 | 100 | self.assertEqual(wait, 9) 101 | 102 | wait = policy.sleep_func(timings) 103 | timings.append(wait) 104 | 105 | self.assertEqual(wait, 25) 106 | 107 | wait = policy.sleep_func(timings) 108 | timings.append(wait) 109 | 110 | self.assertEqual(wait, 25) 111 | 112 | @patch.object(retry, "time") 113 | def test_until_elapsed(self, mock_time): 114 | state = {"now": time.time()} 115 | 116 | policy = retry.RetryPolicy.until_elapsed(timeout=4) 117 | 118 | def increment_time(*args): 119 | state["now"] += 1 120 | return state["now"] 121 | 122 | mock_time.time.side_effect = increment_time 123 | 124 | self.assertEqual(policy.try_limit, None) 125 | 126 | timings = [] 127 | 128 | wait = policy.sleep_func(timings) 129 | timings.append(state["now"]) 130 | 131 | self.assertEqual(wait, 3) 132 | self.assertEqual(policy.sleep_func(timings), 3) 133 | self.assertEqual(policy.sleep_func(timings), 2) 134 | self.assertEqual(policy.sleep_func(timings), 1) 135 | self.assertEqual(policy.sleep_func(timings), 0) 136 | 137 | @testing.gen_test 138 | def test_failed_enforcement(self): 139 | policy = retry.RetryPolicy.once() 140 | 141 | request = Mock() 142 | 143 | yield policy.enforce(request) 144 | 145 | with self.assertRaises(exc.FailedRetry): 146 | yield policy.enforce(request) 147 | 148 | @testing.gen_test 149 | def test_timeout_failure(self): 150 | 151 | def in_the_past(_): 152 | return -1 153 | 154 | request = Mock() 155 | 156 | policy = retry.RetryPolicy(try_limit=3, sleep_func=in_the_past) 157 | 158 | yield policy.enforce(request) 159 | 160 | with self.assertRaises(exc.FailedRetry): 161 | yield policy.enforce(request) 162 | 163 | @patch.object(retry.gen, "sleep") 164 | @testing.gen_test 165 | def test_enforcing_wait_time_sleeps(self, mock_gen_sleep): 166 | f = concurrent.Future() 167 | f.set_result(None) 168 | 169 | mock_gen_sleep.return_value = f 170 | 171 | def in_the_future(_): 172 | return 60 173 | 174 | request = Mock() 175 | 176 | policy = retry.RetryPolicy(try_limit=3, sleep_func=in_the_future) 177 | 178 | yield policy.enforce(request) 179 | 180 | self.assertFalse(mock_gen_sleep.called) 181 | 182 | yield policy.enforce(request) 183 | 184 | mock_gen_sleep.assert_called_once_with(60) 185 | -------------------------------------------------------------------------------- /tests/test_states.py: -------------------------------------------------------------------------------- 1 | from tornado import testing 2 | 3 | from zoonado import states 4 | 5 | 6 | class StatesTests(testing.AsyncTestCase): 7 | 8 | def setUp(self): 9 | super(StatesTests, self).setUp() 10 | 11 | self.fsm = states.SessionStateMachine() 12 | 13 | def test_defaults_to_lost_state(self): 14 | self.assertEqual(self.fsm.current_state, states.States.LOST) 15 | 16 | def test_fsm_equality(self): 17 | assert self.fsm == states.States.LOST 18 | assert self.fsm != states.States.CONNECTED 19 | assert self.fsm != states.States.SUSPENDED 20 | 21 | self.fsm.transition_to(states.States.CONNECTED) 22 | 23 | assert self.fsm != states.States.LOST 24 | assert self.fsm == states.States.CONNECTED 25 | assert self.fsm != states.States.SUSPENDED 26 | 27 | @testing.gen_test 28 | def test_waiting_for_a_state(self): 29 | wait = self.fsm.wait_for(states.States.CONNECTED) 30 | 31 | assert wait.done() is False 32 | 33 | self.fsm.transition_to(states.States.CONNECTED) 34 | 35 | yield wait 36 | 37 | @testing.gen_test 38 | def test_waiting_for_any_of_a_few_states(self): 39 | wait = self.fsm.wait_for( 40 | states.States.CONNECTED, states.States.READ_ONLY 41 | ) 42 | 43 | assert wait.done() is False 44 | 45 | self.fsm.transition_to(states.States.CONNECTED) 46 | 47 | yield wait 48 | 49 | self.fsm.transition_to(states.States.LOST) 50 | 51 | wait = self.fsm.wait_for( 52 | states.States.CONNECTED, states.States.READ_ONLY 53 | ) 54 | 55 | assert wait.done() is False 56 | 57 | self.fsm.transition_to(states.States.READ_ONLY) 58 | 59 | yield wait 60 | 61 | @testing.gen_test 62 | def test_waiting_on_current_state_yield_immediately(self): 63 | yield self.fsm.wait_for(states.States.LOST) 64 | 65 | def test_valid_transitions(self): 66 | # lost/initial sessions can connect 67 | assert self.fsm == states.States.LOST 68 | self.fsm.transition_to(states.States.CONNECTED) 69 | 70 | # connected sessions can be suspended 71 | assert self.fsm == states.States.CONNECTED 72 | self.fsm.transition_to(states.States.SUSPENDED) 73 | 74 | # suspended session can reconnect 75 | assert self.fsm == states.States.SUSPENDED 76 | self.fsm.transition_to(states.States.CONNECTED) 77 | 78 | # connected sessions can be lost (when closing) 79 | assert self.fsm == states.States.CONNECTED 80 | self.fsm.transition_to(states.States.LOST) 81 | 82 | # lost sessions can reconnect as read-only 83 | assert self.fsm == states.States.LOST 84 | self.fsm.transition_to(states.States.READ_ONLY) 85 | 86 | # read-only sessions can become fully connected 87 | assert self.fsm == states.States.READ_ONLY 88 | self.fsm.transition_to(states.States.CONNECTED) 89 | 90 | self.fsm.transition_to(states.States.SUSPENDED) 91 | 92 | # suspended sessions can reconnect as read-only 93 | assert self.fsm == states.States.SUSPENDED 94 | self.fsm.transition_to(states.States.READ_ONLY) 95 | 96 | # read-only sessions can be lost (when closing) 97 | assert self.fsm == states.States.READ_ONLY 98 | self.fsm.transition_to(states.States.LOST) 99 | 100 | self.fsm.transition_to(states.States.READ_ONLY) 101 | 102 | # read-only sessions can be suspended 103 | assert self.fsm == states.States.READ_ONLY 104 | self.fsm.transition_to(states.States.SUSPENDED) 105 | 106 | # suspended connections can be lost 107 | assert self.fsm == states.States.SUSPENDED 108 | self.fsm.transition_to(states.States.LOST) 109 | 110 | def test_invalid_transitions(self): 111 | # lost sessions cannot be re-lost 112 | with self.assertRaises(RuntimeError): 113 | self.fsm.transition_to(states.States.LOST) 114 | 115 | # lost sessions cannot be suspended 116 | with self.assertRaises(RuntimeError): 117 | self.fsm.transition_to(states.States.SUSPENDED) 118 | 119 | self.fsm.transition_to(states.States.CONNECTED) 120 | 121 | # connected sessions cannot become read-only (suspended or lost first) 122 | with self.assertRaises(RuntimeError): 123 | self.fsm.transition_to(states.States.READ_ONLY) 124 | 125 | # connected sessions cannot be connected again 126 | with self.assertRaises(RuntimeError): 127 | self.fsm.transition_to(states.States.CONNECTED) 128 | 129 | self.fsm.transition_to(states.States.SUSPENDED) 130 | 131 | # suspended sessions cannot be re-suspended 132 | with self.assertRaises(RuntimeError): 133 | self.fsm.transition_to(states.States.SUSPENDED) 134 | 135 | self.fsm.transition_to(states.States.READ_ONLY) 136 | 137 | # read-only sessions cannot be made read-only twice 138 | with self.assertRaises(RuntimeError): 139 | self.fsm.transition_to(states.States.READ_ONLY) 140 | -------------------------------------------------------------------------------- /tests/test_transaction.py: -------------------------------------------------------------------------------- 1 | from mock import Mock 2 | from tornado import testing, concurrent 3 | 4 | from zoonado import transaction, protocol, exc 5 | 6 | 7 | class TransactionTests(testing.AsyncTestCase): 8 | 9 | def test_instantiation(self): 10 | client = Mock() 11 | 12 | txn = transaction.Transaction(client) 13 | 14 | self.assertEqual(txn.client, client) 15 | self.assertIsInstance(txn.request, protocol.TransactionRequest) 16 | 17 | def test_accumulates_requests(self): 18 | txn = transaction.Transaction(Mock()) 19 | 20 | txn.check_version("/foo/bar", version=8) 21 | txn.set_data("/foo/bar", "some data") 22 | txn.delete("/foo/bazz") 23 | 24 | self.assertIsInstance( 25 | txn.request.requests[0], protocol.CheckVersionRequest 26 | ) 27 | self.assertIsInstance( 28 | txn.request.requests[1], protocol.SetDataRequest 29 | ) 30 | self.assertIsInstance( 31 | txn.request.requests[2], protocol.DeleteRequest 32 | ) 33 | 34 | def test_check_version_operation(self): 35 | client = Mock() 36 | 37 | txn = transaction.Transaction(client) 38 | 39 | txn.check_version("/foo/bar", version=8) 40 | 41 | sub_request = txn.request.requests[-1] 42 | 43 | self.assertIsInstance(sub_request, protocol.CheckVersionRequest) 44 | 45 | self.assertEqual(sub_request.path, client.normalize_path.return_value) 46 | client.normalize_path.assert_called_once_with("/foo/bar") 47 | self.assertEqual(sub_request.version, 8) 48 | 49 | def test_create_operation(self): 50 | client = Mock() 51 | client.features.create_with_stat = False 52 | 53 | txn = transaction.Transaction(client) 54 | 55 | txn.create("/foo/bar", data="data", ephemeral=True) 56 | 57 | sub_request = txn.request.requests[-1] 58 | 59 | self.assertIsInstance(sub_request, protocol.CreateRequest) 60 | 61 | self.assertEqual(sub_request.path, client.normalize_path.return_value) 62 | client.normalize_path.assert_called_once_with("/foo/bar") 63 | 64 | self.assertEqual(sub_request.data, "data") 65 | 66 | self.assertEqual(sub_request.acl, client.default_acl) 67 | 68 | self.assertEqual(sub_request.flags, 1) # just ephemeral flag set 69 | 70 | def test_create_can_use_create_with_stat(self): 71 | client = Mock() 72 | client.features.create_with_stat = True 73 | 74 | txn = transaction.Transaction(client) 75 | 76 | txn.create("/foo/bar", data="data", ephemeral=True) 77 | 78 | sub_request = txn.request.requests[-1] 79 | 80 | self.assertIsInstance(sub_request, protocol.Create2Request) 81 | 82 | def test_create_container_when_not_available(self): 83 | client = Mock() 84 | client.features.containers = False 85 | 86 | txn = transaction.Transaction(client) 87 | 88 | with self.assertRaises(ValueError): 89 | txn.create("/foo/bar", data="data", container=True) 90 | 91 | def test_set_data(self): 92 | client = Mock() 93 | 94 | txn = transaction.Transaction(client) 95 | 96 | txn.set_data("/foo/bar", '{"what": "hey"}', version=7) 97 | 98 | sub_request = txn.request.requests[-1] 99 | 100 | self.assertIsInstance(sub_request, protocol.SetDataRequest) 101 | 102 | self.assertEqual(sub_request.path, client.normalize_path.return_value) 103 | client.normalize_path.assert_called_once_with("/foo/bar") 104 | 105 | self.assertEqual(sub_request.data, '{"what": "hey"}') 106 | 107 | self.assertEqual(sub_request.version, 7) 108 | 109 | def test_delete(self): 110 | client = Mock() 111 | 112 | txn = transaction.Transaction(client) 113 | 114 | txn.delete("/foo/bazz", version=3) 115 | 116 | sub_request = txn.request.requests[-1] 117 | 118 | self.assertIsInstance(sub_request, protocol.DeleteRequest) 119 | 120 | self.assertEqual(sub_request.path, client.normalize_path.return_value) 121 | client.normalize_path.assert_called_once_with("/foo/bazz") 122 | 123 | self.assertEqual(sub_request.version, 3) 124 | 125 | @testing.gen_test 126 | def test_committing_with_no_operations(self): 127 | client = Mock() 128 | 129 | txn = transaction.Transaction(client) 130 | 131 | with self.assertRaises(ValueError): 132 | yield txn.commit() 133 | 134 | @testing.gen_test 135 | def test_commit_success(self): 136 | client = Mock() 137 | 138 | def fake_normalize(path): 139 | return "/normed" + path 140 | 141 | def fake_denormalize(path): 142 | return "/de" + path 143 | 144 | client.normalize_path.side_effect = fake_normalize 145 | client.denormalize_path.side_effect = fake_denormalize 146 | 147 | responses = [ 148 | protocol.CreateResponse(path="/foo/bar"), 149 | protocol.CheckVersionResponse(), 150 | protocol.SetDataResponse(stat=Mock()), 151 | protocol.DeleteResponse(), 152 | ] 153 | 154 | response = Mock(responses=responses) 155 | f = concurrent.Future() 156 | f.set_result(response) 157 | client.send.return_value = f 158 | 159 | txn = transaction.Transaction(client) 160 | 161 | txn.create("/foo/bar", data="bazz") 162 | txn.check_version("/foo/bazz", version=8) 163 | txn.set_data("/foo/bazz", "blee") 164 | txn.delete("/foo/bloo", version=5) 165 | 166 | result = yield txn.commit() 167 | 168 | assert result 169 | 170 | self.assertEqual(result.created, set(["/de/normed/foo/bar"])) 171 | self.assertEqual(result.checked, set(["/de/normed/foo/bazz"])) 172 | self.assertEqual(result.updated, set(["/de/normed/foo/bazz"])) 173 | self.assertEqual(result.deleted, set(["/de/normed/foo/bloo"])) 174 | 175 | @testing.gen_test 176 | def test_commit_failure(self): 177 | client = Mock() 178 | 179 | def fake_normalize(path): 180 | return "/normed" + path 181 | 182 | def fake_denormalize(path): 183 | return "/de" + path 184 | 185 | client.normalize_path.side_effect = fake_normalize 186 | client.denormalize_path.side_effect = fake_denormalize 187 | 188 | responses = [ 189 | exc.RuntimeInconsistency(), 190 | exc.RolledBack(), 191 | exc.DataInconsistency(), 192 | exc.RuntimeInconsistency(), 193 | ] 194 | 195 | response = Mock(responses=responses) 196 | f = concurrent.Future() 197 | f.set_result(response) 198 | client.send.return_value = f 199 | 200 | txn = transaction.Transaction(client) 201 | 202 | txn.create("/foo/bar", data="bazz") 203 | txn.check_version("/foo/bazz", version=8) 204 | txn.set_data("/foo/bazz", "blee") 205 | txn.delete("/foo/bloo", version=5) 206 | 207 | result = yield txn.commit() 208 | 209 | assert not result 210 | 211 | self.assertEqual(result.created, set([])) 212 | self.assertEqual(result.checked, set([])) 213 | self.assertEqual(result.updated, set([])) 214 | self.assertEqual(result.deleted, set([])) 215 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py27,py35,pypy 3 | skipsdist = True 4 | 5 | [testenv] 6 | usedevelop = True 7 | deps = 8 | pytest 9 | pytest-cov 10 | pytest-flake8 11 | mock 12 | commands = pytest -q --cov=zoonado --cov-report= --flake8 {toxinidir} 13 | 14 | [testenv:pypy] 15 | usedevelop = True 16 | deps = 17 | pytest 18 | mock 19 | commands = pytest -q {toxinidir} 20 | -------------------------------------------------------------------------------- /zoonado/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | version_info = (0, 9, 2) 3 | 4 | __version__ = ".".join((str(point) for point in version_info)) 5 | 6 | try: 7 | import tornado # noqa 8 | from .client import Zoonado # noqa 9 | from .protocol import WatchEvent # noqa 10 | from .protocol.acl import ACL # noqa 11 | from .retry import RetryPolicy # noqa 12 | except ImportError: # pragma: no cover 13 | pass 14 | -------------------------------------------------------------------------------- /zoonado/client.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | import functools 4 | import logging 5 | 6 | from tornado import gen, concurrent 7 | 8 | from zoonado import protocol, exc 9 | 10 | from .recipes.proxy import RecipeProxy 11 | from .session import Session 12 | from .transaction import Transaction 13 | from .features import Features 14 | from .encoding import default_encoder, default_decoder 15 | 16 | 17 | log = logging.getLogger(__name__) 18 | 19 | 20 | def znode_method(fn): 21 | 22 | @functools.wraps(fn) 23 | def wrapper(client, path, *args, **kwargs): 24 | path = client.normalize_path(path) 25 | 26 | return fn(client, path, *args, **kwargs) 27 | 28 | return wrapper 29 | 30 | 31 | class Zoonado(object): 32 | 33 | def __init__( 34 | self, 35 | servers, 36 | chroot=None, 37 | data_encoder=None, 38 | data_decoder=None, 39 | session_timeout=10, 40 | default_acl=None, 41 | retry_policy=None, 42 | allow_read_only=False, 43 | ): 44 | self.chroot = None 45 | if chroot: 46 | self.chroot = self.normalize_path(chroot) 47 | log.info("Using chroot '%s'", self.chroot) 48 | 49 | self.data_encoder = data_encoder or default_encoder 50 | self.data_decoder = data_decoder or default_decoder 51 | 52 | self.session = Session( 53 | servers, session_timeout, retry_policy, allow_read_only 54 | ) 55 | 56 | self.default_acl = default_acl or [protocol.UNRESTRICTED_ACCESS] 57 | 58 | self.stat_cache = {} 59 | 60 | self.recipes = RecipeProxy(self) 61 | 62 | def normalize_path(self, path): 63 | if self.chroot: 64 | path = "/".join([self.chroot, path]) 65 | 66 | normalized = "/".join([ 67 | name for name in path.split("/") 68 | if name 69 | ]) 70 | 71 | return "/" + normalized 72 | 73 | def denormalize_path(self, path): 74 | if self.chroot and path.startswith(self.chroot): 75 | path = path[len(self.chroot):] 76 | 77 | return path 78 | 79 | @gen.coroutine 80 | def start(self): 81 | yield self.session.start() 82 | 83 | if self.chroot: 84 | yield self.ensure_path("/") 85 | 86 | @property 87 | def features(self): 88 | if self.session.conn: 89 | return Features(self.session.conn.version_info) 90 | else: 91 | return Features((0, 0, 0)) 92 | 93 | @gen.coroutine 94 | def send(self, request): 95 | response = yield self.session.send(request) 96 | 97 | if getattr(request, "path", None) and getattr(response, "stat", None): 98 | self.stat_cache[ 99 | self.denormalize_path(request.path) 100 | ] = response.stat 101 | 102 | raise gen.Return(response) 103 | 104 | @gen.coroutine 105 | def close(self): 106 | yield self.session.close() 107 | 108 | def wait_for_event(self, event_type, path): 109 | path = self.normalize_path(path) 110 | 111 | f = concurrent.Future() 112 | 113 | def set_future(_): 114 | if not f.done(): 115 | f.set_result(None) 116 | self.session.remove_watch_callback(event_type, path, set_future) 117 | 118 | self.session.add_watch_callback(event_type, path, set_future) 119 | 120 | return f 121 | 122 | @gen.coroutine 123 | @znode_method 124 | def exists(self, path, watch=False): 125 | try: 126 | yield self.send(protocol.ExistsRequest(path=path, watch=watch)) 127 | except exc.NoNode: 128 | raise gen.Return(False) 129 | 130 | raise gen.Return(True) 131 | 132 | @gen.coroutine 133 | @znode_method 134 | def create( 135 | self, path, data=None, acl=None, 136 | ephemeral=False, sequential=False, container=False, 137 | ): 138 | if container and not self.features.containers: 139 | raise ValueError("Cannot create container, feature unavailable.") 140 | 141 | acl = acl or self.default_acl 142 | 143 | data = self.data_encoder(data) 144 | 145 | if self.features.create_with_stat: 146 | request_class = protocol.Create2Request 147 | else: 148 | request_class = protocol.CreateRequest 149 | 150 | request = request_class(path=path, data=data, acl=acl) 151 | request.set_flags(ephemeral, sequential, container) 152 | 153 | response = yield self.send(request) 154 | 155 | raise gen.Return(self.denormalize_path(response.path)) 156 | 157 | @gen.coroutine 158 | @znode_method 159 | def ensure_path(self, path, acl=None): 160 | acl = acl or self.default_acl 161 | 162 | paths_to_make = [] 163 | for segment in path[1:].split("/"): 164 | if not paths_to_make: 165 | paths_to_make.append("/" + segment) 166 | continue 167 | 168 | paths_to_make.append("/".join([paths_to_make[-1], segment])) 169 | 170 | while paths_to_make: 171 | path = paths_to_make[0] 172 | 173 | if self.features.create_with_stat: 174 | request = protocol.Create2Request(path=path, acl=acl) 175 | else: 176 | request = protocol.CreateRequest(path=path, acl=acl) 177 | request.set_flags( 178 | ephemeral=False, sequential=False, 179 | container=self.features.containers 180 | ) 181 | 182 | try: 183 | yield self.send(request) 184 | except exc.NodeExists: 185 | pass 186 | 187 | paths_to_make.pop(0) 188 | 189 | @gen.coroutine 190 | @znode_method 191 | def delete(self, path, force=False): 192 | version = self.determine_znode_version(path, force) 193 | 194 | yield self.send(protocol.DeleteRequest(path=path, version=version)) 195 | 196 | @gen.coroutine 197 | @znode_method 198 | def get_data(self, path, watch=False): 199 | response = yield self.send( 200 | protocol.GetDataRequest(path=path, watch=watch) 201 | ) 202 | data = self.data_decoder(response.data) 203 | 204 | raise gen.Return(data) 205 | 206 | @gen.coroutine 207 | @znode_method 208 | def set_data(self, path, data, force=False): 209 | data = self.data_encoder(data) 210 | version = self.determine_znode_version(path, force) 211 | 212 | yield self.send( 213 | protocol.SetDataRequest(path=path, data=data, version=version) 214 | ) 215 | 216 | @gen.coroutine 217 | @znode_method 218 | def get_children(self, path, watch=False): 219 | response = yield self.send( 220 | protocol.GetChildren2Request(path=path, watch=watch) 221 | ) 222 | raise gen.Return(response.children) 223 | 224 | @gen.coroutine 225 | @znode_method 226 | def get_acl(self, path): 227 | response = yield self.send(protocol.GetACLRequest(path=path)) 228 | raise gen.Return(response.acl) 229 | 230 | @gen.coroutine 231 | @znode_method 232 | def set_acl(self, path, acl, force=False): 233 | version = self.determine_znode_version(path, force) 234 | 235 | yield self.send( 236 | protocol.SetACLRequest(path=path, acl=acl, version=version) 237 | ) 238 | 239 | def begin_transaction(self): 240 | return Transaction(self) 241 | 242 | def determine_znode_version(self, path, force): 243 | if not force and path in self.stat_cache: 244 | return self.stat_cache[path].version 245 | else: 246 | return -1 247 | -------------------------------------------------------------------------------- /zoonado/compat.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | 4 | def add_metaclass(metaclass): 5 | """ 6 | Class decorator for creating a class with a metaclass. 7 | 8 | Adapted from the six project: 9 | 10 | https://pythonhosted.org/six/ 11 | """ 12 | vars_to_skip = ('__dict__', '__weakref__') 13 | 14 | def wrapper(cls): 15 | copied_dict = { 16 | key: value 17 | for key, value in cls.__dict__.items() 18 | if key not in vars_to_skip 19 | } 20 | return metaclass(cls.__name__, cls.__bases__, copied_dict) 21 | return wrapper 22 | -------------------------------------------------------------------------------- /zoonado/connection.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | import collections 4 | import logging 5 | import re 6 | import struct 7 | import sys 8 | 9 | from tornado import ioloop, iostream, gen, concurrent, tcpclient 10 | 11 | from zoonado import protocol, iterables, exc 12 | 13 | 14 | version_regex = re.compile(r'Zookeeper version: (\d)\.(\d)\.(\d)-.*') 15 | 16 | # all requests and responses are prefixed with a 32-bit int denoting size 17 | size_struct = struct.Struct("!i") 18 | # replies are prefixed with an xid, zxid and error code 19 | reply_header_struct = struct.Struct("!iqi") 20 | 21 | log = logging.getLogger(__name__) 22 | payload_log = logging.getLogger(__name__ + ".payload") 23 | if payload_log.level == logging.NOTSET: 24 | payload_log.setLevel(logging.INFO) 25 | 26 | 27 | class Connection(object): 28 | 29 | def __init__(self, host, port, watch_handler): 30 | self.host = host 31 | self.port = int(port) 32 | 33 | self.stream = None 34 | self.closing = False 35 | 36 | self.version_info = None 37 | self.start_read_only = None 38 | 39 | self.watch_handler = watch_handler 40 | 41 | self.opcode_xref = {} 42 | 43 | self.pending = {} 44 | self.pending_specials = collections.defaultdict(list) 45 | 46 | self.watches = collections.defaultdict(list) 47 | 48 | @gen.coroutine 49 | def connect(self): 50 | client = tcpclient.TCPClient() 51 | 52 | log.debug("Initial connection to server %s:%d", self.host, self.port) 53 | stream = yield client.connect(self.host, self.port) 54 | 55 | log.debug("Sending 'srvr' command to %s:%d", self.host, self.port) 56 | yield stream.write(b"srvr") 57 | answer = yield stream.read_until_close() 58 | 59 | answer = answer.decode("utf8") 60 | 61 | version_line = answer.split("\n")[0] 62 | self.version_info = tuple( 63 | map(int, version_regex.match(version_line).groups()) 64 | ) 65 | self.start_read_only = bool("READ_ONLY" in answer) 66 | 67 | log.debug("Version info: %s", self.version_info) 68 | log.debug("Read-only mode: %s", self.start_read_only) 69 | 70 | log.debug("Actual connection to server %s:%d", self.host, self.port) 71 | self.stream = yield client.connect(self.host, self.port) 72 | 73 | @gen.coroutine 74 | def send_connect(self, request): 75 | # meant to be used before the read_loop starts 76 | payload_log.debug("[SEND] (initial) %s", request) 77 | 78 | payload = request.serialize() 79 | payload = size_struct.pack(len(payload)) + payload 80 | 81 | yield self.stream.write(payload) 82 | 83 | try: 84 | _, zxid, response = yield self.read_response(initial_connect=True) 85 | except Exception: 86 | log.exception("Error reading connect response.") 87 | return 88 | 89 | payload_log.debug("[RECV] (initial) %s", response) 90 | 91 | raise gen.Return((zxid, response)) 92 | 93 | def start_read_loop(self): 94 | ioloop.IOLoop.current().add_callback(self.read_loop) 95 | 96 | def send(self, request, xid=None): 97 | f = concurrent.Future() 98 | 99 | if self.closing: 100 | f.set_exception(exc.ConnectError(self.host, self.port)) 101 | return f 102 | 103 | if request.special_xid: 104 | xid = request.special_xid 105 | 106 | payload_log.debug("[SEND] (xid: %s) %s", xid, request) 107 | 108 | payload = request.serialize(xid) 109 | payload = size_struct.pack(len(payload)) + payload 110 | 111 | self.opcode_xref[xid] = request.opcode 112 | 113 | if xid in protocol.SPECIAL_XIDS: 114 | self.pending_specials[xid].append(f) 115 | else: 116 | self.pending[xid] = f 117 | 118 | def handle_write(write_future): 119 | try: 120 | write_future.result() 121 | except Exception: 122 | self.abort() 123 | 124 | try: 125 | self.stream.write(payload).add_done_callback(handle_write) 126 | except Exception: 127 | self.abort() 128 | 129 | return f 130 | 131 | @gen.coroutine 132 | def read_loop(self): 133 | """ 134 | Infinite loop that reads messages off of the socket while not closed. 135 | 136 | When a message is received its corresponding pending Future is set 137 | to have the message as its result. 138 | 139 | This is never used directly and is fired as a separate callback on the 140 | I/O loop via the `connect()` method. 141 | """ 142 | while not self.closing: 143 | try: 144 | xid, zxid, response = yield self.read_response() 145 | except iostream.StreamClosedError: 146 | return 147 | except Exception: 148 | log.exception("Error reading response.") 149 | self.abort() 150 | return 151 | 152 | payload_log.debug("[RECV] (xid: %s) %s", xid, response) 153 | 154 | if xid == protocol.WATCH_XID: 155 | self.watch_handler(response) 156 | continue 157 | elif xid in protocol.SPECIAL_XIDS: 158 | f = self.pending_specials[xid].pop() 159 | else: 160 | f = self.pending.pop(xid) 161 | 162 | if isinstance(response, Exception): 163 | f.set_exception(response) 164 | else: 165 | f.set_result((zxid, response)) 166 | 167 | @gen.coroutine 168 | def read_response(self, initial_connect=False): 169 | raw_size = yield self.stream.read_bytes(size_struct.size) 170 | size = size_struct.unpack(raw_size)[0] 171 | 172 | # connect and close op replies don't contain a reply header 173 | if initial_connect or self.pending_specials[protocol.CLOSE_XID]: 174 | raw_payload = yield self.stream.read_bytes(size) 175 | response = protocol.ConnectResponse.deserialize(raw_payload) 176 | raise gen.Return((None, None, response)) 177 | 178 | raw_header = yield self.stream.read_bytes(reply_header_struct.size) 179 | xid, zxid, error_code = reply_header_struct.unpack_from(raw_header) 180 | 181 | if error_code: 182 | raise gen.Return((xid, zxid, exc.get_response_error(error_code))) 183 | 184 | size -= reply_header_struct.size 185 | 186 | raw_payload = yield self.stream.read_bytes(size) 187 | 188 | if xid == protocol.WATCH_XID: 189 | response = protocol.WatchEvent.deserialize(raw_payload) 190 | else: 191 | opcode = self.opcode_xref.pop(xid) 192 | response = protocol.response_xref[opcode].deserialize(raw_payload) 193 | 194 | raise gen.Return((xid, zxid, response)) 195 | 196 | def abort(self, exception=exc.ConnectError): 197 | """ 198 | Aborts a connection and puts all pending futures into an error state. 199 | 200 | If ``sys.exc_info()`` is set (i.e. this is being called in an exception 201 | handler) then pending futures will have that exc info set. Otherwise 202 | the given ``exception`` parameter is used (defaults to 203 | ``ConnectError``). 204 | """ 205 | log.warn("Aborting connection to %s:%s", self.host, self.port) 206 | 207 | def abort_pending(f): 208 | exc_info = sys.exc_info() 209 | if any(exc_info): 210 | f.set_exc_info(exc_info) 211 | else: 212 | f.set_exception(exception(self.host, self.port)) 213 | 214 | for pending in self.drain_all_pending(): 215 | abort_pending(pending) 216 | 217 | def drain_all_pending(self): 218 | for special_xid in protocol.SPECIAL_XIDS: 219 | for f in iterables.drain(self.pending_specials[special_xid]): 220 | yield f 221 | for _, f in iterables.drain(self.pending): 222 | yield f 223 | 224 | @gen.coroutine 225 | def close(self, timeout): 226 | if self.closing: 227 | return 228 | 229 | self.closing = True 230 | 231 | pending_with_timeouts = [] 232 | for pending in self.drain_all_pending(): 233 | pending_with_timeouts.append(gen.with_timeout(timeout, pending)) 234 | 235 | try: 236 | yield list(pending_with_timeouts) 237 | except gen.TimeoutError: 238 | yield self.abort(exception=exc.TimeoutError) 239 | finally: 240 | self.stream.close() 241 | -------------------------------------------------------------------------------- /zoonado/encoding.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import six 4 | 5 | 6 | log = logging.getLogger(__name__) 7 | 8 | 9 | def default_encoder(data): 10 | if data is None or isinstance(data, six.binary_type): 11 | return data 12 | 13 | try: 14 | return data.encode("utf8") 15 | except Exception: 16 | log.exception("Error encoding data: %r", data) 17 | raise 18 | 19 | 20 | def default_decoder(data): 21 | if data is None: 22 | return data 23 | 24 | try: 25 | return data.decode("utf8") 26 | except UnicodeDecodeError: 27 | return data 28 | -------------------------------------------------------------------------------- /zoonado/exc.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from .compat import add_metaclass 4 | 5 | 6 | class ZKError(Exception): 7 | pass 8 | 9 | 10 | class ConnectError(ZKError): 11 | def __init__(self, host, port, server_id=None): 12 | self.host = host 13 | self.port = port 14 | self.server_id = server_id 15 | 16 | def __str__(self): 17 | return "Error connecting to %s:%s" % (self.host, self.port) 18 | 19 | 20 | class NoServersError(ZKError): 21 | pass 22 | 23 | 24 | class SessionLost(ZKError): 25 | pass 26 | 27 | 28 | class InvalidClientState(ZKError): 29 | pass 30 | 31 | 32 | class TimeoutError(ZKError): 33 | pass 34 | 35 | 36 | class FailedRetry(ZKError): 37 | pass 38 | 39 | 40 | response_error_xref = {} 41 | 42 | 43 | class ResponseErrorMeta(type): 44 | 45 | def __new__(cls, name, bases, attrs): 46 | new_class = super(ResponseErrorMeta, cls).__new__( 47 | cls, name, bases, attrs 48 | ) 49 | 50 | response_error_xref[new_class.error_code] = new_class 51 | 52 | return new_class 53 | 54 | 55 | @add_metaclass(ResponseErrorMeta) 56 | class ResponseError(ZKError): 57 | error_code = None 58 | 59 | def __str__(self): 60 | return self.__class__.__name__ 61 | 62 | 63 | class UnknownError(ResponseError): 64 | 65 | def __init__(self, error_code): 66 | self.error_code = error_code 67 | 68 | def __str__(self): 69 | return "Unknown error code: %s" % self.error_code 70 | 71 | 72 | def get_response_error(error_code): 73 | if error_code not in response_error_xref: 74 | return UnknownError(error_code) 75 | 76 | return response_error_xref[error_code]() 77 | 78 | 79 | class RolledBack(ResponseError): 80 | error_code = 0 81 | 82 | 83 | class ZKSystemError(ResponseError): 84 | error_code = -1 85 | 86 | 87 | class RuntimeInconsistency(ResponseError): 88 | error_code = -2 89 | 90 | 91 | class DataInconsistency(ResponseError): 92 | error_code = -3 93 | 94 | 95 | class ConnectionLoss(ResponseError): 96 | error_code = -4 97 | 98 | 99 | class MarshallingError(ResponseError): 100 | error_code = -5 101 | 102 | 103 | class Unimplemented(ResponseError): 104 | error_code = -6 105 | 106 | 107 | class OperationTimeout(ResponseError): 108 | error_code = -7 109 | 110 | 111 | class BadArguments(ResponseError): 112 | error_code = -8 113 | 114 | 115 | class UnknownSession(ResponseError): 116 | error_code = -12 117 | 118 | 119 | class NewConfigNoQuorum(ResponseError): 120 | error_code = -13 121 | 122 | 123 | class ReconfigInProcess(ResponseError): 124 | error_code = -14 125 | 126 | 127 | class APIError(ResponseError): 128 | error_code = -100 129 | 130 | 131 | class NoNode(ResponseError): 132 | error_code = -101 133 | 134 | 135 | class NoAuth(ResponseError): 136 | error_code = -102 137 | 138 | 139 | class BadVersion(ResponseError): 140 | error_code = -103 141 | 142 | 143 | class NoChildrenForEphemerals(ResponseError): 144 | error_code = -108 145 | 146 | 147 | class NodeExists(ResponseError): 148 | error_code = -110 149 | 150 | 151 | class NotEmpty(ResponseError): 152 | error_code = -111 153 | 154 | 155 | class SessionExpired(ResponseError): 156 | error_code = -112 157 | 158 | 159 | class InvalidCallback(ResponseError): 160 | error_code = -113 161 | 162 | 163 | class InvalidACL(ResponseError): 164 | error_code = -114 165 | 166 | 167 | class AuthFailed(ResponseError): 168 | error_code = -115 169 | 170 | 171 | class SessionMoved(ResponseError): 172 | error_code = -118 173 | 174 | 175 | class NotReadOnly(ResponseError): 176 | error_code = -119 177 | 178 | 179 | class EphemeralOnLocalSession(ResponseError): 180 | error_code = -120 181 | 182 | 183 | class NoWatcher(ResponseError): 184 | error_code = -121 185 | -------------------------------------------------------------------------------- /zoonado/features.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | 4 | ALL_FEATURES = { 5 | "create_with_stat": (3, 5, 0), 6 | "containers": (3, 5, 1), 7 | "reconfigure": (3, 5, 0), 8 | } 9 | 10 | 11 | class Features(object): 12 | 13 | def __init__(self, version_info): 14 | for feature_name, version_introduced in ALL_FEATURES.items(): 15 | setattr(self, feature_name, version_info >= version_introduced) 16 | -------------------------------------------------------------------------------- /zoonado/iterables.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | 4 | def drain(iterable): 5 | """ 6 | Helper method that empties an iterable as it is iterated over. 7 | 8 | Works for: 9 | 10 | * ``dict`` 11 | * ``collections.deque`` 12 | * ``list`` 13 | * ``set`` 14 | """ 15 | if getattr(iterable, "popleft", False): 16 | def next_item(coll): 17 | return coll.popleft() 18 | elif getattr(iterable, "popitem", False): 19 | def next_item(coll): 20 | return coll.popitem() 21 | else: 22 | def next_item(coll): 23 | return coll.pop() 24 | 25 | while True: 26 | try: 27 | yield next_item(iterable) 28 | except (IndexError, KeyError): 29 | raise StopIteration 30 | -------------------------------------------------------------------------------- /zoonado/protocol/__init__.py: -------------------------------------------------------------------------------- 1 | from .acl import ( # noqa 2 | GetACLRequest, 3 | GetACLResponse, 4 | SetACLRequest, 5 | SetACLResponse, 6 | WORLD_READABLE, 7 | AUTHED_UNRESTRICTED, 8 | UNRESTRICTED_ACCESS, 9 | ) 10 | from .auth import ( # noqa 11 | AuthRequest, 12 | AuthResponse, 13 | AUTH_XID, 14 | ) 15 | from .check import ( # noqa 16 | CheckVersionRequest, 17 | CheckVersionResponse, 18 | ) 19 | from .children import ( # noqa 20 | GetChildrenRequest, 21 | GetChildrenResponse, 22 | GetChildren2Request, 23 | GetChildren2Response, 24 | ) 25 | from .close import ( # noqa 26 | CloseRequest, 27 | CloseResponse, 28 | CLOSE_XID, 29 | ) 30 | from .connect import ( # noqa 31 | ConnectRequest, 32 | ConnectResponse, 33 | ) 34 | from .create import ( # noqa 35 | CreateRequest, 36 | CreateResponse, 37 | Create2Request, 38 | Create2Response, 39 | ) 40 | from .data import ( # noqa 41 | GetDataRequest, 42 | GetDataResponse, 43 | SetDataRequest, 44 | SetDataResponse, 45 | ) 46 | from .delete import ( # noqa 47 | DeleteRequest, 48 | DeleteResponse, 49 | ) 50 | from .exists import ( # noqa 51 | ExistsRequest, 52 | ExistsResponse, 53 | ) 54 | from .ping import ( # noqa 55 | PingRequest, 56 | PingResponse, 57 | PING_XID, 58 | ) 59 | from .reconfig import ( # noqa 60 | ReconfigRequest, 61 | ReconfigResponse, 62 | ) 63 | from .sasl import ( # noqa 64 | SASLRequest, 65 | SASLResponse, 66 | ) 67 | from .sync import ( # noqa 68 | SyncRequest, 69 | SyncResponse, 70 | ) 71 | from .watches import ( # noqa 72 | WatchEvent, 73 | SetWatchesRequest, 74 | SetWatchesResponse, 75 | CheckWatchesRequest, 76 | CheckWatchesResponse, 77 | RemoveWatchesRequest, 78 | RemoveWatchesResponse, 79 | WATCH_XID, 80 | ) 81 | from .response import ( # noqa 82 | response_xref 83 | ) 84 | from .transaction import ( # noqa 85 | TransactionRequest, 86 | TransactionResponse, 87 | ) 88 | 89 | SPECIAL_XIDS = (AUTH_XID, PING_XID, CLOSE_XID) 90 | -------------------------------------------------------------------------------- /zoonado/protocol/acl.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from .request import Request 4 | from .response import Response 5 | from .part import Part 6 | from .stat import Stat 7 | from .primitives import UString, Int, Vector 8 | 9 | 10 | class ID(Part): 11 | """ 12 | """ 13 | parts = ( 14 | ("scheme", UString), 15 | ("id", UString), 16 | ) 17 | 18 | 19 | class ACL(Part): 20 | """ 21 | """ 22 | READ_PERM = 1 << 0 23 | WRITE_PERM = 1 << 1 24 | CREATE_PERM = 1 << 2 25 | DELETE_PERM = 1 << 3 26 | ADMIN_PERM = 1 << 4 27 | 28 | parts = ( 29 | ("perms", Int), 30 | ("id", ID), 31 | ) 32 | 33 | @classmethod 34 | def make( 35 | cls, scheme, id, 36 | read=False, write=False, create=False, delete=False, admin=False 37 | ): 38 | instance = cls(id=ID(scheme=scheme, id=id)) 39 | instance.set_perms(read, write, create, delete, admin) 40 | 41 | return instance 42 | 43 | def set_perms(self, read, write, create, delete, admin): 44 | perms = 0 45 | if read: 46 | perms |= self.READ_PERM 47 | if write: 48 | perms |= self.WRITE_PERM 49 | if create: 50 | perms |= self.CREATE_PERM 51 | if delete: 52 | perms |= self.DELETE_PERM 53 | if admin: 54 | perms |= self.ADMIN_PERM 55 | 56 | self.perms = perms 57 | 58 | 59 | WORLD_READABLE = ACL.make( 60 | scheme="world", id="anyone", 61 | read=True, write=False, create=False, delete=False, admin=False 62 | ) 63 | 64 | AUTHED_UNRESTRICTED = ACL.make( 65 | scheme="auth", id="", 66 | read=True, write=True, create=True, delete=True, admin=True 67 | ) 68 | 69 | UNRESTRICTED_ACCESS = ACL.make( 70 | scheme="world", id="anyone", 71 | read=True, write=True, create=True, delete=True, admin=True 72 | ) 73 | 74 | 75 | class GetACLRequest(Request): 76 | """ 77 | """ 78 | opcode = 6 79 | 80 | parts = ( 81 | ("path", UString), 82 | ) 83 | 84 | 85 | class GetACLResponse(Response): 86 | """ 87 | """ 88 | opcode = 6 89 | 90 | parts = ( 91 | ("acl", Vector.of(ACL)), 92 | ("stat", Stat), 93 | ) 94 | 95 | 96 | class SetACLRequest(Request): 97 | """ 98 | """ 99 | opcode = 7 100 | 101 | parts = ( 102 | ("path", UString), 103 | ("acl", Vector.of(ACL)), 104 | ("version", Int), 105 | ) 106 | 107 | 108 | class SetACLResponse(Response): 109 | """ 110 | """ 111 | opcode = 7 112 | 113 | parts = ( 114 | ("stat", Stat), 115 | ) 116 | -------------------------------------------------------------------------------- /zoonado/protocol/auth.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from .request import Request 4 | from .response import Response 5 | from .primitives import Int, Buffer, UString 6 | 7 | 8 | AUTH_XID = -4 9 | 10 | 11 | class AuthRequest(Request): 12 | """ 13 | """ 14 | opcode = 100 15 | special_xid = AUTH_XID 16 | 17 | parts = ( 18 | ("type", Int), 19 | ("scheme", UString), 20 | ("auth", Buffer), 21 | ) 22 | 23 | 24 | class AuthResponse(Response): 25 | """ 26 | """ 27 | opcode = 100 28 | 29 | parts = () 30 | -------------------------------------------------------------------------------- /zoonado/protocol/check.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from .request import Request 4 | from .response import Response 5 | from .primitives import Int, UString 6 | 7 | 8 | class CheckVersionRequest(Request): 9 | """ 10 | """ 11 | opcode = 13 12 | 13 | parts = ( 14 | ("path", UString), 15 | ("version", Int), 16 | ) 17 | 18 | 19 | class CheckVersionResponse(Response): 20 | """ 21 | """ 22 | opcode = 13 23 | 24 | parts = () 25 | -------------------------------------------------------------------------------- /zoonado/protocol/children.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from .request import Request 4 | from .response import Response 5 | from .stat import Stat 6 | from .primitives import Bool, UString, Vector 7 | 8 | 9 | class GetChildrenRequest(Request): 10 | """ 11 | """ 12 | opcode = 8 13 | 14 | parts = ( 15 | ("path", UString), 16 | ("watch", Bool), 17 | ) 18 | 19 | 20 | class GetChildrenResponse(Response): 21 | """ 22 | """ 23 | opcode = 8 24 | 25 | parts = ( 26 | ("children", Vector.of(UString)), 27 | ) 28 | 29 | 30 | class GetChildren2Request(Request): 31 | """ 32 | """ 33 | opcode = 12 34 | 35 | parts = ( 36 | ("path", UString), 37 | ("watch", Bool), 38 | ) 39 | 40 | 41 | class GetChildren2Response(Response): 42 | """ 43 | """ 44 | opcode = 12 45 | 46 | parts = ( 47 | ("children", Vector.of(UString)), 48 | ("stat", Stat), 49 | ) 50 | -------------------------------------------------------------------------------- /zoonado/protocol/close.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from .request import Request 4 | from .response import Response 5 | 6 | 7 | CLOSE_XID = None 8 | 9 | 10 | class CloseRequest(Request): 11 | """ 12 | """ 13 | opcode = -11 14 | special_xid = CLOSE_XID 15 | 16 | parts = () 17 | 18 | 19 | class CloseResponse(Response): 20 | """ 21 | """ 22 | opcode = -11 23 | 24 | parts = () 25 | -------------------------------------------------------------------------------- /zoonado/protocol/connect.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from .request import Request 4 | from .response import Response 5 | from .primitives import Int, Long, Buffer, Bool 6 | 7 | 8 | class ConnectRequest(Request): 9 | """ 10 | """ 11 | parts = ( 12 | ("protocol_version", Int), 13 | ("last_seen_zxid", Long), 14 | ("timeout", Int), 15 | ("session_id", Long), 16 | ("password", Buffer), 17 | ("read_only", Bool), 18 | ) 19 | 20 | 21 | class ConnectResponse(Response): 22 | """ 23 | """ 24 | parts = ( 25 | ("protocol_version", Int), 26 | ("timeout", Int), 27 | ("session_id", Long), 28 | ("password", Buffer), 29 | ) 30 | -------------------------------------------------------------------------------- /zoonado/protocol/create.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from .request import Request 4 | from .response import Response 5 | from .stat import Stat 6 | from .acl import ACL 7 | from .primitives import UString, Int, Buffer, Vector 8 | 9 | 10 | class CreateRequest(Request): 11 | """ 12 | """ 13 | opcode = 1 14 | 15 | writes_data = True 16 | 17 | EPHEMERAL_FLAG = 1 << 0 18 | SEQUENTIAL_FLAG = 1 << 1 19 | CONTAINER_FLAG = 1 << 2 20 | 21 | parts = ( 22 | ("path", UString), 23 | ("data", Buffer), 24 | ("acl", Vector.of(ACL)), 25 | ("flags", Int), 26 | ) 27 | 28 | def set_flags(self, ephemeral=False, sequential=False, container=False): 29 | flags = 0 30 | if ephemeral: 31 | flags |= self.EPHEMERAL_FLAG 32 | if sequential: 33 | flags |= self.SEQUENTIAL_FLAG 34 | if container: 35 | flags |= self.CONTAINER_FLAG 36 | 37 | self.flags = flags 38 | 39 | 40 | class CreateResponse(Response): 41 | """ 42 | """ 43 | opcode = 1 44 | 45 | parts = ( 46 | ("path", UString), 47 | ) 48 | 49 | 50 | class Create2Request(CreateRequest): 51 | """ 52 | """ 53 | opcode = 15 54 | 55 | 56 | class Create2Response(Response): 57 | """ 58 | """ 59 | opcode = 15 60 | 61 | parts = ( 62 | ("path", UString), 63 | ("stat", Stat), 64 | ) 65 | -------------------------------------------------------------------------------- /zoonado/protocol/data.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from .request import Request 4 | from .response import Response 5 | from .stat import Stat 6 | from .primitives import Bool, UString, Buffer, Int 7 | 8 | 9 | class GetDataRequest(Request): 10 | """ 11 | """ 12 | opcode = 4 13 | 14 | parts = ( 15 | ("path", UString), 16 | ("watch", Bool), 17 | ) 18 | 19 | 20 | class GetDataResponse(Response): 21 | """ 22 | """ 23 | opcode = 4 24 | 25 | parts = ( 26 | ("data", Buffer), 27 | ("stat", Stat) 28 | ) 29 | 30 | 31 | class SetDataRequest(Request): 32 | """ 33 | """ 34 | opcode = 5 35 | 36 | writes_data = True 37 | 38 | parts = ( 39 | ("path", UString), 40 | ("data", Buffer), 41 | ("version", Int), 42 | ) 43 | 44 | 45 | class SetDataResponse(Response): 46 | """ 47 | """ 48 | opcode = 5 49 | 50 | parts = ( 51 | ("stat", Stat), 52 | ) 53 | -------------------------------------------------------------------------------- /zoonado/protocol/delete.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from .request import Request 4 | from .response import Response 5 | from .primitives import UString, Int 6 | 7 | 8 | class DeleteRequest(Request): 9 | """ 10 | """ 11 | opcode = 2 12 | 13 | writes_data = True 14 | 15 | parts = ( 16 | ("path", UString), 17 | ("version", Int), 18 | ) 19 | 20 | 21 | class DeleteResponse(Response): 22 | """ 23 | """ 24 | opcode = 2 25 | 26 | parts = () 27 | -------------------------------------------------------------------------------- /zoonado/protocol/exists.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from .request import Request 4 | from .response import Response 5 | from .stat import Stat 6 | from .primitives import UString, Bool 7 | 8 | 9 | class ExistsRequest(Request): 10 | """ 11 | """ 12 | opcode = 3 13 | 14 | parts = ( 15 | ("path", UString), 16 | ("watch", Bool), 17 | ) 18 | 19 | 20 | class ExistsResponse(Response): 21 | """ 22 | """ 23 | opcode = 3 24 | 25 | parts = ( 26 | ("stat", Stat), 27 | ) 28 | -------------------------------------------------------------------------------- /zoonado/protocol/part.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | import operator 4 | 5 | from .primitives import Primitive 6 | 7 | 8 | class Part(object): 9 | """ 10 | Composable building block used to define Zookeeper protocol parts. 11 | 12 | Behaves much like the `Primitive` class but has named "sub parts" 13 | stored in a ``parts`` class attribute, that can hold any `Part` or 14 | `Primitive` subclass. 15 | """ 16 | parts = () 17 | 18 | def __init__(self, **kwargs): 19 | part_names = set([item[0] for item in self.parts]) 20 | 21 | for name, value in kwargs.items(): 22 | if name not in part_names: 23 | raise ValueError("Unknown part name: '%s'" % name) 24 | part_names.discard(name) 25 | 26 | setattr(self, name, value) 27 | 28 | for name in part_names: 29 | setattr(self, name, None) 30 | 31 | def render(self, parts=None): 32 | """ 33 | Returns a two-element tuple with the ``struct`` format and values. 34 | 35 | Iterates over the applicable sub-parts and calls `render()` on them, 36 | accumulating the format string and values. 37 | 38 | Optionally takes a subset of parts to render, default behavior is to 39 | render all sub-parts belonging to the class. 40 | """ 41 | if not parts: 42 | parts = self.parts 43 | 44 | fmt = [] 45 | data = [] 46 | 47 | for name, part_class in parts: 48 | if issubclass(part_class, Primitive): 49 | part = part_class(getattr(self, name, None)) 50 | else: 51 | part = getattr(self, name, None) 52 | 53 | part_format, part_data = part.render() 54 | 55 | fmt.extend(part_format) 56 | data.extend(part_data) 57 | 58 | return "".join(fmt), data 59 | 60 | @classmethod 61 | def parse(cls, buff, offset): 62 | """ 63 | Given a buffer and offset, returns the parsed value and new offset. 64 | 65 | Calls `parse()` on the given buffer for each sub-part in order and 66 | creates a new instance with the results. 67 | """ 68 | values = {} 69 | 70 | for name, part in cls.parts: 71 | value, new_offset = part.parse(buff, offset) 72 | 73 | values[name] = value 74 | offset = new_offset 75 | 76 | return cls(**values), offset 77 | 78 | def __eq__(self, other): 79 | """ 80 | `Part` instances are equal if all of their sub-parts are also equal. 81 | """ 82 | return all([ 83 | getattr(self, part_name) == getattr(other, part_name) 84 | for part_name, part_class in self.parts 85 | ]) 86 | 87 | def __ne__(self, other): 88 | return not self.__eq__(other) 89 | 90 | def __str__(self): 91 | 92 | def subpart_string(part_info): 93 | part_name, part_class = part_info 94 | 95 | if not part_class.__name__.startswith("VectorOf"): 96 | value = getattr(self, part_name, None) 97 | 98 | if value and part_class.__name__ == "Buffer": 99 | value = value.decode("utf-8", errors="replace") 100 | 101 | return "%s=%s" % (part_name, value) 102 | 103 | return "%s=[%s]" % ( 104 | part_name, 105 | ", ".join([ 106 | str(item) for item in getattr(self, part_name, []) 107 | ]) 108 | ) 109 | 110 | return "%s(%s)" % ( 111 | self.__class__.__name__, 112 | ", ".join([ 113 | subpart_string(part) 114 | for part in sorted(self.parts, key=operator.itemgetter(0)) 115 | ]) 116 | ) 117 | -------------------------------------------------------------------------------- /zoonado/protocol/ping.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from .request import Request 4 | from .response import Response 5 | 6 | 7 | PING_XID = -2 8 | 9 | 10 | class PingRequest(Request): 11 | """ 12 | """ 13 | opcode = 11 14 | special_xid = PING_XID 15 | 16 | parts = () 17 | 18 | 19 | class PingResponse(Response): 20 | """ 21 | """ 22 | opcode = 11 23 | 24 | parts = () 25 | -------------------------------------------------------------------------------- /zoonado/protocol/primitives.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | import struct 4 | 5 | 6 | class Primitive(object): 7 | """ 8 | The most basic structure of the protocol. Subclassed, never used directly. 9 | 10 | Used as a building block for the various actually-used primitives outlined 11 | in the Zookeeper jute file: 12 | 13 | https://github.com/apache/zookeeper/blob/trunk/src/zookeeper.jute 14 | """ 15 | fmt = None 16 | 17 | def __init__(self, value): 18 | self.value = value 19 | 20 | def render(self): 21 | """ 22 | Returns a two-element tuple with the ``struct`` format and list value. 23 | 24 | The value is wrapped in a list, as there are some primitives that deal 25 | with multiple values. Any caller of `render()` should expect a list. 26 | """ 27 | return self.fmt, [self.value] 28 | 29 | @classmethod 30 | def parse(cls, buff, offset): 31 | """ 32 | Given a buffer and offset, returns the parsed value and new offset. 33 | 34 | Uses the ``format`` class attribute to unpack the data from the buffer 35 | and determine the used up number of bytes. 36 | """ 37 | primitive_struct = struct.Struct("!" + cls.fmt) 38 | 39 | value = primitive_struct.unpack_from(buff, offset)[0] 40 | offset += primitive_struct.size 41 | 42 | return value, offset 43 | 44 | def __eq__(self, other): 45 | """ 46 | Basic equality method that tests equality of the ``value`` attributes. 47 | """ 48 | return self.value == other.value 49 | 50 | def __str__(self): 51 | return "%s(%s)" % (self.__class__.__name__, self.value) 52 | 53 | 54 | class VariablePrimitive(Primitive): 55 | """ 56 | Base primitive for variable-length scalar primitives (strings and bytes). 57 | """ 58 | size_primitive = None 59 | 60 | def render_value(self, value): 61 | raise NotImplementedError 62 | 63 | @classmethod 64 | def parse_value(cls, value): 65 | raise NotImplementedError 66 | 67 | def render(self): 68 | """ 69 | Returns the ``struct`` format and list of the size and value. 70 | 71 | The format is derived from the size primitive and the length of the 72 | resulting encoded value (e.g. the format for a string of 'foo' ends 73 | up as 'h3s'. 74 | 75 | .. note :: 76 | The value is expected to be string-able (wrapped in ``str()``) and is 77 | then encoded as UTF-8. 78 | """ 79 | size_format = self.size_primitive.fmt 80 | 81 | if self.value is None: 82 | return size_format, [-1] 83 | 84 | value = self.render_value(self.value) 85 | 86 | size = len(value) 87 | 88 | fmt = "%s%ds" % (size_format, size) 89 | 90 | return fmt, [size, value] 91 | 92 | @classmethod 93 | def parse(cls, buff, offset): 94 | """ 95 | Given a buffer and offset, returns the parsed value and new offset. 96 | 97 | Parses the ``size_primitive`` first to determine how many more bytes to 98 | consume to extract the value. 99 | """ 100 | size, offset = cls.size_primitive.parse(buff, offset) 101 | if size == -1: 102 | return None, offset 103 | 104 | var_struct = struct.Struct("!%ds" % size) 105 | 106 | value = var_struct.unpack_from(buff, offset)[0] 107 | value = cls.parse_value(value) 108 | offset += var_struct.size 109 | 110 | return value, offset 111 | 112 | 113 | class Bool(Primitive): 114 | """ 115 | Represents a boolean (true or false) value. 116 | 117 | Renders as an unsigned char (1 byte). 118 | """ 119 | fmt = "?" 120 | 121 | 122 | class Byte(Primitive): 123 | """ 124 | Represents a single 8-bit byte. 125 | """ 126 | fmt = "b" 127 | 128 | 129 | class Int(Primitive): 130 | """ 131 | Represents an 32-bit signed integer. 132 | """ 133 | fmt = "i" 134 | 135 | 136 | class Long(Primitive): 137 | """ 138 | Represents an 64-bit signed integer. 139 | """ 140 | fmt = "q" 141 | 142 | 143 | class Float(Primitive): 144 | """ 145 | Represents a single-precision floating poing conforming to IEEE 754. 146 | """ 147 | fmt = "f" 148 | 149 | 150 | class Double(Primitive): 151 | """ 152 | Represents a double-precision floating poing conforming to IEEE 754. 153 | """ 154 | fmt = "d" 155 | 156 | 157 | class UString(VariablePrimitive): 158 | """ 159 | Represents a unicode string value, length denoted by a 32-bit integer. 160 | """ 161 | size_primitive = Int 162 | 163 | def render_value(self, value): 164 | return value.encode("utf8") 165 | 166 | @classmethod 167 | def parse_value(cls, value): 168 | return value.decode("utf8") 169 | 170 | def __str__(self): 171 | return self.value 172 | 173 | 174 | class Buffer(VariablePrimitive): 175 | """ 176 | Represents a bytestring value, length denoted by a 32-bit signed integer. 177 | """ 178 | size_primitive = Int 179 | 180 | def render_value(self, value): 181 | return value 182 | 183 | @classmethod 184 | def parse_value(cls, value): 185 | return value 186 | 187 | 188 | class Vector(Primitive): 189 | """ 190 | Represents an array of any arbitrary `Primitive` or ``Part``. 191 | 192 | Not used directly but rather by its ``of()`` classmethod to denote an 193 | ``Vector.of()``. 194 | """ 195 | item_class = None 196 | 197 | @classmethod 198 | def of(cls, part_class): 199 | """ 200 | Creates a new class with the ``item_class`` attribute properly set. 201 | """ 202 | copy = type( 203 | str("VectorOf%s" % part_class.__name__), 204 | cls.__bases__, dict(cls.__dict__) 205 | ) 206 | copy.item_class = part_class 207 | 208 | return copy 209 | 210 | def render(self): 211 | """ 212 | Creates a composite ``struct`` format and the data to render with it. 213 | 214 | The format and data are prefixed with a 32-bit integer denoting the 215 | number of elements, after which each of the items in the array value 216 | are ``render()``-ed and added to the format and data as well. 217 | """ 218 | value = self.value 219 | if value is None: 220 | value = [] 221 | 222 | fmt = [Int.fmt] 223 | data = [len(value)] 224 | 225 | for item_value in value: 226 | if issubclass(self.item_class, Primitive): 227 | item = self.item_class(item_value) 228 | else: 229 | item = item_value 230 | 231 | item_format, item_data = item.render() 232 | fmt.extend(item_format) 233 | data.extend(item_data) 234 | 235 | return "".join(fmt), data 236 | 237 | @classmethod 238 | def parse(cls, buff, offset): 239 | """ 240 | Parses a raw buffer at offset and returns the resulting array value. 241 | 242 | Starts off by `parse()`-ing the 32-bit element count, followed by 243 | parsing items out of the buffer "count" times. 244 | """ 245 | count, offset = Int.parse(buff, offset) 246 | 247 | values = [] 248 | for _ in range(count): 249 | value, new_offset = cls.item_class.parse(buff, offset) 250 | 251 | values.append(value) 252 | offset = new_offset 253 | 254 | return values, offset 255 | 256 | def __str__(self): 257 | return "%s[%s]" % ( 258 | self.item_class.__name__, ", ".join(map(str, self.value)) 259 | ) 260 | -------------------------------------------------------------------------------- /zoonado/protocol/reconfig.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from .request import Request 4 | from .response import Response 5 | from .stat import Stat 6 | from .primitives import Long, UString 7 | 8 | 9 | class ReconfigRequest(Request): 10 | """ 11 | """ 12 | opcode = 16 13 | 14 | parts = ( 15 | ("joining_servers", UString), 16 | ("leaving_servers", UString), 17 | ("new_members", UString), 18 | ("current_config_id", Long), 19 | ) 20 | 21 | 22 | class ReconfigResponse(Response): 23 | """ 24 | """ 25 | opcode = 16 26 | 27 | parts = ( 28 | ("stat", Stat), 29 | ) 30 | -------------------------------------------------------------------------------- /zoonado/protocol/request.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | import logging 4 | import struct 5 | 6 | from six import BytesIO 7 | 8 | from .part import Part 9 | from .primitives import Int 10 | 11 | 12 | log = logging.getLogger(__name__) 13 | 14 | 15 | class Request(Part): 16 | """ 17 | Returns a bytesring representation of the request instance. 18 | 19 | # TODO(wglass): specify how xid and type preamble goes in 20 | 21 | Since this is a ``Part`` subclass the rest is a matter of 22 | appending the result of a ``render()`` call. 23 | """ 24 | opcode = None 25 | special_xid = None 26 | writes_data = False 27 | 28 | def serialize(self, xid=None): 29 | buff = BytesIO() 30 | 31 | formats = [] 32 | data = [] 33 | 34 | if xid is not None: 35 | formats.append(Int.fmt) 36 | data.append(xid) 37 | 38 | if self.opcode: 39 | formats.append(Int.fmt) 40 | data.append(self.opcode) 41 | 42 | payload_format, payload_data = self.render() 43 | formats.append(payload_format) 44 | data.extend(payload_data) 45 | 46 | buff.write(struct.pack("!" + "".join(formats), *data)) 47 | 48 | return buff.getvalue() 49 | -------------------------------------------------------------------------------- /zoonado/protocol/response.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from zoonado.compat import add_metaclass 4 | 5 | from .part import Part 6 | 7 | 8 | response_xref = {} 9 | 10 | 11 | class ResponseMeta(type): 12 | 13 | def __new__(cls, name, bases, attrs): 14 | new_class = super(ResponseMeta, cls).__new__(cls, name, bases, attrs) 15 | 16 | response_xref[new_class.opcode] = new_class 17 | 18 | return new_class 19 | 20 | 21 | @add_metaclass(ResponseMeta) 22 | class Response(Part): 23 | """ 24 | Base class for all operation response classes. 25 | 26 | A simple class, has only an ``opcode`` attribute expected to be defined by 27 | subclasses, and a `deserialize()` classmethod. 28 | """ 29 | opcode = None 30 | 31 | @classmethod 32 | def deserialize(cls, raw_bytes): 33 | """ 34 | Deserializes the given raw bytes into an instance. 35 | 36 | Since this is a subclass of ``Part`` but a top-level one (i.e. no other 37 | subclass of ``Part`` would have a ``Response`` as a part) this merely 38 | has to parse the raw bytes and discard the resulting offset. 39 | """ 40 | instance, _ = cls.parse(raw_bytes, offset=0) 41 | 42 | return instance 43 | -------------------------------------------------------------------------------- /zoonado/protocol/sasl.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from .request import Request 4 | from .response import Response 5 | from .primitives import Buffer 6 | 7 | 8 | class SASLRequest(Request): 9 | """ 10 | """ 11 | opcode = 102 12 | 13 | parts = ( 14 | ("token", Buffer) 15 | ) 16 | 17 | 18 | class SASLResponse(Response): 19 | """ 20 | """ 21 | opcode = 102 22 | 23 | parts = ( 24 | ("token", Buffer) 25 | ) 26 | -------------------------------------------------------------------------------- /zoonado/protocol/stat.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from .part import Part 4 | from .primitives import Long, Int 5 | 6 | 7 | class Stat(Part): 8 | """ 9 | """ 10 | parts = ( 11 | ("created_zxid", Long), 12 | ("last_modified_zxid", Long), 13 | ("created", Long), 14 | ("modified", Long), 15 | ("version", Int), 16 | ("child_version", Int), 17 | ("acl_version", Int), 18 | ("ephemeral_owner", Long), 19 | ("data_length", Int), 20 | ("num_children", Int), 21 | ("last_modified_children", Long), 22 | ) 23 | 24 | 25 | class StatPersisted(Part): 26 | """ 27 | """ 28 | parts = ( 29 | ("created_zxid", Long), 30 | ("last_modified_zxid", Long), 31 | ("created", Long), 32 | ("modified", Long), 33 | ("version", Int), 34 | ("child_version", Int), 35 | ("acl_version", Int), 36 | ("ephemeral_owner", Long), 37 | ("last_modified_children", Long), 38 | ) 39 | -------------------------------------------------------------------------------- /zoonado/protocol/sync.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from .request import Request 4 | from .response import Response 5 | from .primitives import UString 6 | 7 | 8 | class SyncRequest(Request): 9 | """ 10 | """ 11 | opcode = 9 12 | 13 | parts = ( 14 | ("path", UString), 15 | ) 16 | 17 | 18 | class SyncResponse(Response): 19 | """ 20 | """ 21 | opcode = 9 22 | 23 | parts = ( 24 | ("path", UString), 25 | ) 26 | -------------------------------------------------------------------------------- /zoonado/protocol/transaction.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | import logging 4 | import struct 5 | 6 | from six import BytesIO 7 | 8 | from zoonado import exc 9 | 10 | from .request import Request 11 | from .response import Response, response_xref 12 | from .part import Part 13 | from .primitives import Int, Bool 14 | 15 | 16 | error_struct = struct.Struct("!" + Int.fmt) 17 | 18 | 19 | log = logging.getLogger(__name__) 20 | 21 | 22 | class MultiHeader(Part): 23 | """ 24 | """ 25 | parts = ( 26 | ("type", Int), 27 | ("done", Bool), 28 | ("error", Int), 29 | ) 30 | 31 | 32 | class TransactionRequest(Request): 33 | """ 34 | """ 35 | opcode = 14 36 | 37 | def __init__(self, *args, **kwargs): 38 | super(TransactionRequest, self).__init__(*args, **kwargs) 39 | self.requests = [] 40 | 41 | def add(self, request): 42 | self.requests.append(request) 43 | 44 | def serialize(self, xid=None): 45 | buff = BytesIO() 46 | 47 | formats = [] 48 | data = [] 49 | if xid is not None: 50 | formats.append(Int.fmt) 51 | data.append(xid) 52 | if self.opcode: 53 | formats.append(Int.fmt) 54 | data.append(self.opcode) 55 | 56 | for request in self.requests: 57 | header = MultiHeader(type=request.opcode, done=False, error=-1) 58 | header_format, header_data = header.render() 59 | formats.append(header_format) 60 | data.extend(header_data) 61 | 62 | payload_format, payload_data = request.render() 63 | formats.append(payload_format) 64 | data.extend(payload_data) 65 | 66 | footer = MultiHeader(type=-1, done=True, error=-1) 67 | footer_format, footer_data = footer.render() 68 | formats.append(footer_format) 69 | data.extend(footer_data) 70 | 71 | buff.write(struct.pack("!" + "".join(formats), *data)) 72 | 73 | return buff.getvalue() 74 | 75 | def __str__(self): 76 | return "Txn[%s]" % ", ".join(map(str, self.requests)) 77 | 78 | 79 | class TransactionResponse(Response): 80 | """ 81 | """ 82 | opcode = 14 83 | 84 | def __init__(self, *args, **kwargs): 85 | super(TransactionResponse, self).__init__(*args, **kwargs) 86 | self.responses = [] 87 | 88 | @classmethod 89 | def deserialize(cls, raw_bytes): 90 | instance = cls() 91 | 92 | header, offset = MultiHeader.parse(raw_bytes, 0) 93 | while not header.done: 94 | if header.type == -1: 95 | error_code = error_struct.unpack_from(raw_bytes, offset)[0] 96 | offset += error_struct.size 97 | instance.responses.append(exc.get_response_error(error_code)) 98 | header, offset = MultiHeader.parse(raw_bytes, offset) 99 | continue 100 | 101 | response_class = response_xref[header.type] 102 | 103 | response, offset = response_class.parse(raw_bytes, offset) 104 | instance.responses.append(response) 105 | 106 | header, offset = MultiHeader.parse(raw_bytes, offset) 107 | 108 | return instance 109 | 110 | def __str__(self): 111 | return "Txn[%s]" % ", ".join(map(str, self.responses)) 112 | -------------------------------------------------------------------------------- /zoonado/protocol/watches.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from .request import Request 4 | from .response import Response 5 | from .primitives import Int, Long, Vector, UString 6 | 7 | 8 | WATCH_XID = -1 9 | 10 | 11 | class WatchEvent(Response): 12 | """ 13 | """ 14 | 15 | CREATED = 1 16 | DELETED = 2 17 | DATA_CHANGED = 3 18 | CHILDREN_CHANGED = 4 19 | 20 | DISCONNECTED = 0 21 | CONNECTED = 3 22 | AUTH_FAILED = 4 23 | CONNECTED_READ_ONLY = 5 24 | SASL_AUTHENTICATED = 6 25 | SESSION_EXPIRED = -112 26 | 27 | parts = ( 28 | ("type", Int), 29 | ("state", Int), 30 | ("path", UString), 31 | ) 32 | 33 | 34 | class SetWatchesRequest(Request): 35 | """ 36 | """ 37 | opcode = 101 38 | 39 | parts = ( 40 | ("relative_zxid", Long), 41 | ("data_watches", Vector.of(UString)), 42 | ("exist_watches", Vector.of(UString)), 43 | ("child_watches", Vector.of(UString)), 44 | ) 45 | 46 | 47 | class SetWatchesResponse(Response): 48 | """ 49 | """ 50 | opcode = 101 51 | 52 | parts = () 53 | 54 | 55 | class CheckWatchesRequest(Request): 56 | """ 57 | """ 58 | opcode = 17 59 | 60 | parts = ( 61 | ("path", UString), 62 | ("type", Int), 63 | ) 64 | 65 | 66 | class CheckWatchesResponse(Response): 67 | """ 68 | """ 69 | opcode = 17 70 | 71 | parts = () 72 | 73 | 74 | class RemoveWatchesRequest(Request): 75 | """ 76 | """ 77 | opcode = 18 78 | 79 | parts = ( 80 | ("path", UString), 81 | ("type", Int), 82 | ) 83 | 84 | 85 | class RemoveWatchesResponse(Response): 86 | """ 87 | """ 88 | opcode = 18 89 | 90 | parts = () 91 | -------------------------------------------------------------------------------- /zoonado/recipes/__init__.py: -------------------------------------------------------------------------------- 1 | from .recipe import Recipe # noqa 2 | from .proxy import RecipeProxy # noqa 3 | -------------------------------------------------------------------------------- /zoonado/recipes/allocator.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | import collections 4 | import itertools 5 | import json 6 | import logging 7 | 8 | from tornado import gen, ioloop 9 | 10 | from .data_watcher import DataWatcher 11 | from .party import Party 12 | from .lock import Lock 13 | from .recipe import Recipe 14 | 15 | 16 | log = logging.getLogger(__name__) 17 | 18 | 19 | class Allocator(Recipe): 20 | 21 | sub_recipes = { 22 | "party": (Party, ["member_path", "name"]), 23 | "lock": (Lock, ["lock_path"]), 24 | "data_watcher": DataWatcher, 25 | } 26 | 27 | def __init__(self, base_path, name, allocator_fn=None): 28 | self.name = name 29 | 30 | super(Allocator, self).__init__(base_path) 31 | 32 | if allocator_fn is None: 33 | allocator_fn = round_robin 34 | 35 | self.allocator_fn = allocator_fn 36 | 37 | self.active = False 38 | 39 | self.full_allocation = collections.defaultdict(set) 40 | self.full_set = set() 41 | 42 | @property 43 | def lock_path(self): 44 | return self.base_path + "/lock" 45 | 46 | @property 47 | def member_path(self): 48 | return self.base_path + "/members" 49 | 50 | @property 51 | def allocation(self): 52 | return self.full_allocation[self.name] 53 | 54 | def validate(self, new_allocation): 55 | as_list = [] 56 | for subset in new_allocation.values(): 57 | as_list.extend(list(subset)) 58 | 59 | # make sure there are no duplicates among the subsets 60 | assert len(as_list) == len(set(as_list)), ( 61 | "duplicate items found in allocation: %s" % self.full_allocation 62 | ) 63 | 64 | # make sure there's no mismatch beween the full set and allocations 65 | assert len(self.full_set.symmetric_difference(set(as_list))) == 0, ( 66 | "mismatch between full set and allocation: %s vs %s" % ( 67 | self.full_set, self.full_allocation 68 | ) 69 | ) 70 | 71 | @gen.coroutine 72 | def start(self): 73 | self.active = True 74 | 75 | yield self.ensure_path() 76 | 77 | yield self.party.join() 78 | 79 | self.data_watcher.add_callback(self.base_path, self.handle_data_change) 80 | 81 | ioloop.IOLoop.current().add_callback(self.monitor_member_changes) 82 | 83 | @gen.coroutine 84 | def add(self, new_item): 85 | new_set = self.full_set.copy().add(new_item) 86 | yield self.update_set(new_set) 87 | 88 | @gen.coroutine 89 | def remove(self, new_item): 90 | new_set = self.full_set.copy().remove(new_item) 91 | yield self.update_set(new_set) 92 | 93 | @gen.coroutine 94 | def update(self, new_items): 95 | new_items = set(new_items) 96 | data = json.dumps(list(new_items)) 97 | 98 | with (yield self.lock.acquire()): 99 | yield self.client.set_data(self.base_path, data=data) 100 | 101 | def monitor_member_changes(self): 102 | while self.active: 103 | yield self.party.wait_for_change() 104 | if not self.active: 105 | break 106 | 107 | self.allocate() 108 | 109 | def handle_data_change(self, new_set_data): 110 | if new_set_data is None: 111 | return 112 | 113 | new_set_data = set(json.loads(new_set_data)) 114 | if new_set_data == self.full_set: 115 | return 116 | 117 | self.full_set = new_set_data 118 | self.allocate() 119 | 120 | def allocate(self): 121 | new_allocation = self.allocator_fn(self.party.members, self.full_set) 122 | self.validate(new_allocation) 123 | 124 | self.full_allocation = new_allocation 125 | 126 | @gen.coroutine 127 | def stop(self): 128 | yield self.party.leave() 129 | 130 | self.data_watcher.remove_callback( 131 | self.base_path, self.handle_data_change 132 | ) 133 | 134 | 135 | def round_robin(members, items): 136 | """ 137 | Default allocator with a round robin approach. 138 | 139 | In this algorithm, each member of the group is cycled over and given an 140 | item until there are no items left. This assumes roughly equal capacity 141 | for each member and aims for even distribution of item counts. 142 | """ 143 | allocation = collections.defaultdict(set) 144 | 145 | for member, item in zip(itertools.cycle(members), items): 146 | allocation[member].add(item) 147 | 148 | return allocation 149 | -------------------------------------------------------------------------------- /zoonado/recipes/barrier.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | import time 4 | 5 | from tornado import gen 6 | 7 | from zoonado import exc, WatchEvent 8 | 9 | from .recipe import Recipe 10 | 11 | 12 | class Barrier(Recipe): 13 | 14 | def __init__(self, path): 15 | super(Barrier, self).__init__() 16 | self.path = path 17 | 18 | @gen.coroutine 19 | def create(self): 20 | yield self.ensure_path() 21 | 22 | @gen.coroutine 23 | def lift(self): 24 | try: 25 | yield self.client.delete(self.path) 26 | except exc.NoNode: 27 | pass 28 | 29 | @gen.coroutine 30 | def wait(self, timeout=None): 31 | time_limit = None 32 | if timeout is not None: 33 | time_limit = time.time() + timeout 34 | 35 | barrier_lifted = self.client.wait_for_event( 36 | WatchEvent.DELETED, self.path 37 | ) 38 | 39 | if time_limit: 40 | barrier_lifted = gen.with_timeout(time_limit, barrier_lifted) 41 | 42 | exists = yield self.client.exists(path=self.path, watch=True) 43 | if not exists: 44 | return 45 | 46 | try: 47 | yield barrier_lifted 48 | except gen.TimeoutError: 49 | raise exc.TimeoutError 50 | -------------------------------------------------------------------------------- /zoonado/recipes/base_lock.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | import contextlib 4 | import logging 5 | import time 6 | 7 | from tornado import gen, ioloop 8 | 9 | from zoonado import exc, states 10 | 11 | from .sequential import SequentialRecipe 12 | 13 | 14 | log = logging.getLogger(__name__) 15 | 16 | 17 | class BaseLock(SequentialRecipe): 18 | 19 | @gen.coroutine 20 | def wait_in_line(self, znode_label, timeout=None, blocked_by=None): 21 | time_limit = None 22 | if timeout is not None: 23 | time_limit = time.time() + timeout 24 | 25 | yield self.create_unique_znode(znode_label) 26 | 27 | while True: 28 | if time_limit and time.time() >= time_limit: 29 | raise exc.TimeoutError 30 | 31 | owned_positions, contenders = yield self.analyze_siblings() 32 | if znode_label not in owned_positions: 33 | raise exc.SessionLost 34 | 35 | blockers = contenders[:owned_positions[znode_label]] 36 | if blocked_by: 37 | blockers = [ 38 | contender for contender in blockers 39 | if self.determine_znode_label(contender) in blocked_by 40 | ] 41 | 42 | if not blockers: 43 | break 44 | 45 | yield self.wait_on_sibling(blockers[-1], time_limit) 46 | 47 | raise gen.Return(self.make_contextmanager(znode_label)) 48 | 49 | def make_contextmanager(self, znode_label): 50 | state = {"acquired": True} 51 | 52 | def still_acquired(): 53 | return state["acquired"] 54 | 55 | @gen.coroutine 56 | def handle_session_loss(): 57 | yield self.client.session.state.wait_for(states.States.LOST) 58 | if not state["acquired"]: 59 | return 60 | 61 | log.warn( 62 | "Session expired at some point, lock %s no longer acquired.", 63 | self 64 | ) 65 | state["acquired"] = False 66 | 67 | ioloop.IOLoop.current().add_callback(handle_session_loss) 68 | 69 | @gen.coroutine 70 | def on_exit(): 71 | state["acquired"] = False 72 | yield self.delete_unique_znode(znode_label) 73 | 74 | @contextlib.contextmanager 75 | def context_manager(): 76 | try: 77 | yield still_acquired 78 | finally: 79 | ioloop.IOLoop.current().add_callback(on_exit) 80 | 81 | return context_manager() 82 | 83 | 84 | class LockLostError(exc.ZKError): 85 | pass 86 | -------------------------------------------------------------------------------- /zoonado/recipes/base_watcher.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | import collections 4 | import logging 5 | 6 | from tornado import ioloop, gen 7 | 8 | from zoonado import exc 9 | 10 | from .recipe import Recipe 11 | 12 | 13 | log = logging.getLogger(__name__) 14 | 15 | 16 | class BaseWatcher(Recipe): 17 | 18 | watched_event = None 19 | 20 | def __init__(self, *args, **kwargs): 21 | super(BaseWatcher, self).__init__(*args, **kwargs) 22 | self.callbacks = collections.defaultdict(set) 23 | 24 | def add_callback(self, path, callback): 25 | self.callbacks[path].add(callback) 26 | 27 | if len(self.callbacks[path]) == 1: 28 | ioloop.IOLoop.current().add_callback(self.watch_loop, path) 29 | 30 | def remove_callback(self, path, callback): 31 | self.callbacks[path].discard(callback) 32 | 33 | @gen.coroutine 34 | def fetch(self, path): 35 | raise NotImplementedError 36 | 37 | @gen.coroutine 38 | def watch_loop(self, path): 39 | while self.callbacks[path]: 40 | try: 41 | result = yield self.fetch(path) 42 | except exc.NoNode: 43 | return 44 | 45 | for callback in self.callbacks[path]: 46 | ioloop.IOLoop.current().add_callback(callback, result) 47 | 48 | yield self.client.wait_for_event(self.watched_event, path) 49 | -------------------------------------------------------------------------------- /zoonado/recipes/children_watcher.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | import logging 4 | 5 | from tornado import gen 6 | from zoonado import WatchEvent 7 | 8 | from .base_watcher import BaseWatcher 9 | 10 | 11 | log = logging.getLogger(__name__) 12 | 13 | 14 | class ChildrenWatcher(BaseWatcher): 15 | 16 | watched_event = WatchEvent.CHILDREN_CHANGED 17 | 18 | @gen.coroutine 19 | def fetch(self, path): 20 | log.debug("Fetching children for %s", path) 21 | children = yield self.client.get_children(path=path, watch=True) 22 | raise gen.Return(children) 23 | -------------------------------------------------------------------------------- /zoonado/recipes/counter.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | import logging 4 | 5 | from tornado import gen, concurrent 6 | 7 | from zoonado import exc 8 | 9 | from .data_watcher import DataWatcher 10 | from .recipe import Recipe 11 | 12 | 13 | log = logging.getLogger(__name__) 14 | 15 | 16 | class Counter(Recipe): 17 | 18 | sub_recipes = { 19 | "watcher": DataWatcher 20 | } 21 | 22 | def __init__(self, base_path, use_float=False): 23 | super(Counter, self).__init__(base_path) 24 | 25 | self.value = None 26 | 27 | if use_float: 28 | self.numeric_type = float 29 | else: 30 | self.numeric_type = int 31 | 32 | self.value_sync = concurrent.Future() 33 | 34 | @gen.coroutine 35 | def start(self): 36 | self.watcher.add_callback(self.base_path, self.data_callback) 37 | yield gen.moment 38 | 39 | yield self.ensure_path() 40 | 41 | raw_value = yield self.client.get_data(self.base_path) 42 | self.value = self.numeric_type(raw_value or 0) 43 | 44 | def data_callback(self, new_value): 45 | self.value = self.numeric_type(new_value) 46 | if not self.value_sync.done(): 47 | self.value_sync.set_result(None) 48 | self.value_sync = concurrent.Future() 49 | 50 | @gen.coroutine 51 | def set_value(self, value, force=True): 52 | data = str(value) 53 | yield self.client.set_data(self.base_path, data, force=force) 54 | log.debug("Set value to '%s': successful", data) 55 | yield self.value_sync 56 | 57 | @gen.coroutine 58 | def apply_operation(self, operation): 59 | success = False 60 | while not success: 61 | data = str(operation(self.value)) 62 | try: 63 | yield self.client.set_data(self.base_path, data, force=False) 64 | log.debug("Operation '%s': successful", operation.__name__) 65 | yield self.value_sync 66 | success = True 67 | except exc.BadVersion: 68 | log.debug( 69 | "Operation '%s': version mismatch, retrying", 70 | operation.__name__ 71 | ) 72 | yield self.value_sync 73 | 74 | @gen.coroutine 75 | def incr(self): 76 | 77 | def increment(value): 78 | return value + 1 79 | 80 | yield self.apply_operation(increment) 81 | 82 | @gen.coroutine 83 | def decr(self): 84 | 85 | def decrement(value): 86 | return value - 1 87 | 88 | yield self.apply_operation(decrement) 89 | 90 | @gen.coroutine 91 | def stop(self): 92 | self.watcher.remove_callback(self.base_path, self.data_callback) 93 | -------------------------------------------------------------------------------- /zoonado/recipes/data_watcher.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | import logging 4 | 5 | from tornado import gen 6 | from zoonado import WatchEvent 7 | 8 | from .base_watcher import BaseWatcher 9 | 10 | 11 | log = logging.getLogger(__name__) 12 | 13 | 14 | class DataWatcher(BaseWatcher): 15 | 16 | watched_event = WatchEvent.DATA_CHANGED 17 | 18 | @gen.coroutine 19 | def fetch(self, path): 20 | log.debug("Fetching data for %s", path) 21 | data = yield self.client.get_data(path=path, watch=True) 22 | raise gen.Return(data) 23 | -------------------------------------------------------------------------------- /zoonado/recipes/double_barrier.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | import logging 4 | import time 5 | 6 | from tornado import gen 7 | 8 | from zoonado import exc, WatchEvent 9 | 10 | from .sequential import SequentialRecipe 11 | 12 | log = logging.getLogger(__name__) 13 | 14 | 15 | class DoubleBarrier(SequentialRecipe): 16 | 17 | def __init__(self, base_path, min_participants): 18 | super(DoubleBarrier, self).__init__(base_path) 19 | self.min_participants = min_participants 20 | 21 | @property 22 | def sentinel_path(self): 23 | return self.sibling_path("sentinel") 24 | 25 | @gen.coroutine 26 | def enter(self, timeout=None): 27 | log.debug("Entering double barrier %s", self.base_path) 28 | time_limit = None 29 | if timeout is not None: 30 | time_limit = time.time() + timeout 31 | 32 | barrier_lifted = self.client.wait_for_event( 33 | WatchEvent.CREATED, self.sentinel_path 34 | ) 35 | if time_limit: 36 | barrier_lifted = gen.with_timeout(time_limit, barrier_lifted) 37 | 38 | exists = yield self.client.exists(path=self.sentinel_path, watch=True) 39 | 40 | yield self.create_unique_znode("worker") 41 | 42 | _, participants = yield self.analyze_siblings() 43 | 44 | if exists: 45 | return 46 | 47 | elif len(participants) >= self.min_participants: 48 | yield self.create_znode(self.sentinel_path) 49 | return 50 | 51 | try: 52 | yield barrier_lifted 53 | except gen.TimeoutError: 54 | raise exc.TimeoutError 55 | 56 | @gen.coroutine 57 | def leave(self, timeout=None): 58 | log.debug("Leaving double barrier %s", self.base_path) 59 | time_limit = None 60 | if timeout is not None: 61 | time_limit = time.time() + timeout 62 | 63 | owned_positions, participants = yield self.analyze_siblings() 64 | while len(participants) > 1: 65 | if owned_positions["worker"] == 0: 66 | yield self.wait_on_sibling(participants[-1], time_limit) 67 | else: 68 | yield self.delete_unique_znode("worker") 69 | yield self.wait_on_sibling(participants[0], time_limit) 70 | 71 | owned_positions, participants = yield self.analyze_siblings() 72 | 73 | if len(participants) == 1 and "worker" in owned_positions: 74 | yield self.delete_unique_znode("worker") 75 | try: 76 | yield self.client.delete(self.sentinel_path) 77 | except exc.NoNode: 78 | pass 79 | -------------------------------------------------------------------------------- /zoonado/recipes/election.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | import time 4 | 5 | from tornado import gen, ioloop, concurrent 6 | 7 | from .sequential import SequentialRecipe 8 | 9 | 10 | class LeaderElection(SequentialRecipe): 11 | 12 | def __init__(self, base_path): 13 | super(LeaderElection, self).__init__(base_path) 14 | self.has_leadership = False 15 | 16 | self.leadership_future = concurrent.Future() 17 | 18 | @gen.coroutine 19 | def join(self): 20 | yield self.create_unique_znode("candidate") 21 | yield self.check_position() 22 | 23 | @gen.coroutine 24 | def check_position(self, _=None): 25 | owned_positions, candidates = yield self.analyze_siblings() 26 | if "candidate" not in owned_positions: 27 | return 28 | 29 | position = owned_positions["candidate"] 30 | 31 | self.has_leadership = bool(position == 0) 32 | 33 | if self.has_leadership: 34 | self.leadership_future.set_result(None) 35 | return 36 | 37 | moved_up = self.wait_on_sibling(candidates[position - 1]) 38 | 39 | ioloop.IOLoop.current().add_future(moved_up, self.check_position) 40 | 41 | @gen.coroutine 42 | def wait_for_leadership(self, timeout=None): 43 | if self.has_leadership: 44 | return 45 | 46 | time_limit = None 47 | if timeout is not None: 48 | time_limit = time.time() + timeout 49 | 50 | if time_limit: 51 | yield gen.with_timeout(time_limit, self.leadership_future) 52 | else: 53 | yield self.leadership_future 54 | 55 | @gen.coroutine 56 | def resign(self): 57 | yield self.delete_unique_znode("candidate") 58 | -------------------------------------------------------------------------------- /zoonado/recipes/lease.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | import logging 4 | import time 5 | 6 | from tornado import gen, ioloop 7 | from zoonado import exc 8 | 9 | from .sequential import SequentialRecipe 10 | 11 | 12 | log = logging.getLogger(__name__) 13 | 14 | 15 | class Lease(SequentialRecipe): 16 | 17 | def __init__(self, base_path, limit=1): 18 | super(Lease, self).__init__(base_path) 19 | self.limit = limit 20 | 21 | @gen.coroutine 22 | def obtain(self, duration): 23 | lessees = yield self.client.get_children(self.base_path) 24 | 25 | if len(lessees) >= self.limit: 26 | raise gen.Return(False) 27 | 28 | time_limit = time.time() + duration.total_seconds() 29 | 30 | try: 31 | yield self.create_unique_znode("lease", data=str(time_limit)) 32 | except exc.NodeExists: 33 | log.warn("Lease for %s already obtained.", self.base_path) 34 | 35 | ioloop.IOLoop.current().call_at(time_limit, self.release) 36 | 37 | raise gen.Return(True) 38 | 39 | @gen.coroutine 40 | def release(self): 41 | yield self.delete_unique_znode("lease") 42 | -------------------------------------------------------------------------------- /zoonado/recipes/lock.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from tornado import gen 4 | 5 | from zoonado import exc 6 | 7 | from .base_lock import BaseLock 8 | 9 | 10 | class Lock(BaseLock): 11 | 12 | @gen.coroutine 13 | def acquire(self, timeout=None): 14 | result = None 15 | while not result: 16 | try: 17 | result = yield self.wait_in_line("lock", timeout) 18 | except exc.SessionLost: 19 | continue 20 | 21 | raise gen.Return(result) 22 | -------------------------------------------------------------------------------- /zoonado/recipes/party.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | import logging 4 | 5 | from tornado import gen, concurrent 6 | 7 | from .children_watcher import ChildrenWatcher 8 | from .sequential import SequentialRecipe 9 | 10 | 11 | log = logging.getLogger(__name__) 12 | 13 | 14 | class Party(SequentialRecipe): 15 | 16 | sub_recipes = { 17 | "watcher": ChildrenWatcher, 18 | } 19 | 20 | def __init__(self, base_path, name): 21 | super(Party, self).__init__(base_path) 22 | 23 | self.name = name 24 | self.members = [] 25 | self.change_future = None 26 | 27 | @gen.coroutine 28 | def join(self): 29 | yield self.create_unique_znode(self.name) 30 | 31 | _, siblings = yield self.analyze_siblings() 32 | self.update_members(siblings) 33 | 34 | self.watcher.add_callback(self.base_path, self.update_members) 35 | 36 | @gen.coroutine 37 | def wait_for_change(self): 38 | if not self.change_future or self.change_future.done(): 39 | self.change_future = concurrent.Future() 40 | 41 | yield self.change_future 42 | 43 | @gen.coroutine 44 | def leave(self): 45 | self.watcher.remove_callback(self.base_path, self.update_members) 46 | yield self.delete_unique_znode(self.name) 47 | 48 | def update_members(self, raw_sibling_names): 49 | new_members = [ 50 | self.determine_znode_label(sibling) 51 | for sibling in raw_sibling_names 52 | ] 53 | 54 | self.members = new_members 55 | if self.change_future and not self.change_future.done(): 56 | self.change_future.set_result(new_members) 57 | -------------------------------------------------------------------------------- /zoonado/recipes/proxy.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | import logging 4 | import pkg_resources 5 | 6 | from .recipe import Recipe 7 | 8 | 9 | ENTRY_POINT = "zoonado.recipes" 10 | 11 | log = logging.getLogger(__name__) 12 | 13 | 14 | class RecipeClassProxy(object): 15 | 16 | def __init__(self, client, recipe_class): 17 | self.client = client 18 | self.recipe_class = recipe_class 19 | 20 | def __call__(self, *args, **kwargs): 21 | recipe = self.recipe_class(*args, **kwargs) 22 | recipe.set_client(self.client) 23 | return recipe 24 | 25 | 26 | class RecipeProxy(object): 27 | 28 | def __init__(self, client): 29 | self.client = client 30 | 31 | self.installed_classes = {} 32 | self.gather_installed_classes() 33 | 34 | def __getattr__(self, name): 35 | if name not in self.installed_classes: 36 | raise AttributeError("No such recipe: %s" % name) 37 | 38 | return RecipeClassProxy(self.client, self.installed_classes[name]) 39 | 40 | def gather_installed_classes(self): 41 | for entry_point in pkg_resources.iter_entry_points(ENTRY_POINT): 42 | try: 43 | recipe_class = entry_point.load() 44 | except ImportError as e: 45 | log.error( 46 | "Could not load recipe %s: %s", entry_point.name, str(e) 47 | ) 48 | continue 49 | 50 | if not issubclass(recipe_class, Recipe): 51 | log.error( 52 | "Could not load recipe %s: not a Recipe subclass", 53 | entry_point.name 54 | ) 55 | continue 56 | 57 | if not recipe_class.validate_dependencies(): 58 | log.error( 59 | "Could not load recipe %s: %s has unmet dependencies", 60 | entry_point.name, recipe_class.__name__ 61 | ) 62 | continue 63 | 64 | log.debug("Loaded recipe %s", recipe_class.__name__) 65 | self.installed_classes[recipe_class.__name__] = recipe_class 66 | -------------------------------------------------------------------------------- /zoonado/recipes/recipe.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from tornado import gen 4 | 5 | from zoonado import exc 6 | 7 | 8 | class Recipe(object): 9 | 10 | sub_recipes = {} 11 | 12 | def __init__(self, base_path="/"): 13 | self.client = None 14 | self.base_path = base_path 15 | 16 | for attribute_name, recipe_class in self.sub_recipes.items(): 17 | recipe_args = ["base_path"] 18 | if isinstance(recipe_class, tuple): 19 | recipe_class, recipe_args = recipe_class 20 | 21 | recipe_args = [getattr(self, arg) for arg in recipe_args] 22 | 23 | recipe = recipe_class(*recipe_args) 24 | 25 | setattr(self, attribute_name, recipe) 26 | 27 | def set_client(self, client): 28 | self.client = client 29 | for sub_recipe in self.sub_recipes.keys(): 30 | getattr(self, sub_recipe).set_client(client) 31 | 32 | @classmethod 33 | def validate_dependencies(cls): 34 | return True 35 | 36 | @gen.coroutine 37 | def ensure_path(self): 38 | yield self.client.ensure_path(self.base_path) 39 | 40 | @gen.coroutine 41 | def create_znode(self, path): 42 | try: 43 | yield self.client.create(path) 44 | except exc.NodeExists: 45 | pass 46 | except exc.NoNode: 47 | try: 48 | yield self.ensure_path() 49 | except exc.NodeExists: 50 | pass 51 | -------------------------------------------------------------------------------- /zoonado/recipes/sequential.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | import logging 4 | import re 5 | import uuid 6 | 7 | from tornado import gen 8 | from zoonado import exc, WatchEvent 9 | 10 | from .recipe import Recipe 11 | 12 | 13 | log = logging.getLogger(__name__) 14 | 15 | sequential_re = re.compile(r'.*[0-9]{10}$') 16 | 17 | 18 | class SequentialRecipe(Recipe): 19 | 20 | def __init__(self, base_path): 21 | super(SequentialRecipe, self).__init__(base_path) 22 | self.guid = uuid.uuid4().hex 23 | 24 | self.owned_paths = {} 25 | 26 | def sequence_number(self, sibling): 27 | return int(sibling[-10:]) 28 | 29 | def determine_znode_label(self, sibling): 30 | return sibling.rsplit("-", 2)[0] 31 | 32 | def sibling_path(self, path): 33 | return "/".join([self.base_path, path]) 34 | 35 | @gen.coroutine 36 | def create_unique_znode(self, znode_label, data=None): 37 | path = self.sibling_path(znode_label + "-" + self.guid + "-") 38 | 39 | try: 40 | created_path = yield self.client.create( 41 | path, data=data, ephemeral=True, sequential=True 42 | ) 43 | except exc.NoNode: 44 | yield self.ensure_path() 45 | created_path = yield self.client.create( 46 | path, data=data, ephemeral=True, sequential=True 47 | ) 48 | 49 | self.owned_paths[znode_label] = created_path 50 | 51 | @gen.coroutine 52 | def delete_unique_znode(self, znode_label): 53 | try: 54 | yield self.client.delete(self.owned_paths[znode_label]) 55 | except exc.NoNode: 56 | pass 57 | 58 | @gen.coroutine 59 | def analyze_siblings(self): 60 | siblings = yield self.client.get_children(self.base_path) 61 | siblings = [name for name in siblings if sequential_re.match(name)] 62 | 63 | siblings.sort(key=self.sequence_number) 64 | 65 | owned_positions = {} 66 | 67 | for index, path in enumerate(siblings): 68 | if self.guid in path: 69 | owned_positions[self.determine_znode_label(path)] = index 70 | 71 | raise gen.Return((owned_positions, siblings)) 72 | 73 | @gen.coroutine 74 | def wait_on_sibling(self, sibling, time_limit=None): 75 | log.debug("Waiting on sibling %s", sibling) 76 | 77 | path = self.sibling_path(sibling) 78 | 79 | unblocked = self.client.wait_for_event(WatchEvent.DELETED, path) 80 | if time_limit: 81 | unblocked = gen.with_timeout(time_limit, unblocked) 82 | 83 | exists = yield self.client.exists(path=path, watch=True) 84 | if not exists: 85 | unblocked.set_result(None) 86 | 87 | try: 88 | yield unblocked 89 | except gen.TimeoutError: 90 | raise exc.TimeoutError 91 | -------------------------------------------------------------------------------- /zoonado/recipes/shared_lock.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from tornado import gen 4 | 5 | from zoonado import exc 6 | 7 | from .base_lock import BaseLock 8 | 9 | 10 | class SharedLock(BaseLock): 11 | 12 | @gen.coroutine 13 | def acquire_read(self, timeout=None): 14 | result = None 15 | while not result: 16 | try: 17 | result = yield self.wait_in_line( 18 | "read", timeout, blocked_by=("write") 19 | ) 20 | except exc.SessionLost: 21 | continue 22 | 23 | raise gen.Return(result) 24 | 25 | @gen.coroutine 26 | def acquire_write(self, timeout=None): 27 | result = None 28 | while not result: 29 | try: 30 | result = yield self.wait_in_line("write", timeout) 31 | except exc.SessionLost: 32 | continue 33 | 34 | raise gen.Return(result) 35 | -------------------------------------------------------------------------------- /zoonado/recipes/tree_cache.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | import logging 4 | 5 | import six 6 | from tornado import gen, ioloop 7 | 8 | from .children_watcher import ChildrenWatcher 9 | from .data_watcher import DataWatcher 10 | from .recipe import Recipe 11 | 12 | 13 | log = logging.getLogger(__name__) 14 | 15 | 16 | class TreeCache(Recipe): 17 | 18 | sub_recipes = { 19 | "data_watcher": DataWatcher, 20 | "child_watcher": ChildrenWatcher, 21 | } 22 | 23 | def __init__(self, base_path, defaults=None): 24 | super(TreeCache, self).__init__(base_path) 25 | self.defaults = defaults or {} 26 | self.root = None 27 | 28 | @gen.coroutine 29 | def start(self): 30 | log.debug("Starting znode tree cache at %s", self.base_path) 31 | 32 | self.root = ZNodeCache( 33 | self.base_path, self.defaults, 34 | self.client, self.data_watcher, self.child_watcher, 35 | ) 36 | 37 | yield self.ensure_path() 38 | 39 | yield self.root.start() 40 | 41 | def stop(self): 42 | self.root.stop() 43 | 44 | def __getattr__(self, attribute): 45 | return getattr(self.root, attribute) 46 | 47 | def as_dict(self): 48 | return self.root.as_dict() 49 | 50 | 51 | class ZNodeCache(object): 52 | 53 | def __init__(self, path, defaults, client, data_watcher, child_watcher): 54 | self.path = path 55 | 56 | self.client = client 57 | self.defaults = defaults 58 | 59 | self.data_watcher = data_watcher 60 | self.child_watcher = child_watcher 61 | 62 | self.children = {} 63 | self.data = None 64 | 65 | @property 66 | def dot_path(self): 67 | return self.path[1:].replace("/", ".") 68 | 69 | @property 70 | def value(self): 71 | return self.data 72 | 73 | def __getattr__(self, name): 74 | if name not in self.children: 75 | raise AttributeError 76 | 77 | return self.children[name] 78 | 79 | @gen.coroutine 80 | def start(self): 81 | data, children = yield [ 82 | self.client.get_data(self.path), 83 | self.client.get_children(self.path) 84 | ] 85 | 86 | self.data = data 87 | for child in children: 88 | self.add_child_znode_cache(child) 89 | 90 | yield [child.start() for child in self.children.values()] 91 | 92 | self.data_watcher.add_callback(self.path, self.data_callback) 93 | self.child_watcher.add_callback(self.path, self.child_callback) 94 | 95 | def stop(self): 96 | self.data_watcher.remove_callback(self.path, self.data_callback) 97 | self.child_watcher.remove_callback(self.path, self.child_callback) 98 | 99 | def child_callback(self, new_children): 100 | removed_children = set(self.children.keys()) - set(new_children) 101 | added_children = set(new_children) - set(self.children.keys()) 102 | 103 | for removed in removed_children: 104 | log.debug("Removed child %s", self.dot_path + "." + removed) 105 | child = self.children.pop(removed) 106 | child.stop() 107 | 108 | for added in added_children: 109 | log.debug("added child %s", self.dot_path + "." + added) 110 | self.add_child_znode_cache(added) 111 | ioloop.IOLoop.current().add_callback(self.children[added].start) 112 | 113 | def data_callback(self, data): 114 | log.debug("New value for %s: %r", self.dot_path, data) 115 | self.data = data 116 | 117 | def add_child_znode_cache(self, child_name): 118 | self.children[child_name] = ZNodeCache( 119 | self.path + "/" + child_name, self.defaults.get(child_name, {}), 120 | self.client, self.data_watcher, self.child_watcher 121 | ) 122 | 123 | def as_dict(self): 124 | if self.children: 125 | return { 126 | child_path: child_znode.as_dict() 127 | for child_path, child_znode in six.iteritems(self.children) 128 | } 129 | 130 | return self.data 131 | -------------------------------------------------------------------------------- /zoonado/retry.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | import collections 4 | import logging 5 | import time 6 | 7 | from tornado import gen 8 | 9 | from zoonado import exc 10 | 11 | 12 | log = logging.getLogger(__name__) 13 | 14 | 15 | class RetryPolicy(object): 16 | 17 | def __init__(self, try_limit, sleep_func): 18 | self.try_limit = try_limit 19 | self.sleep_func = sleep_func 20 | 21 | self.timings = collections.defaultdict(list) 22 | 23 | @gen.coroutine 24 | def enforce(self, request=None): 25 | self.timings[id(request)].append(time.time()) 26 | 27 | tries = len(self.timings[id(request)]) 28 | if tries == 1: 29 | return 30 | 31 | if self.try_limit is not None and tries >= self.try_limit: 32 | raise exc.FailedRetry 33 | 34 | wait_time = self.sleep_func(self.timings[id(request)]) 35 | if wait_time is None or wait_time == 0: 36 | return 37 | elif wait_time < 0: 38 | raise exc.FailedRetry 39 | 40 | log.debug("Waiting %d seconds until next try.", wait_time) 41 | yield gen.sleep(wait_time) 42 | 43 | def clear(self, request): 44 | self.timings.pop(id(request), None) 45 | 46 | @classmethod 47 | def once(cls): 48 | return cls.n_times(1) 49 | 50 | @classmethod 51 | def n_times(cls, n): 52 | 53 | def never_wait(_): 54 | return None 55 | 56 | return cls(try_limit=n, sleep_func=never_wait) 57 | 58 | @classmethod 59 | def forever(cls): 60 | 61 | def never_wait(_): 62 | return None 63 | 64 | return cls(try_limit=None, sleep_func=never_wait) 65 | 66 | @classmethod 67 | def exponential_backoff(cls, base=2, maximum=None): 68 | 69 | def exponential(timings): 70 | wait_time = base ** len(timings) 71 | if maximum is not None: 72 | wait_time = min(maximum, wait_time) 73 | 74 | return wait_time 75 | 76 | return cls(try_limit=None, sleep_func=exponential) 77 | 78 | @classmethod 79 | def until_elapsed(cls, timeout): 80 | 81 | def elapsed_time(timings): 82 | if timings: 83 | first_timing = timings[0] 84 | else: 85 | first_timing = time.time() 86 | 87 | return (first_timing + timeout) - time.time() 88 | 89 | return cls(try_limit=None, sleep_func=elapsed_time) 90 | -------------------------------------------------------------------------------- /zoonado/session.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | import collections 4 | import logging 5 | import random 6 | 7 | from tornado import gen, ioloop, iostream 8 | 9 | from zoonado import protocol, exc 10 | from .connection import Connection 11 | from .states import States, SessionStateMachine 12 | from .retry import RetryPolicy 13 | 14 | 15 | DEFAULT_ZOOKEEPER_PORT = 2181 16 | 17 | MAX_FIND_WAIT = 60 # in seconds 18 | 19 | HEARTBEAT_FREQUENCY = 3 # heartbeats per timeout interval 20 | 21 | 22 | log = logging.getLogger(__name__) 23 | 24 | 25 | class Session(object): 26 | 27 | def __init__(self, servers, timeout, retry_policy, allow_read_only): 28 | self.hosts = [] 29 | for server in servers.split(","): 30 | if ":" in server: 31 | host, port = server.split(":") 32 | else: 33 | host = server 34 | port = DEFAULT_ZOOKEEPER_PORT 35 | 36 | self.hosts.append((host, port)) 37 | 38 | self.conn = None 39 | self.state = SessionStateMachine() 40 | 41 | self.retry_policy = retry_policy or RetryPolicy.forever() 42 | self.allow_read_only = allow_read_only 43 | 44 | self.xid = 0 45 | self.last_zxid = None 46 | 47 | self.session_id = None 48 | self.timeout = timeout 49 | self.password = b'\x00' 50 | 51 | self.heartbeat_handle = None 52 | 53 | self.watch_callbacks = collections.defaultdict(set) 54 | 55 | self.closing = False 56 | 57 | @gen.coroutine 58 | def ensure_safe_state(self, writing=False): 59 | safe_states = [States.CONNECTED] 60 | if self.allow_read_only and not writing: 61 | safe_states.append(States.READ_ONLY) 62 | 63 | if self.state in safe_states: 64 | return 65 | 66 | yield self.state.wait_for(*safe_states) 67 | 68 | @gen.coroutine 69 | def start(self): 70 | io_loop = ioloop.IOLoop.current() 71 | io_loop.add_callback(self.set_heartbeat) 72 | io_loop.add_callback(self.repair_loop) 73 | 74 | yield self.ensure_safe_state() 75 | 76 | @gen.coroutine 77 | def find_server(self, allow_read_only): 78 | conn = None 79 | 80 | retry_policy = RetryPolicy.exponential_backoff(maximum=MAX_FIND_WAIT) 81 | 82 | while not conn: 83 | yield retry_policy.enforce() 84 | 85 | servers = random.sample(self.hosts, len(self.hosts)) 86 | for host, port in servers: 87 | log.info("Connecting to %s:%s", host, port) 88 | conn = yield self.make_connection(host, port) 89 | if not conn or (conn.start_read_only and not allow_read_only): 90 | continue 91 | 92 | if not conn: 93 | log.warn("No servers available, will keep trying.") 94 | 95 | old_conn = self.conn 96 | self.conn = conn 97 | 98 | io_loop = ioloop.IOLoop.current() 99 | 100 | if old_conn: 101 | io_loop.add_callback(old_conn.close, self.timeout) 102 | 103 | if conn.start_read_only: 104 | io_loop.add_callback(self.find_server, allow_read_only=False) 105 | 106 | @gen.coroutine 107 | def make_connection(self, host, port): 108 | conn = Connection(host, port, watch_handler=self.event_dispatch) 109 | try: 110 | yield conn.connect() 111 | except Exception: 112 | log.exception("Couldn't connect to %s:%s", host, port) 113 | return 114 | 115 | raise gen.Return(conn) 116 | 117 | @gen.coroutine 118 | def establish_session(self): 119 | log.info("Establising session.") 120 | zxid, response = yield self.conn.send_connect( 121 | protocol.ConnectRequest( 122 | protocol_version=0, 123 | last_seen_zxid=self.last_zxid or 0, 124 | timeout=int((self.timeout or 0) * 1000), 125 | session_id=self.session_id or 0, 126 | password=self.password, 127 | read_only=self.allow_read_only, 128 | ) 129 | ) 130 | self.last_zxid = zxid 131 | 132 | if response.session_id == 0: # invalid session, probably expired 133 | self.state.transition_to(States.LOST) 134 | raise exc.SessionLost() 135 | 136 | log.info("Got session id %s", hex(response.session_id)) 137 | log.info("Negotiated timeout: %s seconds", response.timeout / 1000) 138 | 139 | self.session_id = response.session_id 140 | self.password = response.password 141 | self.timeout = response.timeout / 1000 142 | 143 | self.last_zxid = zxid 144 | 145 | @gen.coroutine 146 | def repair_loop(self): 147 | while not self.closing: 148 | yield self.state.wait_for(States.SUSPENDED, States.LOST) 149 | if self.closing: 150 | break 151 | 152 | yield self.find_server(allow_read_only=self.allow_read_only) 153 | 154 | session_was_lost = self.state == States.LOST 155 | 156 | try: 157 | yield self.establish_session() 158 | except exc.SessionLost: 159 | self.conn.abort(exc.SessionLost) 160 | yield self.conn.close(self.timeout) 161 | self.session_id = None 162 | self.password = b'\x00' 163 | continue 164 | 165 | if self.conn.start_read_only: 166 | self.state.transition_to(States.READ_ONLY) 167 | else: 168 | self.state.transition_to(States.CONNECTED) 169 | 170 | self.conn.start_read_loop() 171 | 172 | if session_was_lost: 173 | yield self.set_existing_watches() 174 | 175 | @gen.coroutine 176 | def send(self, request): 177 | response = None 178 | while not response: 179 | yield self.retry_policy.enforce(request) 180 | yield self.ensure_safe_state(writing=request.writes_data) 181 | 182 | try: 183 | self.xid += 1 184 | zxid, response = yield self.conn.send(request, xid=self.xid) 185 | self.last_zxid = zxid 186 | self.set_heartbeat() 187 | self.retry_policy.clear(request) 188 | except exc.ConnectError: 189 | self.state.transition_to(States.SUSPENDED) 190 | 191 | raise gen.Return(response) 192 | 193 | def set_heartbeat(self): 194 | timeout = self.timeout / HEARTBEAT_FREQUENCY 195 | 196 | io_loop = ioloop.IOLoop.current() 197 | 198 | if self.heartbeat_handle: 199 | io_loop.remove_timeout(self.heartbeat_handle) 200 | 201 | self.heartbeat_handle = io_loop.call_later(timeout, self.heartbeat) 202 | 203 | @gen.coroutine 204 | def heartbeat(self): 205 | if self.closing: 206 | return 207 | yield self.ensure_safe_state() 208 | 209 | try: 210 | zxid, _ = yield self.conn.send(protocol.PingRequest()) 211 | self.last_zxid = zxid 212 | except (exc.ConnectError, iostream.StreamClosedError): 213 | self.state.transition_to(States.SUSPENDED) 214 | finally: 215 | self.set_heartbeat() 216 | 217 | def add_watch_callback(self, event_type, path, callback): 218 | self.watch_callbacks[(event_type, path)].add(callback) 219 | 220 | def remove_watch_callback(self, event_type, path, callback): 221 | self.watch_callbacks[(event_type, path)].discard(callback) 222 | 223 | def event_dispatch(self, event): 224 | log.debug("Got watch event: %s", event) 225 | 226 | if event.type: 227 | key = (event.type, event.path) 228 | for callback in self.watch_callbacks[key]: 229 | ioloop.IOLoop.current().add_callback(callback, event.path) 230 | return 231 | 232 | if event.state == protocol.WatchEvent.DISCONNECTED: 233 | log.error("Got 'disconnected' watch event.") 234 | self.state.transition_to(States.LOST) 235 | elif event.state == protocol.WatchEvent.SESSION_EXPIRED: 236 | log.error("Got 'session expired' watch event.") 237 | self.state.transition_to(States.LOST) 238 | elif event.state == protocol.WatchEvent.AUTH_FAILED: 239 | log.error("Got 'auth failed' watch event.") 240 | self.state.transition_to(States.LOST) 241 | elif event.state == protocol.WatchEvent.CONNECTED_READ_ONLY: 242 | log.warn("Got 'connected read only' watch event.") 243 | self.state.transition_to(States.READ_ONLY) 244 | elif event.state == protocol.WatchEvent.SASL_AUTHENTICATED: 245 | log.info("Authentication successful.") 246 | elif event.state == protocol.WatchEvent.CONNECTED: 247 | log.info("Got 'connected' watch event.") 248 | self.state.transition_to(States.CONNECTED) 249 | 250 | @gen.coroutine 251 | def set_existing_watches(self): 252 | if not self.watch_callbacks: 253 | return 254 | 255 | request = protocol.SetWatchesRequest( 256 | relative_zxid=self.last_zxid or 0, 257 | data_watches=[], 258 | exist_watches=[], 259 | child_watches=[], 260 | ) 261 | 262 | for event_type, path in self.watch_callbacks.keys(): 263 | if event_type == protocol.WatchEvent.CREATED: 264 | request.exist_watches.append(path) 265 | if event_type == protocol.WatchEvent.DATA_CHANGED: 266 | request.data_watches.append(path) 267 | elif event_type == protocol.WatchEvent.CHILDREN_CHANGED: 268 | request.child_watches.append(path) 269 | 270 | yield self.send(request) 271 | 272 | @gen.coroutine 273 | def close(self): 274 | self.closing = True 275 | 276 | yield self.send(protocol.CloseRequest()) 277 | self.state.transition_to(States.LOST) 278 | 279 | yield self.conn.close(self.timeout) 280 | -------------------------------------------------------------------------------- /zoonado/states.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | import collections 4 | import logging 5 | 6 | from tornado import concurrent 7 | 8 | from .iterables import drain 9 | 10 | 11 | log = logging.getLogger(__name__) 12 | 13 | 14 | class States(object): 15 | 16 | CONNECTED = "connected" 17 | SUSPENDED = "suspended" 18 | READ_ONLY = "read_only" 19 | LOST = "lost" 20 | 21 | 22 | class SessionStateMachine(object): 23 | 24 | valid_transitions = set([ 25 | (States.LOST, States.CONNECTED), 26 | (States.LOST, States.READ_ONLY), 27 | (States.CONNECTED, States.SUSPENDED), 28 | (States.CONNECTED, States.LOST), 29 | (States.READ_ONLY, States.CONNECTED), 30 | (States.READ_ONLY, States.SUSPENDED), 31 | (States.READ_ONLY, States.LOST), 32 | (States.SUSPENDED, States.CONNECTED), 33 | (States.SUSPENDED, States.READ_ONLY), 34 | (States.SUSPENDED, States.LOST), 35 | ]) 36 | 37 | def __init__(self): 38 | self.current_state = States.LOST 39 | self.futures = collections.defaultdict(set) 40 | 41 | def transition_to(self, state): 42 | if (self.current_state, state) not in self.valid_transitions: 43 | raise RuntimeError( 44 | "Invalid session state transition: %s -> %s" % ( 45 | self.current_state, state 46 | ) 47 | ) 48 | 49 | log.debug("Session transition: %s -> %s", self.current_state, state) 50 | 51 | self.current_state = state 52 | 53 | for future in drain(self.futures[state]): 54 | if not future.done(): 55 | future.set_result(None) 56 | 57 | def wait_for(self, *states): 58 | f = concurrent.Future() 59 | 60 | if self.current_state in states: 61 | f.set_result(None) 62 | else: 63 | for state in states: 64 | self.futures[state].add(f) 65 | 66 | return f 67 | 68 | def __eq__(self, state): 69 | return self.current_state == state 70 | 71 | def __ne__(self, state): 72 | return self.current_state != state 73 | -------------------------------------------------------------------------------- /zoonado/transaction.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from tornado import gen 4 | 5 | from zoonado import protocol 6 | 7 | 8 | class Transaction(object): 9 | 10 | def __init__(self, client): 11 | self.client = client 12 | self.request = protocol.TransactionRequest() 13 | 14 | def check_version(self, path, version): 15 | path = self.client.normalize_path(path) 16 | 17 | self.request.add( 18 | protocol.CheckVersionRequest(path=path, version=version) 19 | ) 20 | 21 | def create( 22 | self, path, data=None, acl=None, 23 | ephemeral=False, sequential=False, container=False 24 | ): 25 | if container and not self.client.features.containers: 26 | raise ValueError("Cannot create container, feature unavailable.") 27 | 28 | path = self.client.normalize_path(path) 29 | acl = acl or self.client.default_acl 30 | 31 | if self.client.features.create_with_stat: 32 | request_class = protocol.Create2Request 33 | else: 34 | request_class = protocol.CreateRequest 35 | 36 | request = request_class(path=path, data=data, acl=acl) 37 | request.set_flags(ephemeral, sequential, container) 38 | 39 | self.request.add(request) 40 | 41 | def set_data(self, path, data, version=-1): 42 | path = self.client.normalize_path(path) 43 | 44 | self.request.add( 45 | protocol.SetDataRequest(path=path, data=data, version=version) 46 | ) 47 | 48 | def delete(self, path, version=-1): 49 | path = self.client.normalize_path(path) 50 | 51 | self.request.add( 52 | protocol.DeleteRequest(path=path, version=version) 53 | ) 54 | 55 | @gen.coroutine 56 | def commit(self): 57 | if not self.request.requests: 58 | raise ValueError("No operations to commit.") 59 | 60 | response = yield self.client.send(self.request) 61 | pairs = zip(self.request.requests, response.responses) 62 | 63 | result = Result() 64 | for request, reply in pairs: 65 | if isinstance(reply, protocol.CheckVersionResponse): 66 | result.checked.add(self.client.denormalize_path(request.path)) 67 | elif isinstance(reply, protocol.CreateResponse): 68 | result.created.add(self.client.denormalize_path(request.path)) 69 | elif isinstance(reply, protocol.SetDataResponse): 70 | result.updated.add(self.client.denormalize_path(request.path)) 71 | elif isinstance(reply, protocol.DeleteResponse): 72 | result.deleted.add(self.client.denormalize_path(request.path)) 73 | 74 | raise gen.Return(result) 75 | 76 | 77 | class Result(object): 78 | 79 | def __init__(self): 80 | self.checked = set() 81 | self.created = set() 82 | self.updated = set() 83 | self.deleted = set() 84 | 85 | def __bool__(self): 86 | return sum([ 87 | len(self.checked), 88 | len(self.created), 89 | len(self.updated), 90 | len(self.deleted), 91 | ]) > 0 92 | 93 | def __nonzero__(self): 94 | return self.__bool__() 95 | --------------------------------------------------------------------------------