├── src └── etcd │ ├── tests │ ├── integration │ │ ├── __init__.py │ │ ├── test_ssl.py │ │ ├── helpers.py │ │ └── test_simple.py │ ├── __init__.py │ ├── unit │ │ ├── __init__.py │ │ ├── test_result.py │ │ ├── test_client.py │ │ ├── test_lock.py │ │ ├── test_old_request.py │ │ └── test_request.py │ └── test_auth.py │ ├── lock.py │ ├── __init__.py │ ├── auth.py │ └── client.py ├── MANIFEST.in ├── .gitignore ├── download_etcd.sh ├── docs-source ├── api.rst ├── index.rst └── conf.py ├── black.toml ├── tox.ini ├── AUTHORS ├── .github └── workflows │ └── python-package.yaml ├── LICENSE.txt ├── setup.py ├── NEWS.txt └── README.rst /src/etcd/tests/integration/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/etcd/tests/__init__.py: -------------------------------------------------------------------------------- 1 | from . import unit 2 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include AUTHORS 2 | include LICENSE.txt 3 | include README.rst 4 | include NEWS.txt 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | 3 | .installed.cfg 4 | bin 5 | develop-eggs 6 | eggs 7 | .eggs 8 | .idea 9 | *.egg-info 10 | 11 | tmp 12 | build 13 | dist 14 | docs 15 | .coverage 16 | .venv 17 | .env 18 | -------------------------------------------------------------------------------- /download_etcd.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | VERSION=${1:-2.3.7} 4 | mkdir -p bin 5 | URL="https://github.com/coreos/etcd/releases/download/v${VERSION}/etcd-v${VERSION}-linux-amd64.tar.gz" 6 | curl -L $URL | tar -C ./bin --strip-components=1 -xzvf - "etcd-v${VERSION}-linux-amd64/etcd" 7 | mv bin/etcd /usr/local/bin/ 8 | -------------------------------------------------------------------------------- /docs-source/api.rst: -------------------------------------------------------------------------------- 1 | API Documentation 2 | ========================= 3 | .. automodule:: etcd 4 | :members: 5 | .. autoclass:: Client 6 | :special-members: 7 | :members: 8 | :exclude-members: __weakref__ 9 | .. autoclass:: Lock 10 | :special-members: 11 | :members: 12 | :exclude-members: __weakref__ 13 | -------------------------------------------------------------------------------- /black.toml: -------------------------------------------------------------------------------- 1 | [tool.black] 2 | line-length = 100 3 | target-version = ['py37'] 4 | include = '\.pyi?$' 5 | exclude = ''' 6 | ( 7 | \.eggs # exclude a few common directories in the 8 | | \.git # root of the project 9 | | \.hg 10 | | \.mypy_cache 11 | | \.tox 12 | | \.venv 13 | | venv 14 | | _build 15 | | buck-out 16 | | build 17 | | dist 18 | ) 19 | ''' 20 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | minversion = 2.5.0 3 | envlist = py{3,37}-{style,unit} 4 | skip_missing_interpreters = True 5 | 6 | [testenv] 7 | usedevelop = True 8 | basepython = 9 | py3: python3 10 | py37: python3.7 11 | description = 12 | style: Style consistency checker 13 | unit: Run unit tests. 14 | py3: (Python 3.x) 15 | py37: (Python 3.7) 16 | 17 | commands = 18 | ; style: flake8 19 | style: black --config black.toml --check src 20 | unit: pytest --cov=etcd src/etcd/tests/ --cov-report=term-missing 21 | 22 | deps = 23 | style: flake8 24 | style: black 25 | unit: pytest-cov 26 | unit: pyOpenSSL>=0.14 27 | 28 | [flake8] 29 | max-line-length = 100 30 | statistics = True 31 | exclude = .venv,.eggs,.tox,build,venv 32 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Maintainers: 2 | ----------- 3 | Jose Plana (jplana) 4 | Giuseppe Lavagetto (lavagetto) 5 | Shaun Crampton (fasaxc) 6 | 7 | Contributors: 8 | ------------ 9 | Aleksandar Veselinovic 10 | Alexander Brand 11 | Alexander Kukushkin 12 | Alex Chan 13 | Alex Ianchici 14 | Ainlolcat 15 | Bartlomiej Biernacki 16 | Bradley Cicenas 17 | Christoph Heer 18 | Gigi Sayfan 19 | Hogenmiller 20 | Huangdong 21 | Jimmy Zelinskie 22 | Jim Rollenhagen 23 | John Kristensen 24 | Joshua Conner 25 | Lars Bahner 26 | Matthew Barnes 27 | Matthias Urlichs 28 | Michal Witkowski 29 | Mike Place 30 | Nick Bartos 31 | Mingqing 32 | Peter Wagner 33 | Realityone 34 | Roberto Aguilar 35 | Roy Smith 36 | Ryan Fowler 37 | Samuel Marks 38 | Sergio Castaño Arteaga 39 | Shaun Crampton 40 | Sigmund Augdal 41 | Simeon Visser 42 | Simon Gomizelj 43 | SkyLothar 44 | Spike Curtis 45 | Stephen Milner 46 | Taylor McKinnon 47 | Tobe 48 | Tomas Kral 49 | Tom Denham 50 | Toshiya Kawasaki 51 | WillPlatnick 52 | Weizheng Xu 53 | WooParadog 54 | Wei Tie 55 | -------------------------------------------------------------------------------- /src/etcd/tests/unit/__init__.py: -------------------------------------------------------------------------------- 1 | import etcd 2 | import unittest 3 | import urllib3 4 | import json 5 | 6 | try: 7 | import mock 8 | except ImportError: 9 | from unittest import mock 10 | 11 | 12 | class TestClientApiBase(unittest.TestCase): 13 | def setUp(self): 14 | self.client = etcd.Client() 15 | 16 | def _prepare_response(self, s, d, cluster_id=None): 17 | if isinstance(d, dict): 18 | data = json.dumps(d).encode("utf-8") 19 | else: 20 | data = d.encode("utf-8") 21 | 22 | r = mock.create_autospec(urllib3.response.HTTPResponse)() 23 | r.status = s 24 | r.data = data 25 | r.getheader.return_value = cluster_id or "abcd1234" 26 | return r 27 | 28 | def _mock_api(self, status, d, cluster_id=None): 29 | resp = self._prepare_response(status, d, cluster_id=cluster_id) 30 | self.client.api_execute = mock.MagicMock(return_value=resp) 31 | 32 | def _mock_exception(self, exc, msg): 33 | self.client.api_execute = mock.Mock(side_effect=exc(msg)) 34 | -------------------------------------------------------------------------------- /.github/workflows/python-package.yaml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a variety of Python versions 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python 3 | 4 | name: Python package 5 | 6 | on: 7 | push: 8 | branches: [ "master" ] 9 | pull_request: 10 | branches: [ "master" ] 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | strategy: 17 | fail-fast: false 18 | matrix: 19 | python-version: ["3.9", "3.10", "3.11"] 20 | 21 | steps: 22 | - uses: actions/checkout@v3 23 | - name: Set up Python ${{ matrix.python-version }} 24 | uses: actions/setup-python@v3 25 | with: 26 | python-version: ${{ matrix.python-version }} 27 | - name: Install dependencies 28 | run: | 29 | ./download_etcd.sh 3.4.0 30 | python -m pip install --upgrade pip 31 | python -m pip install tox coveralls 32 | - name: Test with tox 33 | env: 34 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 35 | run: | 36 | tox 37 | coveralls 38 | 39 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2013,2014,2015 Jose Plana Mario, Giuseppe Lavagetto 4 | Modifications, Copyright (c) 2015 Metaswitch Networks Limited 5 | Modifications, Copyright (c) 2015 The Wikimedia Foundation, Inc. 6 | 7 | Permission is hereby granted, free of charge, to any person obtaining a copy of 8 | this software and associated documentation files (the "Software"), to deal in 9 | the Software without restriction, including without limitation the rights to 10 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 11 | the Software, and to permit persons to whom the Software is furnished to do so, 12 | subject to the following conditions: 13 | 14 | The above copyright notice and this permission notice shall be included in all 15 | copies or substantial portions of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 19 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 20 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 21 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 22 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | import sys, os 3 | 4 | here = os.path.abspath(os.path.dirname(__file__)) 5 | README = open(os.path.join(here, "README.rst")).read() 6 | NEWS = open(os.path.join(here, "NEWS.txt")).read() 7 | 8 | 9 | version = "0.5.0" 10 | 11 | install_requires = ["urllib3>=1.7.1", "dnspython>=1.13.0"] 12 | 13 | test_requires = ["mock", "pytest", "pyOpenSSL>=0.14"] 14 | 15 | setup( 16 | name="python-etcd", 17 | version=version, 18 | description="A python client for etcd", 19 | long_description=README + "\n\n" + NEWS, 20 | classifiers=[ 21 | "Topic :: System :: Distributed Computing", 22 | "Topic :: Software Development :: Libraries", 23 | "License :: OSI Approved :: MIT License", 24 | "Programming Language :: Python :: 3", 25 | "Topic :: Database :: Front-Ends", 26 | ], 27 | keywords="etcd raft distributed log api client", 28 | author="Jose Plana", 29 | author_email="jplana@gmail.com", 30 | url="http://github.com/jplana/python-etcd", 31 | license="MIT", 32 | packages=find_packages("src"), 33 | package_dir={"": "src"}, 34 | include_package_data=True, 35 | zip_safe=False, 36 | install_requires=install_requires, 37 | tests_require=test_requires, 38 | test_suite="nose.collector", 39 | ) 40 | -------------------------------------------------------------------------------- /NEWS.txt: -------------------------------------------------------------------------------- 1 | News 2 | ==== 3 | 0.5.0 4 | ----- 5 | *Release date: 31-Oct-2023 6 | 7 | * Drop python 2.x compatibility (should still work) 8 | * Move to use pytest 9 | * Support urllib3 v2, including support of self-signed certs 10 | * Fix version check to avoid crashes with non-official releases 11 | * Correctly handle watch timeouts in lock 12 | * Allow trying more than one domain when looking up SRV records 13 | * Support auth API both <= 2.2.5 and >= 2.3.0 14 | * Use github actions instead than travis 15 | 16 | 0.4.5 17 | ----- 18 | *Release date: 3-Mar-2017* 19 | 20 | * Remove dnspython2/3 requirement 21 | * Change property name setter in lock 22 | * Fixed acl tests 23 | * Added version/cluster_version properties to client 24 | * Fixes in lock when used as context manager 25 | * Fixed improper usage of urllib3 exceptions 26 | * Minor fixes for error classes 27 | * In lock return modifiedIndex to watch changes 28 | * In lock fix context manager exception handling 29 | * Improvments to the documentation 30 | * Remove _base_uri only after refresh from cluster 31 | * Avoid double update of _machines_cache 32 | 33 | 34 | 0.4.4 35 | ----- 36 | *Release date: 10-Jan-2017* 37 | 38 | * Fix some tests 39 | * Use sys,version_info tuple, instead of named tuple 40 | * Improve & fix documentation 41 | * Fix python3 specific problem when blocking on contented lock 42 | * Add refresh key method 43 | * Add custom lock prefix support 44 | 45 | 46 | 0.4.3 47 | ----- 48 | *Release date: 14-Dec-2015* 49 | 50 | * Fix check for parameters in case of connection error 51 | * Python 3.5 compatibility and general python3 cleanups 52 | * Added authentication and module for managing ACLs 53 | * Added srv record-based DNS discovery 54 | * Fixed (again) logging of cluster id changes 55 | * Fixed leader lookup 56 | * Properly retry request on exception 57 | * Client: clean up open connections when deleting 58 | 59 | 0.4.2 60 | ----- 61 | *Release date: 8-Oct-2015* 62 | 63 | * Fixed lock documentation 64 | * Fixed lock sequences due to etcd 2.2 change 65 | * Better exception management during response processing 66 | * Fixed logging of cluster ID changes 67 | * Fixed subtree results 68 | * Do not check cluster ID if etcd responses don't contain the ID 69 | * Added a cause to EtcdConnectionFailed 70 | 71 | 72 | 0.4.1 73 | ----- 74 | *Release date: 1-Aug-2015* 75 | 76 | * Added client-side leader election 77 | * Added stats endpoints 78 | * Added logging 79 | * Better exception handling 80 | * Check for cluster ID on each request 81 | * Added etcd.Client.members and fixed etcd.Client.leader 82 | * Removed locking and election etcd support 83 | * Allow the use of etcd proxies with reconnections 84 | * Implement pop: Remove key from etc and return the corresponding value. 85 | * Eternal watcher can be now recursive 86 | * Fix etcd.Client machines 87 | * Do not send parameters with `None` value to etcd 88 | * Support ttl=0 in write. 89 | * Moved pyOpenSSL into test requirements. 90 | * Always set certificate information so redirects from http to https work. 91 | 92 | 93 | 0.3.3 94 | ----- 95 | *Release date: 12-Apr-2015* 96 | 97 | * Forward leaves_only value in get_subtree() recursive calls 98 | * Fix README prevExists->prevExist 99 | * Added configurable version_prefix 100 | * Added support for recursive watch 101 | * Better error handling support (more detailed exceptions) 102 | * Fixed some unreliable tests 103 | 104 | 105 | 0.3.2 106 | ----- 107 | 108 | *Release date: 4-Aug-2014* 109 | 110 | * Fixed generated documentation version. 111 | 112 | 113 | 0.3.1 114 | ----- 115 | 116 | *Release date: 4-Aug-2014* 117 | 118 | * Added consisten read option 119 | * Fixed timeout parameter in read() 120 | * Added atomic delete parameter support 121 | * Fixed delete behaviour 122 | * Added update method that allows atomic updated on results 123 | * Fixed checks on write() 124 | * Added leaves generator to EtcdResult and get_subtree for recursive fetch 125 | * Added etcd_index to EtcdResult 126 | * Changed ethernal -> eternal 127 | * Updated urllib3 & pyOpenSSL libraries 128 | * Several performance fixes 129 | * Better parsing of etcd_index and raft_index 130 | * Removed duplicated tests 131 | * Added several integration and unit tests 132 | * Use etcd v0.3.0 in travis 133 | * Execute test using `python setup.py test` and nose 134 | 135 | 136 | 0.3.0 137 | ----- 138 | 139 | *Release date: 18-Jan-2014* 140 | 141 | * API v2 support 142 | * Python 3.3 compatibility 143 | 144 | 145 | 0.2.1 146 | ----- 147 | 148 | *Release data: 30-Nov-2013* 149 | 150 | * SSL support 151 | * Added support for subdirectories in results. 152 | * Improve test 153 | * Added support for reconnections, allowing death node tolerance. 154 | 155 | 156 | 0.2.0 157 | ----- 158 | 159 | *Release date: 30-Sep-2013* 160 | 161 | * Allow fetching of multiple keys (sub-nodes) 162 | 163 | 164 | 0.1 165 | --- 166 | 167 | *Release date: 18-Sep-2013* 168 | 169 | * Initial release 170 | -------------------------------------------------------------------------------- /src/etcd/tests/unit/test_result.py: -------------------------------------------------------------------------------- 1 | import etcd 2 | import unittest 3 | import json 4 | import urllib3 5 | 6 | try: 7 | import mock 8 | except ImportError: 9 | from unittest import mock 10 | 11 | 12 | class TestEtcdResult(unittest.TestCase): 13 | def test_get_subtree_1_level(self): 14 | """ 15 | Test get_subtree() for a read with tree 1 level deep. 16 | """ 17 | response = { 18 | "node": { 19 | "key": "/test", 20 | "value": "hello", 21 | "expiration": None, 22 | "ttl": None, 23 | "modifiedIndex": 5, 24 | "createdIndex": 1, 25 | "newKey": False, 26 | "dir": False, 27 | } 28 | } 29 | result = etcd.EtcdResult(**response) 30 | self.assertEqual(result.key, response["node"]["key"]) 31 | self.assertEqual(result.value, response["node"]["value"]) 32 | 33 | # Get subtree returns itself, whether or not leaves_only 34 | subtree = list(result.get_subtree(leaves_only=True)) 35 | self.assertListEqual([result], subtree) 36 | subtree = list(result.get_subtree(leaves_only=False)) 37 | self.assertListEqual([result], subtree) 38 | 39 | def test_get_subtree_2_level(self): 40 | """ 41 | Test get_subtree() for a read with tree 2 levels deep. 42 | """ 43 | leaf0 = { 44 | "key": "/test/leaf0", 45 | "value": "hello1", 46 | "expiration": None, 47 | "ttl": None, 48 | "modifiedIndex": 5, 49 | "createdIndex": 1, 50 | "newKey": False, 51 | "dir": False, 52 | } 53 | leaf1 = { 54 | "key": "/test/leaf1", 55 | "value": "hello2", 56 | "expiration": None, 57 | "ttl": None, 58 | "modifiedIndex": 6, 59 | "createdIndex": 2, 60 | "newKey": False, 61 | "dir": False, 62 | } 63 | testnode = { 64 | "node": { 65 | "key": "/test/", 66 | "expiration": None, 67 | "ttl": None, 68 | "modifiedIndex": 6, 69 | "createdIndex": 2, 70 | "newKey": False, 71 | "dir": True, 72 | "nodes": [leaf0, leaf1], 73 | } 74 | } 75 | result = etcd.EtcdResult(**testnode) 76 | self.assertEqual(result.key, "/test/") 77 | self.assertTrue(result.dir) 78 | 79 | # Get subtree returns just two leaves for leaves only. 80 | subtree = list(result.get_subtree(leaves_only=True)) 81 | self.assertEqual(subtree[0].key, "/test/leaf0") 82 | self.assertEqual(subtree[1].key, "/test/leaf1") 83 | self.assertEqual(len(subtree), 2) 84 | 85 | # Get subtree returns leaves and directory. 86 | subtree = list(result.get_subtree(leaves_only=False)) 87 | self.assertEqual(subtree[0].key, "/test/") 88 | self.assertEqual(subtree[1].key, "/test/leaf0") 89 | self.assertEqual(subtree[2].key, "/test/leaf1") 90 | self.assertEqual(len(subtree), 3) 91 | 92 | def test_get_subtree_3_level(self): 93 | """ 94 | Test get_subtree() for a read with tree 3 levels deep. 95 | """ 96 | leaf0 = { 97 | "key": "/test/mid0/leaf0", 98 | "value": "hello1", 99 | } 100 | leaf1 = { 101 | "key": "/test/mid0/leaf1", 102 | "value": "hello2", 103 | } 104 | leaf2 = { 105 | "key": "/test/mid1/leaf2", 106 | "value": "hello1", 107 | } 108 | leaf3 = { 109 | "key": "/test/mid1/leaf3", 110 | "value": "hello2", 111 | } 112 | mid0 = {"key": "/test/mid0/", "dir": True, "nodes": [leaf0, leaf1]} 113 | mid1 = {"key": "/test/mid1/", "dir": True, "nodes": [leaf2, leaf3]} 114 | testnode = {"node": {"key": "/test/", "dir": True, "nodes": [mid0, mid1]}} 115 | result = etcd.EtcdResult(**testnode) 116 | self.assertEqual(result.key, "/test/") 117 | self.assertTrue(result.dir) 118 | 119 | # Get subtree returns just two leaves for leaves only. 120 | subtree = list(result.get_subtree(leaves_only=True)) 121 | self.assertEqual(subtree[0].key, "/test/mid0/leaf0") 122 | self.assertEqual(subtree[1].key, "/test/mid0/leaf1") 123 | self.assertEqual(subtree[2].key, "/test/mid1/leaf2") 124 | self.assertEqual(subtree[3].key, "/test/mid1/leaf3") 125 | self.assertEqual(len(subtree), 4) 126 | 127 | # Get subtree returns leaves and directory. 128 | subtree = list(result.get_subtree(leaves_only=False)) 129 | self.assertEqual(subtree[0].key, "/test/") 130 | self.assertEqual(subtree[1].key, "/test/mid0/") 131 | self.assertEqual(subtree[2].key, "/test/mid0/leaf0") 132 | self.assertEqual(subtree[3].key, "/test/mid0/leaf1") 133 | self.assertEqual(subtree[4].key, "/test/mid1/") 134 | self.assertEqual(subtree[5].key, "/test/mid1/leaf2") 135 | self.assertEqual(subtree[6].key, "/test/mid1/leaf3") 136 | self.assertEqual(len(subtree), 7) 137 | -------------------------------------------------------------------------------- /docs-source/index.rst: -------------------------------------------------------------------------------- 1 | Python-etcd documentation 2 | ========================= 3 | 4 | A python client for Etcd https://github.com/coreos/etcd 5 | 6 | 7 | 8 | 9 | Installation 10 | ------------ 11 | 12 | Pre-requirements 13 | ................ 14 | 15 | Install etcd 16 | 17 | 18 | From source 19 | ........... 20 | 21 | .. code-block:: bash 22 | 23 | $ python setup.py install 24 | 25 | 26 | Usage 27 | ----- 28 | 29 | Create a client object 30 | ...................... 31 | 32 | .. code-block:: python 33 | 34 | import etcd 35 | 36 | client = etcd.Client() # this will create a client against etcd server running on localhost on port 4001 37 | client = etcd.Client(port=4002) 38 | client = etcd.Client(host='127.0.0.1', port=4003) 39 | client = etcd.Client(host='127.0.0.1', port=4003, allow_redirect=False) # wont let you run sensitive commands on non-leader machines, default is true 40 | client = etcd.Client( 41 | host='127.0.0.1', 42 | port=4003, 43 | allow_reconnect=True, 44 | protocol='https',) 45 | 46 | Set a key 47 | ......... 48 | 49 | .. code-block:: python 50 | 51 | client.write('/nodes/n1', 1) 52 | # with ttl 53 | client.write('/nodes/n2', 2, ttl=4) # sets the ttl to 4 seconds 54 | # create only 55 | client.write('/nodes/n3', 'test', prevExist=False) 56 | # Compare and swap values atomically 57 | client.write('/nodes/n3', 'test2', prevValue='test1') #this fails to write 58 | client.write('/nodes/n3', 'test2', prevIndex=10) #this fails to write 59 | # mkdir 60 | client.write('/nodes/queue', None, dir=True) 61 | # Append a value to a queue dir 62 | client.write('/nodes/queue', 'test', append=True) #will write i.e. /nodes/queue/11 63 | client.write('/nodes/queue', 'test2', append=True) #will write i.e. /nodes/queue/12 64 | 65 | You can also atomically update a result: 66 | 67 | .. code:: python 68 | 69 | result = client.read('/foo') 70 | print(result.value) # bar 71 | result.value += u'bar' 72 | updated = client.update(result) # if any other client wrote '/foo' in the meantime this will fail 73 | print(updated.value) # barbar 74 | 75 | 76 | 77 | Get a key 78 | ......... 79 | 80 | .. code-block:: python 81 | 82 | client.read('/nodes/n2').value 83 | #recursively read a directory 84 | r = client.read('/nodes', recursive=True, sorted=True) 85 | for child in r.children: 86 | print("%s: %s" % (child.key,child.value)) 87 | 88 | client.read('/nodes/n2', wait=True) #Waits for a change in value in the key before returning. 89 | client.read('/nodes/n2', wait=True, waitIndex=10) 90 | 91 | # raises etcd.EtcdKeyNotFound when key not found 92 | try: 93 | client.read('/invalid/path') 94 | except etcd.EtcdKeyNotFound: 95 | # do something 96 | print "error" 97 | 98 | 99 | Delete a key 100 | ............ 101 | 102 | .. code-block:: python 103 | 104 | client.delete('/nodes/n1') 105 | client.delete('/nodes', dir=True) #spits an error if dir is not empty 106 | client.delete('/nodes', recursive=True) #this works recursively 107 | 108 | Locking module 109 | ~~~~~~~~~~~~~~ 110 | 111 | .. code:: python 112 | 113 | # Initialize the lock object: 114 | # NOTE: this does not acquire a lock yet 115 | client = etcd.Client() 116 | lock = etcd.Lock(client, 'my_lock_name') 117 | 118 | # Use the lock object: 119 | lock.acquire(blocking=True, # will block until the lock is acquired 120 | lock_ttl=None) # lock will live until we release it 121 | lock.is_acquired # True 122 | lock.acquire(lock_ttl=60) # renew a lock 123 | lock.release() # release an existing lock 124 | lock.is_acquired # False 125 | 126 | # The lock object may also be used as a context manager: 127 | client = etcd.Client() 128 | with etcd.Lock(client, 'customer1') as my_lock: 129 | do_stuff() 130 | my_lock.is_acquired # True 131 | my_lock.acquire(lock_ttl=60) 132 | my_lock.is_acquired # False 133 | 134 | 135 | Get machines in the cluster 136 | ........................... 137 | 138 | .. code-block:: python 139 | 140 | client.machines 141 | 142 | 143 | Get leader of the cluster 144 | ......................... 145 | 146 | .. code-block:: python 147 | 148 | client.leader 149 | 150 | 151 | 152 | 153 | Development setup 154 | ----------------- 155 | 156 | To create a buildout, 157 | 158 | .. code-block:: bash 159 | 160 | $ python bootstrap.py 161 | $ bin/buildout 162 | 163 | 164 | to test you should have etcd available in your system path: 165 | 166 | .. code-block:: bash 167 | 168 | $ bin/test 169 | 170 | to generate documentation, 171 | 172 | .. code-block:: bash 173 | 174 | $ cd docs 175 | $ make 176 | 177 | 178 | 179 | Release HOWTO 180 | ------------- 181 | 182 | To make a release, 183 | 184 | 1) Update release date/version in NEWS.txt and setup.py 185 | 2) Run 'python setup.py sdist' 186 | 3) Test the generated source distribution in dist/ 187 | 4) Upload to PyPI: 'python setup.py sdist register upload' 188 | 5) Increase version in setup.py (for next release) 189 | 190 | 191 | List of contributors at https://github.com/jplana/python-etcd/graphs/contributors 192 | 193 | Code documentation 194 | ------------------ 195 | 196 | .. toctree:: 197 | :maxdepth: 2 198 | 199 | api.rst 200 | -------------------------------------------------------------------------------- /src/etcd/tests/test_auth.py: -------------------------------------------------------------------------------- 1 | from etcd.tests.integration.test_simple import EtcdIntegrationTest 2 | from etcd import auth 3 | import etcd 4 | 5 | 6 | class TestEtcdAuthBase(EtcdIntegrationTest): 7 | cl_size = 1 8 | 9 | def setUp(self): 10 | # Sets up the root user, toggles auth 11 | u = auth.EtcdUser(self.client, "root") 12 | u.password = "testpass" 13 | u.write() 14 | self.client = etcd.Client(port=6001, username="root", password="testpass") 15 | self.unauth_client = etcd.Client(port=6001) 16 | a = auth.Auth(self.client) 17 | a.active = True 18 | 19 | def tearDown(self): 20 | u = auth.EtcdUser(self.client, "test_user") 21 | r = auth.EtcdRole(self.client, "test_role") 22 | try: 23 | u.delete() 24 | except: 25 | pass 26 | try: 27 | r.delete() 28 | except: 29 | pass 30 | a = auth.Auth(self.client) 31 | a.active = False 32 | 33 | 34 | class EtcdUserTest(TestEtcdAuthBase): 35 | def test_names(self): 36 | u = auth.EtcdUser(self.client, "test_user") 37 | self.assertEquals(u.names, ["root"]) 38 | 39 | def test_read(self): 40 | u = auth.EtcdUser(self.client, "root") 41 | # Reading an existing user succeeds 42 | try: 43 | u.read() 44 | except Exception: 45 | self.fail("reading the root user raised an exception") 46 | 47 | # roles for said user are fetched 48 | self.assertEquals(u.roles, set(["root"])) 49 | 50 | # The user is correctly rendered out 51 | self.assertEquals(u._to_net(), [{"user": "root", "password": None, "roles": ["root"]}]) 52 | 53 | # An inexistent user raises the appropriate exception 54 | u = auth.EtcdUser(self.client, "user.does.not.exist") 55 | self.assertRaises(etcd.EtcdKeyNotFound, u.read) 56 | 57 | # Reading with an unauthenticated client raises an exception 58 | u = auth.EtcdUser(self.unauth_client, "root") 59 | self.assertRaises(etcd.EtcdInsufficientPermissions, u.read) 60 | 61 | # Generic errors are caught 62 | c = etcd.Client(port=9999) 63 | u = auth.EtcdUser(c, "root") 64 | self.assertRaises(etcd.EtcdException, u.read) 65 | 66 | def test_write_and_delete(self): 67 | # Create an user 68 | u = auth.EtcdUser(self.client, "test_user") 69 | u.roles.add("guest") 70 | u.roles.add("root") 71 | # directly from my suitcase 72 | u.password = "123456" 73 | try: 74 | u.write() 75 | except: 76 | self.fail("creating a user doesn't work") 77 | # Password gets wiped 78 | self.assertEquals(u.password, None) 79 | u.read() 80 | # Verify we can log in as this user and access the auth (it has the 81 | # root role) 82 | cl = etcd.Client(port=6001, username="test_user", password="123456") 83 | ul = auth.EtcdUser(cl, "root") 84 | try: 85 | ul.read() 86 | except etcd.EtcdInsufficientPermissions: 87 | self.fail("Reading auth with the new user is not possible") 88 | 89 | self.assertEquals(u.name, "test_user") 90 | self.assertEquals(u.roles, set(["guest", "root"])) 91 | # set roles as a list, it works! 92 | u.roles = ["guest", "test_group"] 93 | # We need this or the new API will return an internal error 94 | r = auth.EtcdRole(self.client, "test_group") 95 | r.acls = {"*": "R", "/test/*": "RW"} 96 | r.write() 97 | try: 98 | u.write() 99 | except: 100 | self.fail("updating a user you previously created fails") 101 | u.read() 102 | self.assertIn("test_group", u.roles) 103 | 104 | # Unauthorized access is properly handled 105 | ua = auth.EtcdUser(self.unauth_client, "test_user") 106 | self.assertRaises(etcd.EtcdInsufficientPermissions, ua.write) 107 | 108 | # now let's test deletion 109 | du = auth.EtcdUser(self.client, "user.does.not.exist") 110 | self.assertRaises(etcd.EtcdKeyNotFound, du.delete) 111 | 112 | # Delete test_user 113 | u.delete() 114 | self.assertRaises(etcd.EtcdKeyNotFound, u.read) 115 | # Permissions are properly handled 116 | self.assertRaises(etcd.EtcdInsufficientPermissions, ua.delete) 117 | 118 | 119 | class EtcdRoleTest(TestEtcdAuthBase): 120 | def test_names(self): 121 | r = auth.EtcdRole(self.client, "guest") 122 | self.assertListEqual(r.names, ["guest", "root"]) 123 | 124 | def test_read(self): 125 | r = auth.EtcdRole(self.client, "guest") 126 | try: 127 | r.read() 128 | except: 129 | self.fail("Reading an existing role failed") 130 | 131 | # XXX The ACL path result changed from '*' to '/*' at some point 132 | # between etcd-2.2.2 and 2.2.5. They're equivalent so allow 133 | # for both. 134 | if "/*" in r.acls: 135 | self.assertEquals(r.acls, {"/*": "RW"}) 136 | else: 137 | self.assertEquals(r.acls, {"*": "RW"}) 138 | # We can actually skip most other read tests as they are common 139 | # with EtcdUser 140 | 141 | def test_write_and_delete(self): 142 | r = auth.EtcdRole(self.client, "test_role") 143 | r.acls = {"*": "R", "/test/*": "RW"} 144 | try: 145 | r.write() 146 | except: 147 | self.fail("Writing a simple groups should not fail") 148 | 149 | r1 = auth.EtcdRole(self.client, "test_role") 150 | r1.read() 151 | self.assertEquals(r1.acls, r.acls) 152 | r.revoke("/test/*", "W") 153 | r.write() 154 | r1.read() 155 | self.assertEquals(r1.acls, {"*": "R", "/test/*": "R"}) 156 | r.grant("/pub/*", "RW") 157 | r.write() 158 | r1.read() 159 | self.assertEquals(r1.acls["/pub/*"], "RW") 160 | # All other exceptions are tested by the user tests 161 | r1.name = None 162 | self.assertRaises(etcd.EtcdException, r1.write) 163 | # ditto for delete 164 | try: 165 | r.delete() 166 | except: 167 | self.fail("A normal delete should not fail") 168 | self.assertRaises(etcd.EtcdKeyNotFound, r.read) 169 | -------------------------------------------------------------------------------- /src/etcd/tests/integration/test_ssl.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import shutil 4 | import logging 5 | import unittest 6 | import multiprocessing 7 | import tempfile 8 | 9 | import pytest 10 | import urllib3 11 | 12 | import etcd 13 | from . import helpers 14 | from . import test_simple 15 | 16 | log = logging.getLogger() 17 | 18 | 19 | class TestEncryptedAccess(test_simple.EtcdIntegrationTest): 20 | @classmethod 21 | def setUpClass(cls): 22 | program = cls._get_exe() 23 | cls.directory = tempfile.mkdtemp(prefix="python-etcd") 24 | 25 | cls.ca_cert_path = os.path.join(cls.directory, "ca.crt") 26 | ca_key_path = os.path.join(cls.directory, "ca.key") 27 | 28 | cls.ca2_cert_path = os.path.join(cls.directory, "ca2.crt") 29 | ca2_key_path = os.path.join(cls.directory, "ca2.key") 30 | 31 | server_cert_path = os.path.join(cls.directory, "server.crt") 32 | server_key_path = os.path.join(cls.directory, "server.key") 33 | 34 | ca, ca_key = helpers.TestingCA.create_test_ca_certificate( 35 | cls.ca_cert_path, ca_key_path, "TESTCA" 36 | ) 37 | 38 | ca2, ca2_key = helpers.TestingCA.create_test_ca_certificate( 39 | cls.ca2_cert_path, ca2_key_path, "TESTCA2" 40 | ) 41 | 42 | helpers.TestingCA.create_test_certificate( 43 | ca, ca_key, server_cert_path, server_key_path, "127.0.0.1" 44 | ) 45 | 46 | cls.processHelper = helpers.EtcdProcessHelper( 47 | cls.directory, 48 | proc_name=program, 49 | port_range_start=6001, 50 | internal_port_range_start=8001, 51 | tls=True, 52 | ) 53 | 54 | cls.processHelper.run( 55 | number=3, 56 | proc_args=[ 57 | "-cert-file=%s" % server_cert_path, 58 | "-key-file=%s" % server_key_path, 59 | ], 60 | ) 61 | 62 | def test_get_set_unauthenticated(self): 63 | """INTEGRATION: set/get a new value unauthenticated (http->https)""" 64 | 65 | client = etcd.Client(port=6001) 66 | 67 | # Since python 3 raises a MaxRetryError here, this gets caught in 68 | # different code blocks in python 2 and python 3, thus messages are 69 | # different. Python 3 does the right thing(TM), for the record 70 | self.assertRaises(etcd.EtcdException, client.set, "/test_set", "test-key") 71 | 72 | self.assertRaises(etcd.EtcdException, client.get, "/test_set") 73 | 74 | def test_get_set_unauthenticated_missing_ca(self): 75 | """INTEGRATION: try unauthenticated w/out validation (https->https)""" 76 | # This doesn't work for now and will need further inspection 77 | client = etcd.Client(protocol="https", port=6001) 78 | set_result = client.set("/test_set", "test-key") 79 | get_result = client.get("/test_set") 80 | 81 | def test_get_set_unauthenticated_with_ca(self): 82 | """INTEGRATION: try unauthenticated with validation (https->https)""" 83 | client = etcd.Client(protocol="https", port=6001, ca_cert=self.ca2_cert_path) 84 | 85 | self.assertRaises(etcd.EtcdConnectionFailed, client.set, "/test-set", "test-key") 86 | self.assertRaises(etcd.EtcdConnectionFailed, client.get, "/test-set") 87 | 88 | def test_get_set_authenticated(self): 89 | """INTEGRATION: set/get a new value authenticated""" 90 | 91 | client = etcd.Client(port=6001, protocol="https") 92 | 93 | set_result = client.set("/test_set", "test-key") 94 | get_result = client.get("/test_set") 95 | 96 | 97 | class TestClientAuthenticatedAccess(test_simple.EtcdIntegrationTest): 98 | @classmethod 99 | def setUpClass(cls): 100 | program = cls._get_exe() 101 | cls.directory = tempfile.mkdtemp(prefix="python-etcd") 102 | 103 | cls.ca_cert_path = os.path.join(cls.directory, "ca.crt") 104 | ca_key_path = os.path.join(cls.directory, "ca.key") 105 | 106 | server_cert_path = os.path.join(cls.directory, "server.crt") 107 | server_key_path = os.path.join(cls.directory, "server.key") 108 | 109 | cls.client_cert_path = os.path.join(cls.directory, "client.crt") 110 | cls.client_key_path = os.path.join(cls.directory, "client.key") 111 | 112 | cls.client_all_cert = os.path.join(cls.directory, "client-all.crt") 113 | 114 | ca, ca_key = helpers.TestingCA.create_test_ca_certificate(cls.ca_cert_path, ca_key_path) 115 | 116 | helpers.TestingCA.create_test_certificate( 117 | ca, ca_key, server_cert_path, server_key_path, "127.0.0.1" 118 | ) 119 | 120 | helpers.TestingCA.create_test_certificate( 121 | ca, ca_key, cls.client_cert_path, cls.client_key_path 122 | ) 123 | 124 | cls.processHelper = helpers.EtcdProcessHelper( 125 | cls.directory, 126 | proc_name=program, 127 | port_range_start=6001, 128 | internal_port_range_start=8001, 129 | tls=True, 130 | ) 131 | 132 | with open(cls.client_all_cert, "w") as f: 133 | with open(cls.client_key_path, "r") as g: 134 | f.write(g.read()) 135 | with open(cls.client_cert_path, "r") as g: 136 | f.write(g.read()) 137 | 138 | cls.processHelper.run( 139 | number=3, 140 | proc_args=[ 141 | "-cert-file=%s" % server_cert_path, 142 | "-key-file=%s" % server_key_path, 143 | "-trusted-ca-file", 144 | cls.ca_cert_path, 145 | ], 146 | ) 147 | 148 | def test_get_set_unauthenticated(self): 149 | """INTEGRATION: set/get a new value unauthenticated (http->https)""" 150 | 151 | client = etcd.Client(port=6001) 152 | 153 | # See above for the reason of this change 154 | self.assertRaises(etcd.EtcdException, client.set, "/test_set", "test-key") 155 | self.assertRaises(etcd.EtcdException, client.get, "/test_set") 156 | 157 | @pytest.mark.skip(reason="We need non SHA1-signed certs and I won't implement it now.") 158 | def test_get_set_authenticated(self): 159 | """INTEGRATION: connecting to server with mutual auth""" 160 | 161 | client = etcd.Client( 162 | port=6001, 163 | protocol="https", 164 | cert=self.client_all_cert, 165 | ) 166 | 167 | set_result = client.set("/test_set", "test-key") 168 | self.assertEquals("set", set_result.action.lower()) 169 | self.assertEquals("/test_set", set_result.key) 170 | self.assertEquals("test-key", set_result.value) 171 | get_result = client.get("/test_set") 172 | self.assertEquals("get", get_result.action.lower()) 173 | self.assertEquals("/test_set", get_result.key) 174 | self.assertEquals("test-key", get_result.value) 175 | -------------------------------------------------------------------------------- /src/etcd/lock.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import etcd 3 | import uuid 4 | 5 | _log = logging.getLogger(__name__) 6 | 7 | 8 | class Lock(object): 9 | """ 10 | Locking recipe for etcd, inspired by the kazoo recipe for zookeeper 11 | """ 12 | 13 | def __init__(self, client, lock_name): 14 | self.client = client 15 | self.name = lock_name 16 | # props to Netflix Curator for this trick. It is possible for our 17 | # create request to succeed on the server, but for a failure to 18 | # prevent us from getting back the full path name. We prefix our 19 | # lock name with a uuid and can check for its presence on retry. 20 | self._uuid = uuid.uuid4().hex 21 | self.path = "{}/{}".format(client.lock_prefix, lock_name) 22 | self.is_taken = False 23 | self._sequence = None 24 | _log.debug("Initiating lock for %s with uuid %s", self.path, self._uuid) 25 | 26 | @property 27 | def uuid(self): 28 | """ 29 | The unique id of the lock 30 | """ 31 | return self._uuid 32 | 33 | @uuid.setter 34 | def uuid(self, value): 35 | old_uuid = self._uuid 36 | self._uuid = value 37 | if not self._find_lock(): 38 | _log.warning("The hand-set uuid was not found, refusing") 39 | self._uuid = old_uuid 40 | raise ValueError("Inexistent UUID") 41 | 42 | @property 43 | def is_acquired(self): 44 | """ 45 | tells us if the lock is acquired 46 | """ 47 | if not self.is_taken: 48 | _log.debug("Lock not taken") 49 | return False 50 | try: 51 | self.client.read(self.lock_key) 52 | return True 53 | except etcd.EtcdKeyNotFound: 54 | _log.warning("Lock was supposedly taken, but we cannot find it") 55 | self.is_taken = False 56 | return False 57 | 58 | def acquire(self, blocking=True, lock_ttl=3600, timeout=0): 59 | """ 60 | Acquire the lock. 61 | 62 | :param blocking Block until the lock is obtained, or timeout is reached 63 | :param lock_ttl The duration of the lock we acquired, set to None for eternal locks 64 | :param timeout The time to wait before giving up on getting a lock 65 | 66 | Raises: 67 | etcd.EtcdLockExpired: If lock expired when try to acquire. 68 | 69 | etcd.EtcdWatchTimeOut: If timeout is reached. 70 | """ 71 | # First of all try to write, if our lock is not present. 72 | if not self._find_lock(): 73 | _log.debug("Lock not found, writing it to %s", self.path) 74 | res = self.client.write(self.path, self.uuid, ttl=lock_ttl, append=True) 75 | self._set_sequence(res.key) 76 | _log.debug("Lock key %s written, sequence is %s", res.key, self._sequence) 77 | elif lock_ttl: 78 | # Renew our lock if already here! 79 | self.client.write(self.lock_key, self.uuid, ttl=lock_ttl) 80 | 81 | # now get the owner of the lock, and the next lowest sequence 82 | return self._acquired(blocking=blocking, timeout=timeout) 83 | 84 | def release(self): 85 | """ 86 | Release the lock 87 | """ 88 | if not self._sequence: 89 | self._find_lock() 90 | try: 91 | _log.debug("Releasing existing lock %s", self.lock_key) 92 | self.client.delete(self.lock_key) 93 | except etcd.EtcdKeyNotFound: 94 | _log.info("Lock %s not found, nothing to release", self.lock_key) 95 | pass 96 | finally: 97 | self.is_taken = False 98 | 99 | def __enter__(self): 100 | """ 101 | You can use the lock as a contextmanager 102 | """ 103 | self.acquire(blocking=True, lock_ttl=None) 104 | return self 105 | 106 | def __exit__(self, type, value, traceback): 107 | self.release() 108 | return False 109 | 110 | def _acquired(self, blocking=True, timeout=0): 111 | locker, nearest = self._get_locker() 112 | self.is_taken = False 113 | if self.lock_key == locker: 114 | _log.debug("Lock acquired!") 115 | # We own the lock, yay! 116 | self.is_taken = True 117 | return True 118 | else: 119 | self.is_taken = False 120 | if not blocking: 121 | return False 122 | # Let's look for the lock 123 | watch_key = nearest.key 124 | _log.debug("Lock not acquired, now watching %s", watch_key) 125 | t = max(0, timeout) 126 | while True: 127 | try: 128 | r = self.client.watch(watch_key, timeout=t, index=nearest.modifiedIndex + 1) 129 | _log.debug("Detected variation for %s: %s", r.key, r.action) 130 | return self._acquired(blocking=True, timeout=timeout) 131 | except etcd.EtcdKeyNotFound: 132 | _log.debug("Key %s not present anymore, moving on", watch_key) 133 | return self._acquired(blocking=True, timeout=timeout) 134 | except (etcd.EtcdLockExpired, etcd.EtcdWatchTimeOut) as e: 135 | raise e 136 | except etcd.EtcdException: 137 | _log.exception("Unexpected exception") 138 | 139 | @property 140 | def lock_key(self): 141 | if not self._sequence: 142 | raise ValueError("No sequence present.") 143 | return self.path + "/" + str(self._sequence) 144 | 145 | def _set_sequence(self, key): 146 | self._sequence = key.replace(self.path, "").lstrip("/") 147 | 148 | def _find_lock(self): 149 | if self._sequence: 150 | try: 151 | res = self.client.read(self.lock_key) 152 | self._uuid = res.value 153 | return True 154 | except etcd.EtcdKeyNotFound: 155 | return False 156 | elif self._uuid: 157 | try: 158 | for r in self.client.read(self.path, recursive=True).leaves: 159 | if r.value == self._uuid: 160 | self._set_sequence(r.key) 161 | return True 162 | except etcd.EtcdKeyNotFound: 163 | pass 164 | return False 165 | 166 | def _get_locker(self): 167 | results = [res for res in self.client.read(self.path, recursive=True).leaves] 168 | if not self._sequence: 169 | self._find_lock() 170 | l = sorted([r.key for r in results]) 171 | _log.debug("Lock keys found: %s", l) 172 | try: 173 | i = l.index(self.lock_key) 174 | if i == 0: 175 | _log.debug("No key before our one, we are the locker") 176 | return (l[0], None) 177 | else: 178 | _log.debug("Locker: %s, key to watch: %s", l[0], l[i - 1]) 179 | return (l[0], next(x for x in results if x.key == l[i - 1])) 180 | except ValueError: 181 | # Something very wrong is going on, most probably 182 | # our lock has expired 183 | raise etcd.EtcdLockExpired("Lock not found") 184 | -------------------------------------------------------------------------------- /src/etcd/tests/unit/test_client.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import etcd 3 | import dns.name 4 | import dns.rdtypes.IN.SRV 5 | import dns.resolver 6 | from etcd.tests.unit import TestClientApiBase 7 | 8 | try: 9 | import mock 10 | except ImportError: 11 | from unittest import mock 12 | 13 | 14 | class TestClient(TestClientApiBase): 15 | def test_instantiate(self): 16 | """client can be instantiated""" 17 | client = etcd.Client() 18 | assert client is not None 19 | 20 | def test_default_host(self): 21 | """default host is 127.0.0.1""" 22 | client = etcd.Client() 23 | assert client.host == "127.0.0.1" 24 | 25 | def test_default_port(self): 26 | """default port is 4001""" 27 | client = etcd.Client() 28 | assert client.port == 4001 29 | 30 | def test_default_prefix(self): 31 | client = etcd.Client() 32 | assert client.version_prefix == "/v2" 33 | 34 | def test_default_protocol(self): 35 | """default protocol is http""" 36 | client = etcd.Client() 37 | assert client.protocol == "http" 38 | 39 | def test_default_read_timeout(self): 40 | """default read_timeout is 60""" 41 | client = etcd.Client() 42 | assert client.read_timeout == 60 43 | 44 | def test_default_allow_redirect(self): 45 | """default allow_redirect is True""" 46 | client = etcd.Client() 47 | assert client.allow_redirect 48 | 49 | def test_default_username(self): 50 | """default username is None""" 51 | client = etcd.Client() 52 | assert client.username is None 53 | 54 | def test_default_password(self): 55 | """default username is None""" 56 | client = etcd.Client() 57 | assert client.password is None 58 | 59 | def test_set_host(self): 60 | """can change host""" 61 | client = etcd.Client(host="192.168.1.1") 62 | assert client.host == "192.168.1.1" 63 | 64 | def test_set_port(self): 65 | """can change port""" 66 | client = etcd.Client(port=4002) 67 | assert client.port == 4002 68 | 69 | def test_set_prefix(self): 70 | client = etcd.Client(version_prefix="/etcd") 71 | assert client.version_prefix == "/etcd" 72 | 73 | def test_set_protocol(self): 74 | """can change protocol""" 75 | client = etcd.Client(protocol="https") 76 | assert client.protocol == "https" 77 | 78 | def test_set_read_timeout(self): 79 | """can set read_timeout""" 80 | client = etcd.Client(read_timeout=45) 81 | assert client.read_timeout == 45 82 | 83 | def test_set_allow_redirect(self): 84 | """can change allow_redirect""" 85 | client = etcd.Client(allow_redirect=False) 86 | assert not client.allow_redirect 87 | 88 | def test_default_base_uri(self): 89 | """default uri is http://127.0.0.1:4001""" 90 | client = etcd.Client() 91 | assert client.base_uri == "http://127.0.0.1:4001" 92 | 93 | def test_set_base_uri(self): 94 | """can change base uri""" 95 | client = etcd.Client(host="192.168.1.1", port=4003, protocol="https") 96 | assert client.base_uri == "https://192.168.1.1:4003" 97 | 98 | def test_set_use_proxies(self): 99 | """can set the use_proxies flag""" 100 | client = etcd.Client(use_proxies=True) 101 | assert client._use_proxies 102 | 103 | def test_set_username_only(self): 104 | client = etcd.Client(username="username") 105 | assert client.username is None 106 | 107 | def test_set_password_only(self): 108 | client = etcd.Client(password="password") 109 | assert client.password is None 110 | 111 | def test_set_username_password(self): 112 | client = etcd.Client(username="username", password="password") 113 | assert client.username == "username" 114 | assert client.password == "password" 115 | 116 | def test_get_headers_with_auth(self): 117 | client = etcd.Client(username="username", password="password") 118 | assert client._get_headers() == {"authorization": "Basic dXNlcm5hbWU6cGFzc3dvcmQ="} 119 | 120 | def test__set_version_info(self): 121 | """Verify _set_version_info makes the proper call to the server""" 122 | data = {"etcdserver": "2.2.3", "etcdcluster": "2.3.0"} 123 | self._mock_api(200, data) 124 | self.client.api_execute.return_value.getheader.return_value = None 125 | # Create the client and make the call. 126 | self.client._set_version_info() 127 | 128 | # Verify we call the proper endpoint 129 | self.client.api_execute.assert_called_once_with("/version", self.client._MGET) 130 | # Verify the properties while we are here 131 | self.assertEqual("2.2.3", self.client.version) 132 | self.assertEqual("2.3.0", self.client.cluster_version) 133 | 134 | def test_version_property(self): 135 | """Ensure the version property is set on first access.""" 136 | data = {"etcdserver": "2.2.3", "etcdcluster": "2.3.0"} 137 | self._mock_api(200, data) 138 | self.client.api_execute.return_value.getheader.return_value = None 139 | 140 | # Verify the version property is set 141 | self.assertEqual("2.2.3", self.client.version) 142 | 143 | def test_cluster_version_property(self): 144 | """Ensure the cluster version property is set on first access.""" 145 | data = {"etcdserver": "2.2.3", "etcdcluster": "2.3.0"} 146 | self._mock_api(200, data) 147 | self.client.api_execute.return_value.getheader.return_value = None 148 | # Verify the cluster_version property is set 149 | self.assertEqual("2.3.0", self.client.cluster_version) 150 | 151 | def test_get_headers_without_auth(self): 152 | client = etcd.Client() 153 | assert client._get_headers() == {} 154 | 155 | def test_allow_reconnect(self): 156 | """Fails if allow_reconnect is false and a list of hosts is given""" 157 | with self.assertRaises(etcd.EtcdException): 158 | etcd.Client( 159 | host=(("localhost", 4001), ("localhost", 4002)), 160 | ) 161 | # This doesn't raise an exception 162 | client = etcd.Client( 163 | host=(("localhost", 4001), ("localhost", 4002)), 164 | allow_reconnect=True, 165 | use_proxies=True, 166 | ) 167 | 168 | def test_discover(self): 169 | """Tests discovery.""" 170 | answers = [] 171 | for i in range(1, 3): 172 | r = mock.create_autospec(dns.rdtypes.IN.SRV.SRV) 173 | r.port = 2379 174 | try: 175 | method = dns.name.from_unicode 176 | except AttributeError: 177 | method = dns.name.from_text 178 | r.target = method("etcd{}.example.com".format(i)) 179 | answers.append(r) 180 | dns.resolver.query = mock.create_autospec(dns.resolver.query, return_value=answers) 181 | self.machines = etcd.Client.machines 182 | etcd.Client.machines = mock.create_autospec( 183 | etcd.Client.machines, return_value=["https://etcd2.example.com:2379"] 184 | ) 185 | c = etcd.Client(srv_domain="example.com", allow_reconnect=True, protocol="https") 186 | etcd.Client.machines = self.machines 187 | self.assertEqual(c.host, "etcd1.example.com") 188 | self.assertEqual(c.port, 2379) 189 | self.assertEqual(c._machines_cache, ["https://etcd2.example.com:2379"]) 190 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | python-etcd documentation 2 | ========================= 3 | 4 | A python client for Etcd https://github.com/coreos/etcd 5 | 6 | Official documentation: http://python-etcd.readthedocs.org/ 7 | 8 | .. image:: https://travis-ci.org/jplana/python-etcd.png?branch=master 9 | :target: https://travis-ci.org/jplana/python-etcd 10 | 11 | .. image:: https://coveralls.io/repos/jplana/python-etcd/badge.svg?branch=master&service=github 12 | :target: https://coveralls.io/github/jplana/python-etcd?branch=master 13 | 14 | Installation 15 | ------------ 16 | 17 | Pre-requirements 18 | ~~~~~~~~~~~~~~~~ 19 | 20 | This version of python-etcd will only work correctly with the etcd server version 2.0.x or later. If you are running an older version of etcd, please use python-etcd 0.3.3 or earlier. 21 | 22 | This client is known to work with python 2.7 and with python 3.3 or above. It is not tested or expected to work in more outdated versions of python. 23 | 24 | From source 25 | ~~~~~~~~~~~ 26 | 27 | .. code:: bash 28 | 29 | $ python setup.py install 30 | 31 | From Pypi 32 | ~~~~~~~~~ 33 | 34 | .. code:: bash 35 | 36 | $ python -m pip install python-etcd 37 | 38 | Usage 39 | ----- 40 | 41 | The basic methods of the client have changed compared to previous versions, to reflect the new API structure; however a compatibility layer has been maintained so that you don't necessarily need to rewrite all your existing code. 42 | 43 | Create a client object 44 | ~~~~~~~~~~~~~~~~~~~~~~ 45 | 46 | .. code:: python 47 | 48 | import etcd 49 | 50 | client = etcd.Client() # this will create a client against etcd server running on localhost on port 4001 51 | client = etcd.Client(port=4002) 52 | client = etcd.Client(host='127.0.0.1', port=4003) 53 | client = etcd.Client(host=(('127.0.0.1', 4001), ('127.0.0.1', 4002), ('127.0.0.1', 4003))) 54 | client = etcd.Client(host='127.0.0.1', port=4003, allow_redirect=False) # wont let you run sensitive commands on non-leader machines, default is true 55 | # If you have defined a SRV record for _etcd._tcp.example.com pointing to the clients 56 | client = etcd.Client(srv_domain='example.com', protocol="https") 57 | # create a client against https://api.example.com:443/etcd 58 | client = etcd.Client(host='api.example.com', protocol='https', port=443, version_prefix='/etcd') 59 | 60 | Write a key 61 | ~~~~~~~~~~~ 62 | 63 | .. code:: python 64 | 65 | client.write('/nodes/n1', 1) 66 | # with ttl 67 | client.write('/nodes/n2', 2, ttl=4) # sets the ttl to 4 seconds 68 | client.set('/nodes/n2', 1) # Equivalent, for compatibility reasons. 69 | 70 | Read a key 71 | ~~~~~~~~~~ 72 | 73 | .. code:: python 74 | 75 | client.read('/nodes/n2').value 76 | client.read('/nodes', recursive = True) #get all the values of a directory, recursively. 77 | client.get('/nodes/n2').value 78 | 79 | # raises etcd.EtcdKeyNotFound when key not found 80 | try: 81 | client.read('/invalid/path') 82 | except etcd.EtcdKeyNotFound: 83 | # do something 84 | print "error" 85 | 86 | 87 | Delete a key 88 | ~~~~~~~~~~~~ 89 | 90 | .. code:: python 91 | 92 | client.delete('/nodes/n1') 93 | 94 | Atomic Compare and Swap 95 | ~~~~~~~~~~~~~~~~~~~~~~~ 96 | 97 | .. code:: python 98 | 99 | client.write('/nodes/n2', 2, prevValue = 4) # will set /nodes/n2 's value to 2 only if its previous value was 4 and 100 | client.write('/nodes/n2', 2, prevExist = False) # will set /nodes/n2 's value to 2 only if the key did not exist before 101 | client.write('/nodes/n2', 2, prevIndex = 30) # will set /nodes/n2 's value to 2 only if the key was last modified at index 30 102 | client.test_and_set('/nodes/n2', 2, 4) #equivalent to client.write('/nodes/n2', 2, prevValue = 4) 103 | 104 | You can also atomically update a result: 105 | 106 | .. code:: python 107 | 108 | result = client.read('/foo') 109 | print(result.value) # bar 110 | result.value += u'bar' 111 | updated = client.update(result) # if any other client wrote '/foo' in the meantime this will fail 112 | print(updated.value) # barbar 113 | 114 | Watch a key 115 | ~~~~~~~~~~~ 116 | 117 | .. code:: python 118 | 119 | client.read('/nodes/n1', wait = True) # will wait till the key is changed, and return once its changed 120 | client.read('/nodes/n1', wait = True, timeout=30) # will wait till the key is changed, and return once its changed, or exit with an exception after 30 seconds. 121 | client.read('/nodes/n1', wait = True, waitIndex = 10) # get all changes on this key starting from index 10 122 | client.watch('/nodes/n1') #equivalent to client.read('/nodes/n1', wait = True) 123 | client.watch('/nodes/n1', index = 10) 124 | 125 | Refreshing key TTL 126 | ~~~~~~~~~~~~~~~~~~ 127 | 128 | (Since etcd 2.3.0) Keys in etcd can be refreshed without notifying current watchers. 129 | 130 | This can be achieved by setting the refresh to true when updating a TTL. 131 | 132 | You cannot update the value of a key when refreshing it. 133 | 134 | .. code:: python 135 | 136 | client.write('/nodes/n1', 'value', ttl=30) # sets the ttl to 30 seconds 137 | client.refresh('/nodes/n1', ttl=600) # refresh ttl to 600 seconds, without notifying current watchers 138 | 139 | Locking module 140 | ~~~~~~~~~~~~~~ 141 | 142 | .. code:: python 143 | 144 | # Initialize the lock object: 145 | # NOTE: this does not acquire a lock yet 146 | client = etcd.Client() 147 | # Or you can custom lock prefix, default is '/_locks/' if you are using HEAD 148 | client = etcd.Client(lock_prefix='/my_etcd_root/_locks') 149 | lock = etcd.Lock(client, 'my_lock_name') 150 | 151 | # Use the lock object: 152 | lock.acquire(blocking=True, # will block until the lock is acquired 153 | lock_ttl=None) # lock will live until we release it 154 | lock.is_acquired # True 155 | lock.acquire(lock_ttl=60) # renew a lock 156 | lock.release() # release an existing lock 157 | lock.is_acquired # False 158 | 159 | # The lock object may also be used as a context manager: 160 | client = etcd.Client() 161 | with etcd.Lock(client, 'customer1') as my_lock: 162 | do_stuff() 163 | my_lock.is_acquired # True 164 | my_lock.acquire(lock_ttl=60) 165 | my_lock.is_acquired # False 166 | 167 | 168 | Get machines in the cluster 169 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~ 170 | 171 | .. code:: python 172 | 173 | client.machines 174 | 175 | Get leader of the cluster 176 | ~~~~~~~~~~~~~~~~~~~~~~~~~ 177 | 178 | .. code:: python 179 | 180 | client.leader 181 | 182 | Generate a sequential key in a directory 183 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 184 | 185 | .. code:: python 186 | 187 | x = client.write("/dir/name", "value", append=True) 188 | print("generated key: " + x.key) 189 | print("stored value: " + x.value) 190 | 191 | List contents of a directory 192 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 193 | 194 | .. code:: python 195 | 196 | #stick a couple values in the directory 197 | client.write("/dir/name", "value1", append=True) 198 | client.write("/dir/name", "value2", append=True) 199 | 200 | directory = client.get("/dir/name") 201 | 202 | # loop through directory children 203 | for result in directory.children: 204 | print(result.key + ": " + result.value) 205 | 206 | # or just get the first child value 207 | print(directory.children.next().value) 208 | 209 | Development setup 210 | ----------------- 211 | 212 | To check your code, 213 | 214 | .. code:: bash 215 | 216 | $ tox 217 | 218 | to test you should have etcd available in your system path: 219 | 220 | .. code:: bash 221 | 222 | $ command -v etcd 223 | 224 | to generate documentation, 225 | 226 | .. code:: bash 227 | 228 | $ cd docs 229 | $ make 230 | 231 | Release HOWTO 232 | ------------- 233 | 234 | To make a release 235 | 236 | 1) Update release date/version in NEWS.txt and setup.py 237 | 2) Run 'python setup.py sdist' 238 | 3) Test the generated source distribution in dist/ 239 | 4) Upload to PyPI: 'python setup.py sdist register upload' 240 | -------------------------------------------------------------------------------- /src/etcd/tests/unit/test_lock.py: -------------------------------------------------------------------------------- 1 | import etcd 2 | 3 | try: 4 | import mock 5 | except ImportError: 6 | from unittest import mock 7 | from etcd.tests.unit import TestClientApiBase 8 | 9 | 10 | class TestClientLock(TestClientApiBase): 11 | def recursive_read(self): 12 | nodes = [ 13 | { 14 | "key": "/_locks/test_lock/1", 15 | "value": "2qwwwq", 16 | "modifiedIndex": 33, 17 | "createdIndex": 33, 18 | }, 19 | { 20 | "key": "/_locks/test_lock/34", 21 | "value": self.locker.uuid, 22 | "modifiedIndex": 34, 23 | "createdIndex": 34, 24 | }, 25 | ] 26 | d = { 27 | "action": "get", 28 | "node": { 29 | "dir": True, 30 | "nodes": [{"key": "/_locks/test_lock", "dir": True, "nodes": nodes}], 31 | }, 32 | } 33 | self._mock_api(200, d) 34 | 35 | def setUp(self): 36 | super(TestClientLock, self).setUp() 37 | self.locker = etcd.Lock(self.client, "test_lock") 38 | 39 | def test_initialization(self): 40 | """ 41 | Verify the lock gets initialized correctly 42 | """ 43 | self.assertEqual(self.locker.name, "test_lock") 44 | self.assertEqual(self.locker.path, "/_locks/test_lock") 45 | self.assertEqual(self.locker.is_taken, False) 46 | 47 | def test_acquire(self): 48 | """ 49 | Acquiring a precedingly inexistent lock works. 50 | """ 51 | l = etcd.Lock(self.client, "test_lock") 52 | l._find_lock = mock.MagicMock(spec=l._find_lock, return_value=False) 53 | l._acquired = mock.MagicMock(spec=l._acquired, return_value=True) 54 | # Mock the write 55 | d = { 56 | "action": "set", 57 | "node": { 58 | "modifiedIndex": 190, 59 | "key": "/_locks/test_lock/1", 60 | "value": l.uuid, 61 | }, 62 | } 63 | self._mock_api(200, d) 64 | self.assertEqual(l.acquire(), True) 65 | self.assertEqual(l._sequence, "1") 66 | 67 | def test_is_acquired(self): 68 | """ 69 | Test is_acquired 70 | """ 71 | self.locker._sequence = "1" 72 | d = { 73 | "action": "get", 74 | "node": { 75 | "modifiedIndex": 190, 76 | "key": "/_locks/test_lock/1", 77 | "value": self.locker.uuid, 78 | }, 79 | } 80 | self._mock_api(200, d) 81 | self.locker.is_taken = True 82 | self.assertEqual(self.locker.is_acquired, True) 83 | 84 | def test_is_not_acquired(self): 85 | """ 86 | Test is_acquired failures 87 | """ 88 | self.locker._sequence = "2" 89 | self.locker.is_taken = False 90 | self.assertEqual(self.locker.is_acquired, False) 91 | self.locker.is_taken = True 92 | self._mock_exception(etcd.EtcdKeyNotFound, self.locker.lock_key) 93 | self.assertEqual(self.locker.is_acquired, False) 94 | self.assertEqual(self.locker.is_taken, False) 95 | 96 | def test_acquired(self): 97 | """ 98 | Test the acquiring primitives 99 | """ 100 | self.locker._sequence = "4" 101 | retval = ("/_locks/test_lock/4", None) 102 | self.locker._get_locker = mock.MagicMock(return_value=retval) 103 | self.assertTrue(self.locker._acquired()) 104 | self.assertTrue(self.locker.is_taken) 105 | retval = ("/_locks/test_lock/1", "/_locks/test_lock/4") 106 | self.locker._get_locker = mock.MagicMock(return_value=retval) 107 | self.assertFalse(self.locker._acquired(blocking=False)) 108 | self.assertFalse(self.locker.is_taken) 109 | d = { 110 | "action": "delete", 111 | "node": { 112 | "modifiedIndex": 190, 113 | "key": "/_locks/test_lock/1", 114 | "value": self.locker.uuid, 115 | }, 116 | } 117 | self._mock_api(200, d) 118 | returns = [ 119 | ("/_locks/test_lock/1", "/_locks/test_lock/4"), 120 | ("/_locks/test_lock/4", None), 121 | ] 122 | 123 | def side_effect(): 124 | return returns.pop() 125 | 126 | self.locker._get_locker = mock.MagicMock(side_effect=side_effect) 127 | self.assertTrue(self.locker._acquired()) 128 | 129 | def test_acquired_no_timeout(self): 130 | self.locker._sequence = 4 131 | returns = [ 132 | ("/_locks/test_lock/4", None), 133 | ( 134 | "/_locks/test_lock/1", 135 | etcd.EtcdResult(node={"key": "/_locks/test_lock/4", "modifiedIndex": 1}), 136 | ), 137 | ] 138 | 139 | def side_effect(): 140 | return returns.pop() 141 | 142 | d = { 143 | "action": "get", 144 | "node": { 145 | "modifiedIndex": 190, 146 | "key": "/_locks/test_lock/4", 147 | "value": self.locker.uuid, 148 | }, 149 | } 150 | self._mock_api(200, d) 151 | 152 | self.locker._get_locker = mock.create_autospec( 153 | self.locker._get_locker, side_effect=side_effect 154 | ) 155 | self.assertTrue(self.locker._acquired()) 156 | 157 | def test_lock_key(self): 158 | """ 159 | Test responses from the lock_key property 160 | """ 161 | with self.assertRaises(ValueError): 162 | self.locker.lock_key 163 | self.locker._sequence = "5" 164 | self.assertEqual("/_locks/test_lock/5", self.locker.lock_key) 165 | 166 | def test_set_sequence(self): 167 | self.locker._set_sequence("/_locks/test_lock/10") 168 | self.assertEqual("10", self.locker._sequence) 169 | 170 | def test_find_lock(self): 171 | d = { 172 | "action": "get", 173 | "node": { 174 | "modifiedIndex": 190, 175 | "key": "/_locks/test_lock/1", 176 | "value": self.locker.uuid, 177 | }, 178 | } 179 | self._mock_api(200, d) 180 | self.locker._sequence = "1" 181 | self.assertTrue(self.locker._find_lock()) 182 | # Now let's pretend the lock is not there 183 | self._mock_exception(etcd.EtcdKeyNotFound, self.locker.lock_key) 184 | self.assertFalse(self.locker._find_lock()) 185 | self.locker._sequence = None 186 | self.recursive_read() 187 | self.assertTrue(self.locker._find_lock()) 188 | self.assertEqual(self.locker._sequence, "34") 189 | 190 | def test_get_locker(self): 191 | self.recursive_read() 192 | self.assertEqual( 193 | ( 194 | "/_locks/test_lock/1", 195 | etcd.EtcdResult( 196 | node={ 197 | "newKey": False, 198 | "_children": [], 199 | "createdIndex": 33, 200 | "modifiedIndex": 33, 201 | "value": "2qwwwq", 202 | "expiration": None, 203 | "key": "/_locks/test_lock/1", 204 | "ttl": None, 205 | "action": None, 206 | "dir": False, 207 | } 208 | ), 209 | ), 210 | self.locker._get_locker(), 211 | ) 212 | with self.assertRaises(etcd.EtcdLockExpired): 213 | self.locker._sequence = "35" 214 | self.locker._get_locker() 215 | 216 | def test_release(self): 217 | d = { 218 | "action": "delete", 219 | "node": { 220 | "modifiedIndex": 190, 221 | "key": "/_locks/test_lock/1", 222 | "value": self.locker.uuid, 223 | }, 224 | } 225 | self._mock_api(200, d) 226 | self.locker._sequence = 1 227 | self.locker.is_taken = True 228 | self.locker.release() 229 | self.assertFalse(self.locker.is_taken) 230 | -------------------------------------------------------------------------------- /src/etcd/tests/integration/helpers.py: -------------------------------------------------------------------------------- 1 | import re 2 | import shutil 3 | import subprocess 4 | import tempfile 5 | import logging 6 | import time 7 | import hashlib 8 | import uuid 9 | 10 | from OpenSSL import crypto 11 | 12 | 13 | class EtcdProcessHelper(object): 14 | def __init__( 15 | self, 16 | base_directory, 17 | proc_name="etcd", 18 | port_range_start=4001, 19 | internal_port_range_start=7001, 20 | cluster=False, 21 | tls=False, 22 | ): 23 | self.base_directory = base_directory 24 | self.proc_name = proc_name 25 | self.port_range_start = port_range_start 26 | self.internal_port_range_start = internal_port_range_start 27 | self.processes = {} 28 | self.cluster = cluster 29 | self.schema = "http://" 30 | if tls: 31 | self.schema = "https://" 32 | self.compat_args = self.check_compat_args() 33 | 34 | def check_compat_args(self): 35 | version_re = re.compile(r"^etcd version:\s+(\d)\.(\d)", re.I) 36 | version_data = subprocess.check_output([self.proc_name, "--version"]).decode("utf-8") 37 | match = version_re.match(version_data) 38 | if match is not None: 39 | etcd_version = (int(match.group(1)), int(match.group(2))) 40 | else: 41 | etcd_version = (0, 0) 42 | if etcd_version[0] < 3 or (etcd_version[0] == 3 and etcd_version[1] < 4): 43 | return [] 44 | else: 45 | return ["--enable-v2=true"] 46 | 47 | def run(self, number=1, proc_args=[]): 48 | if number > 1: 49 | initial_cluster = ",".join( 50 | [ 51 | "test-node-{}={}127.0.0.1:{}".format( 52 | slot, "http://", self.internal_port_range_start + slot 53 | ) 54 | for slot in range(0, number) 55 | ] 56 | ) 57 | proc_args.extend( 58 | [ 59 | "-initial-cluster", 60 | initial_cluster, 61 | "-initial-cluster-state", 62 | "new", 63 | ] 64 | ) 65 | else: 66 | proc_args.extend( 67 | [ 68 | "-initial-cluster", 69 | "test-node-0=http://127.0.0.1:{}".format(self.internal_port_range_start), 70 | "-initial-cluster-state", 71 | "new", 72 | ] 73 | ) 74 | 75 | for i in range(0, number): 76 | self.add_one(i, proc_args) 77 | 78 | def stop(self): 79 | log = logging.getLogger() 80 | for key in [k for k in self.processes.keys()]: 81 | self.kill_one(key) 82 | 83 | def add_one(self, slot, proc_args=None): 84 | log = logging.getLogger() 85 | directory = tempfile.mkdtemp(dir=self.base_directory, prefix="python-etcd.%d-" % slot) 86 | 87 | log.debug("Created directory %s" % directory) 88 | client = "%s127.0.0.1:%d" % (self.schema, self.port_range_start + slot) 89 | peer = "%s127.0.0.1:%d" % ( 90 | "http://", 91 | self.internal_port_range_start + slot, 92 | ) 93 | daemon_args = [ 94 | self.proc_name, 95 | "-data-dir", 96 | directory, 97 | "-name", 98 | "test-node-%d" % slot, 99 | "-initial-advertise-peer-urls", 100 | peer, 101 | "-listen-peer-urls", 102 | peer, 103 | "-advertise-client-urls", 104 | client, 105 | "-listen-client-urls", 106 | client, 107 | ] 108 | daemon_args.extend(self.compat_args) 109 | if proc_args: 110 | daemon_args.extend(proc_args) 111 | 112 | daemon = subprocess.Popen(daemon_args) 113 | log.debug("Started %d" % daemon.pid) 114 | log.debug("Params: %s" % daemon_args) 115 | time.sleep(2) 116 | self.processes[slot] = (directory, daemon) 117 | 118 | def kill_one(self, slot): 119 | log = logging.getLogger() 120 | data_dir, process = self.processes.pop(slot) 121 | process.kill() 122 | time.sleep(2) 123 | log.debug("Killed etcd pid:%d", process.pid) 124 | shutil.rmtree(data_dir) 125 | log.debug("Removed directory %s" % data_dir) 126 | 127 | 128 | class TestingCA(object): 129 | @classmethod 130 | def create_test_ca_certificate(cls, cert_path, key_path, cn=None): 131 | k = crypto.PKey() 132 | k.generate_key(crypto.TYPE_RSA, 4096) 133 | cert = crypto.X509() 134 | 135 | if not cn: 136 | serial = uuid.uuid4().int 137 | else: 138 | md5_hash = hashlib.md5() 139 | md5_hash.update(cn.encode("utf-8")) 140 | serial = int(md5_hash.hexdigest(), 36) 141 | cert.get_subject().CN = cn 142 | 143 | cert.get_subject().C = "ES" 144 | cert.get_subject().ST = "State" 145 | cert.get_subject().L = "City" 146 | cert.get_subject().O = "Organization" 147 | cert.get_subject().OU = "Organizational Unit" 148 | cert.set_serial_number(serial) 149 | cert.gmtime_adj_notBefore(0) 150 | cert.gmtime_adj_notAfter(315360000) 151 | cert.set_issuer(cert.get_subject()) 152 | cert.set_pubkey(k) 153 | cert.add_extensions( 154 | [ 155 | crypto.X509Extension( 156 | "basicConstraints".encode("ascii"), 157 | False, 158 | "CA:TRUE".encode("ascii"), 159 | ), 160 | crypto.X509Extension( 161 | "keyUsage".encode("ascii"), 162 | False, 163 | "keyCertSign, cRLSign".encode("ascii"), 164 | ), 165 | crypto.X509Extension( 166 | "subjectKeyIdentifier".encode("ascii"), 167 | False, 168 | "hash".encode("ascii"), 169 | subject=cert, 170 | ), 171 | ] 172 | ) 173 | 174 | cert.add_extensions( 175 | [ 176 | crypto.X509Extension( 177 | "authorityKeyIdentifier".encode("ascii"), 178 | False, 179 | "keyid:always".encode("ascii"), 180 | issuer=cert, 181 | ) 182 | ] 183 | ) 184 | 185 | cert.sign(k, "sha1") 186 | 187 | with open(cert_path, "w") as f: 188 | f.write(crypto.dump_certificate(crypto.FILETYPE_PEM, cert).decode("utf-8")) 189 | 190 | with open(key_path, "w") as f: 191 | f.write(crypto.dump_privatekey(crypto.FILETYPE_PEM, k).decode("utf-8")) 192 | 193 | return cert, k 194 | 195 | @classmethod 196 | def create_test_certificate(cls, ca, ca_key, cert_path, key_path, cn=None): 197 | k = crypto.PKey() 198 | k.generate_key(crypto.TYPE_RSA, 4096) 199 | cert = crypto.X509() 200 | 201 | if not cn: 202 | serial = uuid.uuid4().int 203 | else: 204 | md5_hash = hashlib.md5() 205 | md5_hash.update(cn.encode("utf-8")) 206 | serial = int(md5_hash.hexdigest(), 36) 207 | cert.get_subject().CN = cn 208 | 209 | cert.get_subject().C = "ES" 210 | cert.get_subject().ST = "State" 211 | cert.get_subject().L = "City" 212 | cert.get_subject().O = "Organization" 213 | cert.get_subject().OU = "Organizational Unit" 214 | 215 | cert.add_extensions( 216 | [ 217 | crypto.X509Extension( 218 | "keyUsage".encode("ascii"), 219 | False, 220 | "nonRepudiation,digitalSignature,keyEncipherment".encode("ascii"), 221 | ), 222 | crypto.X509Extension( 223 | "extendedKeyUsage".encode("ascii"), 224 | False, 225 | "clientAuth,serverAuth".encode("ascii"), 226 | ), 227 | crypto.X509Extension( 228 | "subjectAltName".encode("ascii"), 229 | False, 230 | "IP: 127.0.0.1".encode("ascii"), 231 | ), 232 | ] 233 | ) 234 | 235 | cert.gmtime_adj_notBefore(0) 236 | cert.gmtime_adj_notAfter(315360000) 237 | cert.set_issuer(ca.get_subject()) 238 | cert.set_pubkey(k) 239 | cert.set_serial_number(serial) 240 | 241 | cert.sign(ca_key, "sha1") 242 | 243 | with open(cert_path, "w") as f: 244 | f.write(crypto.dump_certificate(crypto.FILETYPE_PEM, cert).decode("utf-8")) 245 | 246 | with open(key_path, "w") as f: 247 | f.write(crypto.dump_privatekey(crypto.FILETYPE_PEM, k).decode("utf-8")) 248 | -------------------------------------------------------------------------------- /docs-source/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import sys, os 4 | 5 | 6 | class Mock(object): 7 | def __init__(self, *args, **kwargs): 8 | pass 9 | 10 | def __call__(self, *args, **kwargs): 11 | return Mock() 12 | 13 | @classmethod 14 | def __getattr__(cls, name): 15 | if name in ("__file__", "__path__"): 16 | return "/dev/null" 17 | elif name[0] == name[0].upper(): 18 | mockType = type(name, (), {}) 19 | mockType.__module__ = __name__ 20 | return mockType 21 | else: 22 | return Mock() 23 | 24 | 25 | MOCK_MODULES = ["urllib3"] 26 | for mod_name in MOCK_MODULES: 27 | sys.modules[mod_name] = Mock() 28 | 29 | # If extensions (or modules to document with autodoc) are in another directory, 30 | # add these directories to sys.path here. If the directory is relative to the 31 | # documentation root, use os.path.abspath to make it absolute, like shown here. 32 | sys.path.insert(0, os.path.abspath("../src")) 33 | 34 | # -- General configuration ----------------------------------------------------- 35 | 36 | # If your documentation needs a minimal Sphinx version, state it here. 37 | # needs_sphinx = '1.0' 38 | 39 | # Add any Sphinx extension module names here, as strings. They can be extensions 40 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 41 | extensions = ["sphinx.ext.autodoc"] 42 | 43 | # Add any paths that contain templates here, relative to this directory. 44 | templates_path = ["_templates"] 45 | 46 | # The suffix of source filenames. 47 | source_suffix = ".rst" 48 | 49 | # The encoding of source files. 50 | # source_encoding = 'utf-8-sig' 51 | 52 | # The master toctree document. 53 | master_doc = "index" 54 | 55 | # General information about the project. 56 | project = "python-etcd" 57 | copyright = "2013-2015 Jose Plana, Giuseppe Lavagetto" 58 | 59 | # The version info for the project you're documenting, acts as replacement for 60 | # |version| and |release|, also used in various other places throughout the 61 | # built documents. 62 | # 63 | # The short X.Y version. 64 | version = "0.4" 65 | # The full version, including alpha/beta/rc tags. 66 | release = "0.4.3" 67 | 68 | # The language for content autogenerated by Sphinx. Refer to documentation 69 | # for a list of supported languages. 70 | # language = None 71 | 72 | # There are two options for replacing |today|: either, you set today to some 73 | # non-false value, then it is used: 74 | # today = '' 75 | # Else, today_fmt is used as the format for a strftime call. 76 | # today_fmt = '%B %d, %Y' 77 | 78 | # List of patterns, relative to source directory, that match files and 79 | # directories to ignore when looking for source files. 80 | exclude_patterns = ["_build"] 81 | 82 | # The reST default role (used for this markup: `text`) to use for all documents. 83 | # default_role = None 84 | 85 | # If true, '()' will be appended to :func: etc. cross-reference text. 86 | # add_function_parentheses = True 87 | 88 | # If true, the current module name will be prepended to all description 89 | # unit titles (such as .. function::). 90 | # add_module_names = True 91 | 92 | # If true, sectionauthor and moduleauthor directives will be shown in the 93 | # output. They are ignored by default. 94 | # show_authors = False 95 | 96 | # The name of the Pygments (syntax highlighting) style to use. 97 | pygments_style = "sphinx" 98 | 99 | # A list of ignored prefixes for module index sorting. 100 | # modindex_common_prefix = [] 101 | 102 | 103 | # -- Options for HTML output --------------------------------------------------- 104 | 105 | # The theme to use for HTML and HTML Help pages. See the documentation for 106 | # a list of builtin themes. 107 | html_theme = "sphinxdoc" 108 | 109 | # Theme options are theme-specific and customize the look and feel of a theme 110 | # further. For a list of options available for each theme, see the 111 | # documentation. 112 | # html_theme_options = {} 113 | 114 | # Add any paths that contain custom themes here, relative to this directory. 115 | # html_theme_path = [] 116 | 117 | # The name for this set of Sphinx documents. If None, it defaults to 118 | # " v documentation". 119 | # html_title = None 120 | 121 | # A shorter title for the navigation bar. Default is the same as html_title. 122 | # html_short_title = None 123 | 124 | # The name of an image file (relative to this directory) to place at the top 125 | # of the sidebar. 126 | # html_logo = None 127 | 128 | # The name of an image file (within the static path) to use as favicon of the 129 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 130 | # pixels large. 131 | # html_favicon = None 132 | 133 | # Add any paths that contain custom static files (such as style sheets) here, 134 | # relative to this directory. They are copied after the builtin static files, 135 | # so a file named "default.css" will overwrite the builtin "default.css". 136 | html_static_path = ["_static"] 137 | 138 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 139 | # using the given strftime format. 140 | # html_last_updated_fmt = '%b %d, %Y' 141 | 142 | # If true, SmartyPants will be used to convert quotes and dashes to 143 | # typographically correct entities. 144 | # html_use_smartypants = True 145 | 146 | # Custom sidebar templates, maps document names to template names. 147 | # html_sidebars = {} 148 | 149 | # Additional templates that should be rendered to pages, maps page names to 150 | # template names. 151 | # html_additional_pages = {} 152 | 153 | # If false, no module index is generated. 154 | # html_domain_indices = True 155 | 156 | # If false, no index is generated. 157 | # html_use_index = True 158 | 159 | # If true, the index is split into individual pages for each letter. 160 | # html_split_index = False 161 | 162 | # If true, links to the reST sources are added to the pages. 163 | # html_show_sourcelink = True 164 | 165 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 166 | # html_show_sphinx = True 167 | 168 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 169 | # html_show_copyright = True 170 | 171 | # If true, an OpenSearch description file will be output, and all pages will 172 | # contain a tag referring to it. The value of this option must be the 173 | # base URL from which the finished HTML is served. 174 | # html_use_opensearch = '' 175 | 176 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 177 | # html_file_suffix = None 178 | 179 | # Output file base name for HTML help builder. 180 | htmlhelp_basename = "python-etcddoc" 181 | 182 | 183 | # -- Options for LaTeX output -------------------------------------------------- 184 | 185 | latex_elements = { 186 | # The paper size ('letterpaper' or 'a4paper'). 187 | #'papersize': 'letterpaper', 188 | # The font size ('10pt', '11pt' or '12pt'). 189 | #'pointsize': '10pt', 190 | # Additional stuff for the LaTeX preamble. 191 | #'preamble': '', 192 | } 193 | 194 | # Grouping the document tree into LaTeX files. List of tuples 195 | # (source start file, target name, title, author, documentclass [howto/manual]). 196 | latex_documents = [ 197 | ("index", "python-etcd.tex", "python-etcd Documentation", "Jose Plana", "manual"), 198 | ] 199 | 200 | # The name of an image file (relative to this directory) to place at the top of 201 | # the title page. 202 | # latex_logo = None 203 | 204 | # For "manual" documents, if this is true, then toplevel headings are parts, 205 | # not chapters. 206 | # latex_use_parts = False 207 | 208 | # If true, show page references after internal links. 209 | # latex_show_pagerefs = False 210 | 211 | # If true, show URL addresses after external links. 212 | # latex_show_urls = False 213 | 214 | # Documents to append as an appendix to all manuals. 215 | # latex_appendices = [] 216 | 217 | # If false, no module index is generated. 218 | # latex_domain_indices = True 219 | 220 | 221 | # -- Options for manual page output -------------------------------------------- 222 | 223 | # One entry per manual page. List of tuples 224 | # (source start file, name, description, authors, manual section). 225 | man_pages = [("index", "python-etcd", "python-etcd Documentation", ["Jose Plana"], 1)] 226 | 227 | # If true, show URL addresses after external links. 228 | # man_show_urls = False 229 | 230 | 231 | # -- Options for Texinfo output ------------------------------------------------ 232 | 233 | # Grouping the document tree into Texinfo files. List of tuples 234 | # (source start file, target name, title, author, 235 | # dir menu entry, description, category) 236 | texinfo_documents = [ 237 | ( 238 | "index", 239 | "python-etcd", 240 | "python-etcd Documentation", 241 | "Jose Plana", 242 | "python-etcd", 243 | "One line description of project.", 244 | "Miscellaneous", 245 | ), 246 | ] 247 | 248 | # Documents to append as an appendix to all manuals. 249 | # texinfo_appendices = [] 250 | 251 | # If false, no module index is generated. 252 | # texinfo_domain_indices = True 253 | 254 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 255 | # texinfo_show_urls = 'footnote' 256 | -------------------------------------------------------------------------------- /src/etcd/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from .client import Client 3 | from .lock import Lock 4 | 5 | _log = logging.getLogger(__name__) 6 | 7 | # Prevent "no handler" warnings to stderr in projects that do not configure 8 | # logging. 9 | try: 10 | from logging import NullHandler 11 | except ImportError: 12 | # Python <2.7, just define it. 13 | class NullHandler(logging.Handler): 14 | def emit(self, record): 15 | pass 16 | 17 | 18 | _log.addHandler(NullHandler()) 19 | 20 | 21 | class EtcdResult(object): 22 | _node_props = { 23 | "key": None, 24 | "value": None, 25 | "expiration": None, 26 | "ttl": None, 27 | "modifiedIndex": None, 28 | "createdIndex": None, 29 | "newKey": False, 30 | "dir": False, 31 | } 32 | 33 | def __init__(self, action=None, node=None, prevNode=None, **kwdargs): 34 | """ 35 | Creates an EtcdResult object. 36 | 37 | Args: 38 | action (str): The action that resulted in key creation 39 | 40 | node (dict): The dictionary containing all node information. 41 | 42 | prevNode (dict): The dictionary containing previous node information. 43 | 44 | """ 45 | self.action = action 46 | for key, default in self._node_props.items(): 47 | if key in node: 48 | setattr(self, key, node[key]) 49 | else: 50 | setattr(self, key, default) 51 | 52 | self._children = [] 53 | if self.dir and "nodes" in node: 54 | # We keep the data in raw format, converting them only when needed 55 | self._children = node["nodes"] 56 | 57 | if prevNode: 58 | self._prev_node = EtcdResult(None, node=prevNode) 59 | # See issue 38: when returning a write() op etcd has a bogus result. 60 | if self._prev_node.dir and not self.dir: 61 | self.dir = True 62 | 63 | def parse_headers(self, response): 64 | headers = response.getheaders() 65 | self.etcd_index = int(headers.get("x-etcd-index", 1)) 66 | self.raft_index = int(headers.get("x-raft-index", 1)) 67 | 68 | def get_subtree(self, leaves_only=False): 69 | """ 70 | Get all the subtree resulting from a recursive=true call to etcd. 71 | 72 | Args: 73 | leaves_only (bool): if true, only value nodes are returned 74 | 75 | 76 | """ 77 | if not self._children: 78 | # if the current result is a leaf, return itself 79 | yield self 80 | return 81 | else: 82 | # node is not a leaf 83 | if not leaves_only: 84 | yield self 85 | for n in self._children: 86 | node = EtcdResult(None, n) 87 | for child in node.get_subtree(leaves_only=leaves_only): 88 | yield child 89 | return 90 | 91 | @property 92 | def leaves(self): 93 | return self.get_subtree(leaves_only=True) 94 | 95 | @property 96 | def children(self): 97 | """Deprecated, use EtcdResult.leaves instead""" 98 | return self.leaves 99 | 100 | def __eq__(self, other): 101 | if not (type(self) is type(other)): 102 | return False 103 | for k in self._node_props.keys(): 104 | try: 105 | a = getattr(self, k) 106 | b = getattr(other, k) 107 | if a != b: 108 | return False 109 | except: 110 | return False 111 | return True 112 | 113 | def __ne__(self, other): 114 | return not self.__eq__(other) 115 | 116 | def __repr__(self): 117 | return "%s(%r)" % (self.__class__, self.__dict__) 118 | 119 | 120 | class EtcdException(Exception): 121 | 122 | """ 123 | Generic Etcd Exception. 124 | """ 125 | 126 | def __init__(self, message=None, payload=None): 127 | super(EtcdException, self).__init__(message) 128 | self.payload = payload 129 | 130 | 131 | class EtcdValueError(EtcdException, ValueError): 132 | """ 133 | Base class for Etcd value-related errors. 134 | """ 135 | 136 | pass 137 | 138 | 139 | class EtcdCompareFailed(EtcdValueError): 140 | """ 141 | Compare-and-swap failure 142 | """ 143 | 144 | pass 145 | 146 | 147 | class EtcdClusterIdChanged(EtcdException): 148 | """ 149 | The etcd cluster ID changed. This may indicate the cluster was replaced 150 | with a backup. Raised to prevent waiting on an etcd_index that was only 151 | valid on the old cluster. 152 | """ 153 | 154 | pass 155 | 156 | 157 | class EtcdKeyError(EtcdException): 158 | """ 159 | Etcd Generic KeyError Exception 160 | """ 161 | 162 | pass 163 | 164 | 165 | class EtcdKeyNotFound(EtcdKeyError): 166 | """ 167 | Etcd key not found exception (100) 168 | """ 169 | 170 | pass 171 | 172 | 173 | class EtcdNotFile(EtcdKeyError): 174 | """ 175 | Etcd not a file exception (102) 176 | """ 177 | 178 | pass 179 | 180 | 181 | class EtcdNotDir(EtcdKeyError): 182 | """ 183 | Etcd not a directory exception (104) 184 | """ 185 | 186 | pass 187 | 188 | 189 | class EtcdAlreadyExist(EtcdKeyError): 190 | """ 191 | Etcd already exist exception (105) 192 | """ 193 | 194 | pass 195 | 196 | 197 | class EtcdEventIndexCleared(EtcdException): 198 | """ 199 | Etcd event index is outdated and cleared exception (401) 200 | """ 201 | 202 | pass 203 | 204 | 205 | class EtcdConnectionFailed(EtcdException): 206 | """ 207 | Connection to etcd failed. 208 | """ 209 | 210 | def __init__(self, message=None, payload=None, cause=None): 211 | super(EtcdConnectionFailed, self).__init__(message=message, payload=payload) 212 | self.cause = cause 213 | 214 | 215 | class EtcdInsufficientPermissions(EtcdException): 216 | """ 217 | Request failed because of insufficient permissions. 218 | """ 219 | 220 | pass 221 | 222 | 223 | class EtcdWatchTimedOut(EtcdConnectionFailed): 224 | """ 225 | A watch timed out without returning a result. 226 | """ 227 | 228 | pass 229 | 230 | 231 | class EtcdWatcherCleared(EtcdException): 232 | """ 233 | Watcher is cleared due to etcd recovery. 234 | """ 235 | 236 | pass 237 | 238 | 239 | class EtcdLeaderElectionInProgress(EtcdException): 240 | """ 241 | Request failed due to in-progress leader election. 242 | """ 243 | 244 | pass 245 | 246 | 247 | class EtcdRootReadOnly(EtcdKeyError): 248 | """ 249 | Operation is not valid on the root, which is read only. 250 | """ 251 | 252 | pass 253 | 254 | 255 | class EtcdDirNotEmpty(EtcdValueError): 256 | """ 257 | Directory not empty. 258 | """ 259 | 260 | pass 261 | 262 | 263 | class EtcdLockExpired(EtcdException): 264 | """ 265 | Our lock apparently expired while we were trying to acquire it. 266 | """ 267 | 268 | pass 269 | 270 | 271 | class EtcdError(object): 272 | # See https://github.com/coreos/etcd/blob/master/Documentation/v2/errorcode.md 273 | error_exceptions = { 274 | 100: EtcdKeyNotFound, 275 | 101: EtcdCompareFailed, 276 | 102: EtcdNotFile, 277 | # 103: Non-public: no more peers. 278 | 104: EtcdNotDir, 279 | 105: EtcdAlreadyExist, 280 | # 106: Non-public: key is preserved. 281 | 107: EtcdRootReadOnly, 282 | 108: EtcdDirNotEmpty, 283 | # 109: Non-public: existing peer addr. 284 | 110: EtcdInsufficientPermissions, 285 | 200: EtcdValueError, # Not part of v2 286 | 201: EtcdValueError, 287 | 202: EtcdValueError, 288 | 203: EtcdValueError, 289 | 204: EtcdValueError, 290 | 205: EtcdValueError, 291 | 206: EtcdValueError, 292 | 207: EtcdValueError, 293 | 208: EtcdValueError, 294 | 209: EtcdValueError, 295 | 210: EtcdValueError, 296 | # 300: Non-public: Raft internal error. 297 | 301: EtcdLeaderElectionInProgress, 298 | 400: EtcdWatcherCleared, 299 | 401: EtcdEventIndexCleared, 300 | } 301 | 302 | @classmethod 303 | def handle(cls, payload): 304 | """ 305 | Decodes the error and throws the appropriate error message 306 | 307 | :param payload: The decoded JSON error payload as a dict. 308 | """ 309 | error_code = payload.get("errorCode") 310 | message = payload.get("message") 311 | cause = payload.get("cause") 312 | msg = "{} : {}".format(message, cause) 313 | status = payload.get("status") 314 | # Some general status handling, as 315 | # not all endpoints return coherent error messages 316 | if status == 404: 317 | error_code = 100 318 | elif status == 401: 319 | error_code = 110 320 | exc = cls.error_exceptions.get(error_code, EtcdException) 321 | if issubclass(exc, EtcdException): 322 | raise exc(msg, payload) 323 | else: 324 | raise exc(msg) 325 | 326 | 327 | # Attempt to enable urllib3's SNI support, if possible 328 | # Blatantly copied from requests. 329 | try: 330 | from urllib3.contrib import pyopenssl 331 | 332 | pyopenssl.inject_into_urllib3() 333 | except ImportError: 334 | pass 335 | -------------------------------------------------------------------------------- /src/etcd/auth.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import logging 4 | import etcd 5 | 6 | _log = logging.getLogger(__name__) 7 | 8 | 9 | class EtcdAuthBase(object): 10 | entity = "example" 11 | 12 | def __init__(self, client, name): 13 | self.client = client 14 | self.name = name 15 | self.uri = "{}/auth/{}s/{}".format(self.client.version_prefix, self.entity, self.name) 16 | # This will be lazily evaluated if not manually set 17 | self._legacy_api = None 18 | 19 | @property 20 | def legacy_api(self): 21 | if self._legacy_api is None: 22 | # The auth API has changed between 2.2 and 2.3, true story! 23 | major, minor = map(int, self.client.version[:3].split(".")) 24 | self._legacy_api = major < 3 and minor < 3 25 | return self._legacy_api 26 | 27 | @property 28 | def names(self): 29 | key = "{}s".format(self.entity) 30 | uri = "{}/auth/{}".format(self.client.version_prefix, key) 31 | response = self.client.api_execute(uri, self.client._MGET) 32 | if self.legacy_api: 33 | return json.loads(response.data.decode("utf-8"))[key] 34 | else: 35 | return [obj[self.entity] for obj in json.loads(response.data.decode("utf-8"))[key]] 36 | 37 | def read(self): 38 | try: 39 | response = self.client.api_execute(self.uri, self.client._MGET) 40 | except etcd.EtcdInsufficientPermissions as e: 41 | _log.error("Any action on the authorization requires the root role") 42 | raise 43 | except etcd.EtcdKeyNotFound: 44 | _log.info("%s '%s' not found", self.entity, self.name) 45 | raise 46 | except Exception as e: 47 | _log.error( 48 | "Failed to fetch %s in %s%s: %r", 49 | self.entity, 50 | self.client._base_uri, 51 | self.client.version_prefix, 52 | e, 53 | ) 54 | raise etcd.EtcdException("Could not fetch {} '{}'".format(self.entity, self.name)) 55 | 56 | self._from_net(response.data) 57 | 58 | def write(self): 59 | try: 60 | r = self.__class__(self.client, self.name) 61 | r.read() 62 | except etcd.EtcdKeyNotFound: 63 | r = None 64 | try: 65 | for payload in self._to_net(r): 66 | response = self.client.api_execute_json(self.uri, self.client._MPUT, params=payload) 67 | # This will fail if the response is an error 68 | self._from_net(response.data) 69 | except etcd.EtcdInsufficientPermissions as e: 70 | _log.error("Any action on the authorization requires the root role") 71 | raise 72 | except Exception as e: 73 | _log.error("Failed to write %s '%s'", self.entity, self.name) 74 | # TODO: fine-grained exception handling 75 | raise etcd.EtcdException( 76 | "Could not write {} '{}': {}".format(self.entity, self.name, e) 77 | ) 78 | 79 | def delete(self): 80 | try: 81 | _ = self.client.api_execute(self.uri, self.client._MDELETE) 82 | except etcd.EtcdInsufficientPermissions as e: 83 | _log.error("Any action on the authorization requires the root role") 84 | raise 85 | except etcd.EtcdKeyNotFound: 86 | _log.info("%s '%s' not found", self.entity, self.name) 87 | raise 88 | except Exception as e: 89 | _log.error( 90 | "Failed to delete %s in %s%s: %r", 91 | self.entity, 92 | self._base_uri, 93 | self.version_prefix, 94 | e, 95 | ) 96 | raise etcd.EtcdException("Could not delete {} '{}'".format(self.entity, self.name)) 97 | 98 | def _from_net(self, data): 99 | raise NotImplementedError() 100 | 101 | def _to_net(self, old=None): 102 | raise NotImplementedError() 103 | 104 | @classmethod 105 | def new(cls, client, data): 106 | c = cls(client, data[cls.entity]) 107 | c._from_net(data) 108 | return c 109 | 110 | 111 | class EtcdUser(EtcdAuthBase): 112 | """Class to manage in a orm-like way etcd users""" 113 | 114 | entity = "user" 115 | 116 | def __init__(self, client, name): 117 | super(EtcdUser, self).__init__(client, name) 118 | self._roles = set() 119 | self._password = None 120 | 121 | def _from_net(self, data): 122 | d = json.loads(data.decode("utf-8")) 123 | roles = d.get("roles", []) 124 | try: 125 | self.roles = roles 126 | except TypeError: 127 | # with the change of API, PUT responses are different 128 | # from GET reponses, which makes everything so funny. 129 | # Specifically, PUT responses are the same as before... 130 | if self.legacy_api: 131 | raise 132 | self.roles = [obj["role"] for obj in roles] 133 | self.name = d.get("user") 134 | 135 | def _to_net(self, prevobj=None): 136 | if prevobj is None: 137 | retval = [ 138 | { 139 | "user": self.name, 140 | "password": self._password, 141 | "roles": list(self.roles), 142 | } 143 | ] 144 | else: 145 | retval = [] 146 | if self._password: 147 | retval.append({"user": self.name, "password": self._password}) 148 | to_grant = list(self.roles - prevobj.roles) 149 | to_revoke = list(prevobj.roles - self.roles) 150 | if to_grant: 151 | retval.append({"user": self.name, "grant": to_grant}) 152 | if to_revoke: 153 | retval.append({"user": self.name, "revoke": to_revoke}) 154 | # Let's blank the password now 155 | # Even if the user can't be written we don't want it to leak anymore. 156 | self._password = None 157 | return retval 158 | 159 | @property 160 | def roles(self): 161 | return self._roles 162 | 163 | @roles.setter 164 | def roles(self, val): 165 | self._roles = set(val) 166 | 167 | @property 168 | def password(self): 169 | """Empty property for password.""" 170 | return None 171 | 172 | @password.setter 173 | def password(self, new_password): 174 | """Change user's password.""" 175 | self._password = new_password 176 | 177 | def __str__(self): 178 | return json.dumps(self._to_net()[0]) 179 | 180 | 181 | class EtcdRole(EtcdAuthBase): 182 | entity = "role" 183 | 184 | def __init__(self, client, name): 185 | super(EtcdRole, self).__init__(client, name) 186 | self._read_paths = set() 187 | self._write_paths = set() 188 | 189 | def _from_net(self, data): 190 | d = json.loads(data.decode("utf-8")) 191 | self.name = d.get("role") 192 | 193 | try: 194 | kv = d["permissions"]["kv"] 195 | except: 196 | self._read_paths = set() 197 | self._write_paths = set() 198 | return 199 | 200 | self._read_paths = set(kv.get("read", [])) 201 | self._write_paths = set(kv.get("write", [])) 202 | 203 | def _to_net(self, prevobj=None): 204 | retval = [] 205 | if prevobj is None: 206 | retval.append( 207 | { 208 | "role": self.name, 209 | "permissions": { 210 | "kv": { 211 | "read": list(self._read_paths), 212 | "write": list(self._write_paths), 213 | } 214 | }, 215 | } 216 | ) 217 | else: 218 | to_grant = { 219 | "read": list(self._read_paths - prevobj._read_paths), 220 | "write": list(self._write_paths - prevobj._write_paths), 221 | } 222 | to_revoke = { 223 | "read": list(prevobj._read_paths - self._read_paths), 224 | "write": list(prevobj._write_paths - self._write_paths), 225 | } 226 | if [path for sublist in to_revoke.values() for path in sublist]: 227 | retval.append({"role": self.name, "revoke": {"kv": to_revoke}}) 228 | if [path for sublist in to_grant.values() for path in sublist]: 229 | retval.append({"role": self.name, "grant": {"kv": to_grant}}) 230 | return retval 231 | 232 | def grant(self, path, permission): 233 | if permission.upper().find("R") >= 0: 234 | self._read_paths.add(path) 235 | if permission.upper().find("W") >= 0: 236 | self._write_paths.add(path) 237 | 238 | def revoke(self, path, permission): 239 | if permission.upper().find("R") >= 0 and path in self._read_paths: 240 | self._read_paths.remove(path) 241 | if permission.upper().find("W") >= 0 and path in self._write_paths: 242 | self._write_paths.remove(path) 243 | 244 | @property 245 | def acls(self): 246 | perms = {} 247 | try: 248 | for path in self._read_paths: 249 | perms[path] = "R" 250 | for path in self._write_paths: 251 | if path in perms: 252 | perms[path] += "W" 253 | else: 254 | perms[path] = "W" 255 | except: 256 | pass 257 | return perms 258 | 259 | @acls.setter 260 | def acls(self, acls): 261 | self._read_paths = set() 262 | self._write_paths = set() 263 | for path, permission in acls.items(): 264 | self.grant(path, permission) 265 | 266 | def __str__(self): 267 | return json.dumps({"role": self.name, "acls": self.acls}) 268 | 269 | 270 | class Auth(object): 271 | def __init__(self, client): 272 | self.client = client 273 | self.uri = "{}/auth/enable".format(self.client.version_prefix) 274 | 275 | @property 276 | def active(self): 277 | resp = self.client.api_execute(self.uri, self.client._MGET) 278 | return json.loads(resp.data.decode("utf-8"))["enabled"] 279 | 280 | @active.setter 281 | def active(self, value): 282 | if value != self.active: 283 | method = value and self.client._MPUT or self.client._MDELETE 284 | self.client.api_execute(self.uri, method) 285 | -------------------------------------------------------------------------------- /src/etcd/tests/integration/test_simple.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import shutil 4 | import logging 5 | import unittest 6 | import multiprocessing 7 | import tempfile 8 | 9 | import urllib3 10 | 11 | import etcd 12 | from . import helpers 13 | 14 | log = logging.getLogger() 15 | 16 | 17 | class EtcdIntegrationTest(unittest.TestCase): 18 | cl_size = 3 19 | 20 | @classmethod 21 | def setUpClass(cls): 22 | program = cls._get_exe() 23 | cls.directory = tempfile.mkdtemp(prefix="python-etcd") 24 | cls.processHelper = helpers.EtcdProcessHelper( 25 | cls.directory, 26 | proc_name=program, 27 | port_range_start=6001, 28 | internal_port_range_start=8001, 29 | ) 30 | cls.processHelper.run(number=cls.cl_size) 31 | cls.client = etcd.Client(port=6001) 32 | 33 | @classmethod 34 | def tearDownClass(cls): 35 | cls.processHelper.stop() 36 | shutil.rmtree(cls.directory) 37 | 38 | @classmethod 39 | def _is_exe(cls, fpath): 40 | return os.path.isfile(fpath) and os.access(fpath, os.X_OK) 41 | 42 | @classmethod 43 | def _get_exe(cls): 44 | PROGRAM = "etcd" 45 | 46 | program_path = None 47 | 48 | for path in os.environ["PATH"].split(os.pathsep): 49 | path = path.strip('"') 50 | exe_file = os.path.join(path, PROGRAM) 51 | if cls._is_exe(exe_file): 52 | program_path = exe_file 53 | break 54 | 55 | if not program_path: 56 | raise Exception("etcd not in path!!") 57 | 58 | return program_path 59 | 60 | 61 | class TestSimple(EtcdIntegrationTest): 62 | def test_machines(self): 63 | """INTEGRATION: retrieve machines""" 64 | self.assertEquals(self.client.machines[0], "http://127.0.0.1:6001") 65 | 66 | def test_leader(self): 67 | """INTEGRATION: retrieve leader""" 68 | self.assertIn( 69 | self.client.leader["clientURLs"][0], 70 | ["http://127.0.0.1:6001", "http://127.0.0.1:6002", "http://127.0.0.1:6003"], 71 | ) 72 | 73 | def test_get_set_delete(self): 74 | """INTEGRATION: set a new value""" 75 | try: 76 | get_result = self.client.get("/test_set") 77 | assert False 78 | except etcd.EtcdKeyNotFound as e: 79 | pass 80 | 81 | self.assertFalse("/test_set" in self.client) 82 | 83 | set_result = self.client.set("/test_set", "test-key") 84 | self.assertEquals("set", set_result.action.lower()) 85 | self.assertEquals("/test_set", set_result.key) 86 | self.assertEquals("test-key", set_result.value) 87 | 88 | self.assertTrue("/test_set" in self.client) 89 | 90 | get_result = self.client.get("/test_set") 91 | self.assertEquals("get", get_result.action.lower()) 92 | self.assertEquals("/test_set", get_result.key) 93 | self.assertEquals("test-key", get_result.value) 94 | 95 | delete_result = self.client.delete("/test_set") 96 | self.assertEquals("delete", delete_result.action.lower()) 97 | self.assertEquals("/test_set", delete_result.key) 98 | 99 | self.assertFalse("/test_set" in self.client) 100 | 101 | try: 102 | get_result = self.client.get("/test_set") 103 | assert False 104 | except etcd.EtcdKeyNotFound as e: 105 | pass 106 | 107 | def test_update(self): 108 | """INTEGRATION: update a value""" 109 | self.client.set("/foo", 3) 110 | c = self.client.get("/foo") 111 | c.value = int(c.value) + 3 112 | self.client.update(c) 113 | newres = self.client.get("/foo") 114 | self.assertEquals(newres.value, "6") 115 | self.assertRaises(ValueError, self.client.update, c) 116 | 117 | def test_retrieve_subkeys(self): 118 | """INTEGRATION: retrieve multiple subkeys""" 119 | set_result = self.client.write("/subtree/test_set", "test-key1") 120 | set_result = self.client.write("/subtree/test_set1", "test-key2") 121 | set_result = self.client.write("/subtree/test_set2", "test-key3") 122 | get_result = self.client.read("/subtree", recursive=True) 123 | result = [subkey.value for subkey in get_result.leaves] 124 | self.assertEquals(["test-key1", "test-key2", "test-key3"].sort(), result.sort()) 125 | 126 | def test_directory_ttl_update(self): 127 | """INTEGRATION: should be able to update a dir TTL""" 128 | self.client.write("/dir", None, dir=True, ttl=30) 129 | res = self.client.write("/dir", None, dir=True, ttl=31, prevExist=True) 130 | self.assertEquals(res.ttl, 31) 131 | res = self.client.get("/dir") 132 | res.ttl = 120 133 | new_res = self.client.update(res) 134 | self.assertEquals(new_res.ttl, 120) 135 | 136 | 137 | class TestErrors(EtcdIntegrationTest): 138 | def test_is_not_a_file(self): 139 | """INTEGRATION: try to write value to an existing directory""" 140 | 141 | self.client.set("/directory/test-key", "test-value") 142 | self.assertRaises(etcd.EtcdNotFile, self.client.set, "/directory", "test-value") 143 | 144 | def test_test_and_set(self): 145 | """INTEGRATION: try test_and_set operation""" 146 | 147 | set_result = self.client.set("/test-key", "old-test-value") 148 | 149 | set_result = self.client.test_and_set("/test-key", "test-value", "old-test-value") 150 | 151 | self.assertRaises( 152 | ValueError, 153 | self.client.test_and_set, 154 | "/test-key", 155 | "new-value", 156 | "old-test-value", 157 | ) 158 | 159 | def test_creating_already_existing_directory(self): 160 | """INTEGRATION: creating an already existing directory without 161 | `prevExist=True` should fail""" 162 | self.client.write("/mydir", None, dir=True) 163 | 164 | self.assertRaises(etcd.EtcdNotFile, self.client.write, "/mydir", None, dir=True) 165 | self.assertRaises( 166 | etcd.EtcdAlreadyExist, 167 | self.client.write, 168 | "/mydir", 169 | None, 170 | dir=True, 171 | prevExist=False, 172 | ) 173 | 174 | 175 | class TestClusterFunctions(EtcdIntegrationTest): 176 | @classmethod 177 | def setUpClass(cls): 178 | program = cls._get_exe() 179 | cls.directory = tempfile.mkdtemp(prefix="python-etcd") 180 | 181 | cls.processHelper = helpers.EtcdProcessHelper( 182 | cls.directory, 183 | proc_name=program, 184 | port_range_start=6001, 185 | internal_port_range_start=8001, 186 | cluster=True, 187 | ) 188 | 189 | def test_reconnect(self): 190 | """INTEGRATION: get key after the server we're connected fails.""" 191 | self.processHelper.stop() 192 | self.processHelper.run(number=3) 193 | self.client = etcd.Client(port=6001, allow_reconnect=True) 194 | set_result = self.client.set("/test_set", "test-key1") 195 | get_result = self.client.get("/test_set") 196 | 197 | self.assertEquals("test-key1", get_result.value) 198 | 199 | self.processHelper.kill_one(0) 200 | 201 | get_result = self.client.get("/test_set") 202 | self.assertEquals("test-key1", get_result.value) 203 | 204 | def test_reconnect_with_several_hosts_passed(self): 205 | """INTEGRATION: receive several hosts at connection setup.""" 206 | self.processHelper.stop() 207 | self.processHelper.run(number=3) 208 | self.client = etcd.Client( 209 | host=(("127.0.0.1", 6004), ("127.0.0.1", 6001)), allow_reconnect=True 210 | ) 211 | set_result = self.client.set("/test_set", "test-key1") 212 | get_result = self.client.get("/test_set") 213 | 214 | self.assertEquals("test-key1", get_result.value) 215 | 216 | self.processHelper.kill_one(0) 217 | 218 | get_result = self.client.get("/test_set") 219 | self.assertEquals("test-key1", get_result.value) 220 | 221 | def test_reconnect_not_allowed(self): 222 | """INTEGRATION: fail on server kill if not allow_reconnect""" 223 | self.processHelper.stop() 224 | self.processHelper.run(number=3) 225 | self.client = etcd.Client(port=6001, allow_reconnect=False) 226 | self.processHelper.kill_one(0) 227 | self.assertRaises(etcd.EtcdConnectionFailed, self.client.get, "/test_set") 228 | 229 | def test_reconnet_fails(self): 230 | """INTEGRATION: fails to reconnect if no available machines""" 231 | self.processHelper.stop() 232 | # Start with three instances (0, 1, 2) 233 | self.processHelper.run(number=3) 234 | # Connect to instance 0 235 | self.client = etcd.Client(port=6001, allow_reconnect=True) 236 | set_result = self.client.set("/test_set", "test-key1") 237 | 238 | get_result = self.client.get("/test_set") 239 | self.assertEquals("test-key1", get_result.value) 240 | self.processHelper.kill_one(2) 241 | self.processHelper.kill_one(1) 242 | self.processHelper.kill_one(0) 243 | self.assertRaises(etcd.EtcdException, self.client.get, "/test_set") 244 | 245 | 246 | class TestWatch(EtcdIntegrationTest): 247 | def test_watch(self): 248 | """INTEGRATION: Receive a watch event from other process""" 249 | 250 | set_result = self.client.set("/test-key", "test-value") 251 | 252 | queue = multiprocessing.Queue() 253 | 254 | def change_value(key, newValue): 255 | c = etcd.Client(port=6001) 256 | c.set(key, newValue) 257 | 258 | def watch_value(key, queue): 259 | c = etcd.Client(port=6001) 260 | queue.put(c.watch(key).value) 261 | 262 | changer = multiprocessing.Process( 263 | target=change_value, 264 | args=( 265 | "/test-key", 266 | "new-test-value", 267 | ), 268 | ) 269 | 270 | watcher = multiprocessing.Process(target=watch_value, args=("/test-key", queue)) 271 | 272 | watcher.start() 273 | time.sleep(1) 274 | 275 | changer.start() 276 | 277 | value = queue.get(timeout=2) 278 | watcher.join(timeout=5) 279 | changer.join(timeout=5) 280 | 281 | assert value == "new-test-value" 282 | 283 | def test_watch_indexed(self): 284 | """INTEGRATION: Receive a watch event from other process, indexed""" 285 | 286 | set_result = self.client.set("/test-key", "test-value") 287 | set_result = self.client.set("/test-key", "test-value0") 288 | original_index = int(set_result.modifiedIndex) 289 | set_result = self.client.set("/test-key", "test-value1") 290 | set_result = self.client.set("/test-key", "test-value2") 291 | 292 | queue = multiprocessing.Queue() 293 | 294 | def change_value(key, newValue): 295 | c = etcd.Client(port=6001) 296 | c.set(key, newValue) 297 | c.get(key) 298 | 299 | def watch_value(key, index, queue): 300 | c = etcd.Client(port=6001) 301 | for i in range(0, 3): 302 | queue.put(c.watch(key, index=index + i).value) 303 | 304 | proc = multiprocessing.Process( 305 | target=change_value, 306 | args=( 307 | "/test-key", 308 | "test-value3", 309 | ), 310 | ) 311 | 312 | watcher = multiprocessing.Process( 313 | target=watch_value, args=("/test-key", original_index, queue) 314 | ) 315 | 316 | watcher.start() 317 | time.sleep(0.5) 318 | 319 | proc.start() 320 | 321 | for i in range(0, 3): 322 | value = queue.get() 323 | log.debug("index: %d: %s" % (i, value)) 324 | self.assertEquals("test-value%d" % i, value) 325 | 326 | watcher.join(timeout=5) 327 | proc.join(timeout=5) 328 | 329 | def test_watch_generator(self): 330 | """INTEGRATION: Receive a watch event from other process (gen)""" 331 | 332 | set_result = self.client.set("/test-key", "test-value") 333 | 334 | queue = multiprocessing.Queue() 335 | 336 | def change_value(key): 337 | time.sleep(0.5) 338 | c = etcd.Client(port=6001) 339 | for i in range(0, 3): 340 | c.set(key, "test-value%d" % i) 341 | c.get(key) 342 | 343 | def watch_value(key, queue): 344 | c = etcd.Client(port=6001) 345 | for i in range(0, 3): 346 | event = next(c.eternal_watch(key)).value 347 | queue.put(event) 348 | 349 | changer = multiprocessing.Process(target=change_value, args=("/test-key",)) 350 | 351 | watcher = multiprocessing.Process(target=watch_value, args=("/test-key", queue)) 352 | 353 | watcher.start() 354 | changer.start() 355 | 356 | values = ["test-value0", "test-value1", "test-value2"] 357 | for i in range(0, 1): 358 | value = queue.get() 359 | log.debug("index: %d: %s" % (i, value)) 360 | self.assertTrue(value in values) 361 | 362 | watcher.join(timeout=5) 363 | changer.join(timeout=5) 364 | 365 | def test_watch_indexed_generator(self): 366 | """INTEGRATION: Receive a watch event from other process, ixd, (2)""" 367 | 368 | set_result = self.client.set("/test-key", "test-value") 369 | set_result = self.client.set("/test-key", "test-value0") 370 | original_index = int(set_result.modifiedIndex) 371 | set_result = self.client.set("/test-key", "test-value1") 372 | set_result = self.client.set("/test-key", "test-value2") 373 | 374 | queue = multiprocessing.Queue() 375 | 376 | def change_value(key, newValue): 377 | c = etcd.Client(port=6001) 378 | c.set(key, newValue) 379 | 380 | def watch_value(key, index, queue): 381 | c = etcd.Client(port=6001) 382 | iterevents = c.eternal_watch(key, index=index) 383 | for i in range(0, 3): 384 | queue.put(next(iterevents).value) 385 | 386 | proc = multiprocessing.Process( 387 | target=change_value, 388 | args=( 389 | "/test-key", 390 | "test-value3", 391 | ), 392 | ) 393 | 394 | watcher = multiprocessing.Process( 395 | target=watch_value, args=("/test-key", original_index, queue) 396 | ) 397 | 398 | watcher.start() 399 | time.sleep(0.5) 400 | proc.start() 401 | 402 | for i in range(0, 3): 403 | value = queue.get() 404 | log.debug("index: %d: %s" % (i, value)) 405 | self.assertEquals("test-value%d" % i, value) 406 | 407 | watcher.join(timeout=5) 408 | proc.join(timeout=5) 409 | -------------------------------------------------------------------------------- /src/etcd/tests/unit/test_old_request.py: -------------------------------------------------------------------------------- 1 | import etcd 2 | import unittest 3 | 4 | try: 5 | import mock 6 | except ImportError: 7 | from unittest import mock 8 | 9 | from etcd import EtcdException 10 | 11 | 12 | class FakeHTTPResponse(object): 13 | def __init__(self, status, data="", headers=None): 14 | self.status = status 15 | self.data = data.encode("utf-8") 16 | self.headers = headers or { 17 | "x-etcd-cluster-id": "abdef12345", 18 | } 19 | 20 | def getheaders(self): 21 | return self.headers 22 | 23 | def getheader(self, header): 24 | return self.headers[header] 25 | 26 | 27 | class TestClientRequest(unittest.TestCase): 28 | def test_set(self): 29 | """Can set a value""" 30 | client = etcd.Client() 31 | client.api_execute = mock.Mock( 32 | return_value=FakeHTTPResponse( 33 | 201, 34 | '{"action":"SET",' 35 | '"node": {' 36 | '"key":"/testkey",' 37 | '"value":"test",' 38 | '"newKey":true,' 39 | '"expiration":"2013-09-14T00:56:59.316195568+02:00",' 40 | '"ttl":19,"modifiedIndex":183}}', 41 | ) 42 | ) 43 | 44 | result = client.set("/testkey", "test", ttl=19) 45 | 46 | self.assertEqual( 47 | etcd.EtcdResult( 48 | **{ 49 | "action": "SET", 50 | "node": { 51 | "expiration": "2013-09-14T00:56:59.316195568+02:00", 52 | "modifiedIndex": 183, 53 | "key": "/testkey", 54 | "newKey": True, 55 | "ttl": 19, 56 | "value": "test", 57 | }, 58 | } 59 | ), 60 | result, 61 | ) 62 | 63 | def test_test_and_set(self): 64 | """Can test and set a value""" 65 | client = etcd.Client() 66 | client.api_execute = mock.Mock( 67 | return_value=FakeHTTPResponse( 68 | 200, 69 | '{"action":"SET",' 70 | '"node": {' 71 | '"key":"/testkey",' 72 | '"prevValue":"test",' 73 | '"value":"newvalue",' 74 | '"expiration":"2013-09-14T02:09:44.24390976+02:00",' 75 | '"ttl":49,"modifiedIndex":203}}', 76 | ) 77 | ) 78 | result = client.test_and_set("/testkey", "newvalue", "test", ttl=19) 79 | self.assertEqual( 80 | etcd.EtcdResult( 81 | **{ 82 | "action": "SET", 83 | "node": { 84 | "expiration": "2013-09-14T02:09:44.24390976+02:00", 85 | "modifiedIndex": 203, 86 | "key": "/testkey", 87 | "prevValue": "test", 88 | "ttl": 49, 89 | "value": "newvalue", 90 | }, 91 | } 92 | ), 93 | result, 94 | ) 95 | 96 | def test_test_and_test_failure(self): 97 | """Exception will be raised if prevValue != value in test_set""" 98 | 99 | client = etcd.Client() 100 | client.api_execute = mock.Mock( 101 | side_effect=ValueError( 102 | "The given PrevValue is not equal" " to the value of the key : TestAndSet: 1!=3" 103 | ) 104 | ) 105 | try: 106 | result = client.test_and_set("/testkey", "newvalue", "test", ttl=19) 107 | except ValueError as e: 108 | # from ipdb import set_trace; set_trace() 109 | self.assertEqual( 110 | "The given PrevValue is not equal" " to the value of the key : TestAndSet: 1!=3", 111 | str(e), 112 | ) 113 | 114 | def test_delete(self): 115 | """Can delete a value""" 116 | client = etcd.Client() 117 | client.api_execute = mock.Mock( 118 | return_value=FakeHTTPResponse( 119 | 200, 120 | '{"action":"DELETE",' 121 | '"node": {' 122 | '"key":"/testkey",' 123 | '"prevValue":"test",' 124 | '"expiration":"2013-09-14T01:06:35.5242587+02:00",' 125 | '"modifiedIndex":189}}', 126 | ) 127 | ) 128 | result = client.delete("/testkey") 129 | self.assertEqual( 130 | etcd.EtcdResult( 131 | **{ 132 | "action": "DELETE", 133 | "node": { 134 | "expiration": "2013-09-14T01:06:35.5242587+02:00", 135 | "modifiedIndex": 189, 136 | "key": "/testkey", 137 | "prevValue": "test", 138 | }, 139 | } 140 | ), 141 | result, 142 | ) 143 | 144 | def test_get(self): 145 | """Can get a value""" 146 | client = etcd.Client() 147 | client.api_execute = mock.Mock( 148 | return_value=FakeHTTPResponse( 149 | 200, 150 | '{"action":"GET",' 151 | '"node": {' 152 | '"key":"/testkey",' 153 | '"value":"test",' 154 | '"modifiedIndex":190}}', 155 | ) 156 | ) 157 | 158 | result = client.get("/testkey") 159 | self.assertEqual( 160 | etcd.EtcdResult( 161 | **{ 162 | "action": "GET", 163 | "node": {"modifiedIndex": 190, "key": "/testkey", "value": "test"}, 164 | } 165 | ), 166 | result, 167 | ) 168 | 169 | def test_get_multi(self): 170 | """Can get multiple values""" 171 | pass 172 | 173 | def test_get_subdirs(self): 174 | """Can understand dirs in results""" 175 | pass 176 | 177 | def test_not_in(self): 178 | """Can check if key is not in client""" 179 | client = etcd.Client() 180 | client.get = mock.Mock(side_effect=etcd.EtcdKeyNotFound()) 181 | result = "/testkey" not in client 182 | self.assertEqual(True, result) 183 | 184 | def test_in(self): 185 | """Can check if key is in client""" 186 | client = etcd.Client() 187 | client.api_execute = mock.Mock( 188 | return_value=FakeHTTPResponse( 189 | 200, 190 | '{"action":"GET",' 191 | '"node": {' 192 | '"key":"/testkey",' 193 | '"value":"test",' 194 | '"modifiedIndex":190}}', 195 | ) 196 | ) 197 | result = "/testkey" in client 198 | 199 | self.assertEqual(True, result) 200 | 201 | def test_simple_watch(self): 202 | """Can watch values""" 203 | client = etcd.Client() 204 | client.api_execute = mock.Mock( 205 | return_value=FakeHTTPResponse( 206 | 200, 207 | '{"action":"SET",' 208 | '"node": {' 209 | '"key":"/testkey",' 210 | '"value":"test",' 211 | '"newKey":true,' 212 | '"expiration":"2013-09-14T01:35:07.623681365+02:00",' 213 | '"ttl":19,' 214 | '"modifiedIndex":192}}', 215 | ) 216 | ) 217 | result = client.watch("/testkey") 218 | self.assertEqual( 219 | etcd.EtcdResult( 220 | **{ 221 | "action": "SET", 222 | "node": { 223 | "expiration": "2013-09-14T01:35:07.623681365+02:00", 224 | "modifiedIndex": 192, 225 | "key": "/testkey", 226 | "newKey": True, 227 | "ttl": 19, 228 | "value": "test", 229 | }, 230 | } 231 | ), 232 | result, 233 | ) 234 | 235 | def test_index_watch(self): 236 | """Can watch values from index""" 237 | client = etcd.Client() 238 | client.api_execute = mock.Mock( 239 | return_value=FakeHTTPResponse( 240 | 200, 241 | '{"action":"SET",' 242 | '"node": {' 243 | '"key":"/testkey",' 244 | '"value":"test",' 245 | '"newKey":true,' 246 | '"expiration":"2013-09-14T01:35:07.623681365+02:00",' 247 | '"ttl":19,' 248 | '"modifiedIndex":180}}', 249 | ) 250 | ) 251 | result = client.watch("/testkey", index=180) 252 | self.assertEqual( 253 | etcd.EtcdResult( 254 | **{ 255 | "action": "SET", 256 | "node": { 257 | "expiration": "2013-09-14T01:35:07.623681365+02:00", 258 | "modifiedIndex": 180, 259 | "key": "/testkey", 260 | "newKey": True, 261 | "ttl": 19, 262 | "value": "test", 263 | }, 264 | } 265 | ), 266 | result, 267 | ) 268 | 269 | 270 | class TestEventGenerator(object): 271 | def check_watch(self, result): 272 | assert ( 273 | etcd.EtcdResult( 274 | **{ 275 | "action": "SET", 276 | "node": { 277 | "expiration": "2013-09-14T01:35:07.623681365+02:00", 278 | "modifiedIndex": 180, 279 | "key": "/testkey", 280 | "newKey": True, 281 | "ttl": 19, 282 | "value": "test", 283 | }, 284 | } 285 | ) 286 | == result 287 | ) 288 | 289 | def test_eternal_watch(self): 290 | """Can watch values from generator""" 291 | client = etcd.Client() 292 | client.api_execute = mock.Mock( 293 | return_value=FakeHTTPResponse( 294 | 200, 295 | '{"action":"SET",' 296 | '"node": {' 297 | '"key":"/testkey",' 298 | '"value":"test",' 299 | '"newKey":true,' 300 | '"expiration":"2013-09-14T01:35:07.623681365+02:00",' 301 | '"ttl":19,' 302 | '"modifiedIndex":180}}', 303 | ) 304 | ) 305 | for result in range(1, 5): 306 | result = next(client.eternal_watch("/testkey", index=180)) 307 | self.check_watch(result) 308 | 309 | 310 | class TestClientApiExecutor(unittest.TestCase): 311 | def test_get(self): 312 | """http get request""" 313 | client = etcd.Client() 314 | response = FakeHTTPResponse(status=200, data="arbitrary json data") 315 | client.http.request = mock.Mock(return_value=response) 316 | result = client.api_execute("/v1/keys/testkey", client._MGET) 317 | self.assertEqual("arbitrary json data".encode("utf-8"), result.data) 318 | 319 | def test_delete(self): 320 | """http delete request""" 321 | client = etcd.Client() 322 | response = FakeHTTPResponse(status=200, data="arbitrary json data") 323 | client.http.request = mock.Mock(return_value=response) 324 | result = client.api_execute("/v1/keys/testkey", client._MDELETE) 325 | self.assertEqual("arbitrary json data".encode("utf-8"), result.data) 326 | 327 | def test_get_error(self): 328 | """http get error request 101""" 329 | client = etcd.Client() 330 | response = FakeHTTPResponse( 331 | status=400, 332 | data='{"message": "message",' ' "cause": "cause",' ' "errorCode": 100}', 333 | ) 334 | client.http.request = mock.Mock(return_value=response) 335 | try: 336 | client.api_execute("/v2/keys/testkey", client._MGET) 337 | assert False 338 | except etcd.EtcdKeyNotFound as e: 339 | self.assertEqual(str(e), "message : cause") 340 | 341 | def test_put(self): 342 | """http put request""" 343 | client = etcd.Client() 344 | response = FakeHTTPResponse(status=200, data="arbitrary json data") 345 | client.http.request_encode_body = mock.Mock(return_value=response) 346 | result = client.api_execute("/v2/keys/testkey", client._MPUT) 347 | self.assertEqual("arbitrary json data".encode("utf-8"), result.data) 348 | 349 | def test_test_and_set_error(self): 350 | """http post error request 101""" 351 | client = etcd.Client() 352 | response = FakeHTTPResponse( 353 | status=400, 354 | data='{"message": "message", "cause": "cause", "errorCode": 101}', 355 | ) 356 | client.http.request_encode_body = mock.Mock(return_value=response) 357 | payload = {"value": "value", "prevValue": "oldValue", "ttl": "60"} 358 | try: 359 | client.api_execute("/v2/keys/testkey", client._MPUT, payload) 360 | self.fail() 361 | except ValueError as e: 362 | self.assertEqual("message : cause", str(e)) 363 | 364 | def test_set_not_file_error(self): 365 | """http post error request 102""" 366 | client = etcd.Client() 367 | response = FakeHTTPResponse( 368 | status=400, 369 | data='{"message": "message", "cause": "cause", "errorCode": 102}', 370 | ) 371 | client.http.request_encode_body = mock.Mock(return_value=response) 372 | payload = {"value": "value", "prevValue": "oldValue", "ttl": "60"} 373 | try: 374 | client.api_execute("/v2/keys/testkey", client._MPUT, payload) 375 | self.fail() 376 | except etcd.EtcdNotFile as e: 377 | self.assertEqual("message : cause", str(e)) 378 | 379 | def test_get_error_unknown(self): 380 | """http get error request unknown""" 381 | client = etcd.Client() 382 | response = FakeHTTPResponse( 383 | status=400, 384 | data='{"message": "message",' ' "cause": "cause",' ' "errorCode": 42}', 385 | ) 386 | client.http.request = mock.Mock(return_value=response) 387 | try: 388 | client.api_execute("/v2/keys/testkey", client._MGET) 389 | self.fail() 390 | except etcd.EtcdException as e: 391 | self.assertEqual(str(e), "message : cause") 392 | 393 | def test_get_error_request_invalid(self): 394 | """http get error request invalid""" 395 | client = etcd.Client() 396 | response = FakeHTTPResponse(status=400, data="{)*garbage") 397 | client.http.request = mock.Mock(return_value=response) 398 | try: 399 | client.api_execute("/v2/keys/testkey", client._MGET) 400 | self.fail() 401 | except etcd.EtcdException as e: 402 | self.assertEqual(str(e), "Bad response : {)*garbage") 403 | 404 | def test_get_error_invalid(self): 405 | """http get error request invalid""" 406 | client = etcd.Client() 407 | response = FakeHTTPResponse(status=400, data="{){){)*garbage*") 408 | client.http.request = mock.Mock(return_value=response) 409 | self.assertRaises(etcd.EtcdException, client.api_execute, "/v2/keys/testkey", client._MGET) 410 | -------------------------------------------------------------------------------- /src/etcd/tests/unit/test_request.py: -------------------------------------------------------------------------------- 1 | import socket 2 | import urllib3 3 | 4 | import etcd 5 | from etcd.tests.unit import TestClientApiBase 6 | 7 | try: 8 | import mock 9 | except ImportError: 10 | from unittest import mock 11 | 12 | 13 | class TestClientApiInternals(TestClientApiBase): 14 | def test_read_default_timeout(self): 15 | """Read timeout set to the default""" 16 | d = { 17 | "action": "get", 18 | "node": {"modifiedIndex": 190, "key": "/testkey", "value": "test"}, 19 | } 20 | self._mock_api(200, d) 21 | res = self.client.read("/testkey") 22 | self.assertEqual(self.client.api_execute.call_args[1]["timeout"], None) 23 | 24 | def test_read_custom_timeout(self): 25 | """Read timeout set to the supplied value""" 26 | d = { 27 | "action": "get", 28 | "node": {"modifiedIndex": 190, "key": "/testkey", "value": "test"}, 29 | } 30 | self._mock_api(200, d) 31 | self.client.read("/testkey", timeout=15) 32 | self.assertEqual(self.client.api_execute.call_args[1]["timeout"], 15) 33 | 34 | def test_read_no_timeout(self): 35 | """Read timeout disabled""" 36 | d = { 37 | "action": "get", 38 | "node": {"modifiedIndex": 190, "key": "/testkey", "value": "test"}, 39 | } 40 | self._mock_api(200, d) 41 | self.client.read("/testkey", timeout=0) 42 | self.assertEqual(self.client.api_execute.call_args[1]["timeout"], 0) 43 | 44 | def test_write_no_params(self): 45 | """Calling `write` without a value argument will omit the `value` from 46 | the API call params""" 47 | d = { 48 | "action": "set", 49 | "node": { 50 | "createdIndex": 17, 51 | "dir": True, 52 | "key": "/newdir", 53 | "modifiedIndex": 17, 54 | }, 55 | } 56 | self._mock_api(200, d) 57 | self.client.write("/newdir", None, dir=True) 58 | self.assertEqual( 59 | self.client.api_execute.call_args, 60 | (("/v2/keys/newdir", "PUT"), dict(params={"dir": "true"})), 61 | ) 62 | 63 | 64 | class TestClientApiInterface(TestClientApiBase): 65 | """ 66 | All tests defined in this class are executed also in TestClientRequest. 67 | 68 | If a test should be run only in this class, please override the method there. 69 | """ 70 | 71 | def test_machines(self): 72 | """Can request machines""" 73 | data = [ 74 | "http://127.0.0.1:4001", 75 | "http://127.0.0.1:4002", 76 | "http://127.0.0.1:4003", 77 | ] 78 | d = ",".join(data) 79 | self.client.http.request = mock.MagicMock(return_value=self._prepare_response(200, d)) 80 | self.assertEqual(data, self.client.machines) 81 | 82 | @mock.patch("etcd.Client.machines", new_callable=mock.PropertyMock) 83 | def test_use_proxies(self, mocker): 84 | """Do not overwrite the machines cache when using proxies""" 85 | mocker.return_value = [ 86 | "https://10.0.0.2:4001", 87 | "https://10.0.0.3:4001", 88 | "https://10.0.0.4:4001", 89 | ] 90 | c = etcd.Client( 91 | host=(("localhost", 4001), ("localproxy", 4001)), 92 | protocol="https", 93 | allow_reconnect=True, 94 | use_proxies=True, 95 | ) 96 | 97 | self.assertEqual(c._machines_cache, ["https://localproxy:4001"]) 98 | self.assertEqual(c._base_uri, "https://localhost:4001") 99 | self.assertNotIn(c.base_uri, c._machines_cache) 100 | 101 | c = etcd.Client( 102 | host=(("localhost", 4001), ("10.0.0.2", 4001)), 103 | protocol="https", 104 | allow_reconnect=True, 105 | use_proxies=False, 106 | ) 107 | self.assertIn("https://10.0.0.3:4001", c._machines_cache) 108 | self.assertNotIn(c.base_uri, c._machines_cache) 109 | 110 | def test_members(self): 111 | """Can request machines""" 112 | data = { 113 | "members": [ 114 | { 115 | "id": "ce2a822cea30bfca", 116 | "name": "default", 117 | "peerURLs": ["http://localhost:2380", "http://localhost:7001"], 118 | "clientURLs": ["http://127.0.0.1:4001"], 119 | } 120 | ] 121 | } 122 | self._mock_api(200, data) 123 | self.assertEqual(self.client.members["ce2a822cea30bfca"]["id"], "ce2a822cea30bfca") 124 | 125 | def test_self_stats(self): 126 | """Request for stats""" 127 | data = { 128 | "id": "eca0338f4ea31566", 129 | "leaderInfo": { 130 | "leader": "8a69d5f6b7814500", 131 | "startTime": "2014-10-24T13:15:51.186620747-07:00", 132 | "uptime": "10m59.322358947s", 133 | }, 134 | "name": "node3", 135 | "recvAppendRequestCnt": 5944, 136 | "recvBandwidthRate": 570.6254930219969, 137 | "recvPkgRate": 9.00892789741075, 138 | "sendAppendRequestCnt": 0, 139 | "startTime": "2014-10-24T13:15:50.072007085-07:00", 140 | "state": "StateFollower", 141 | } 142 | self._mock_api(200, data) 143 | self.assertEqual(self.client.stats["name"], "node3") 144 | 145 | def test_leader_stats(self): 146 | """Request for leader stats""" 147 | data = {"leader": "924e2e83e93f2560", "followers": {}} 148 | self._mock_api(200, data) 149 | self.assertEqual(self.client.leader_stats["leader"], "924e2e83e93f2560") 150 | 151 | @mock.patch("etcd.Client.members", new_callable=mock.PropertyMock) 152 | def test_leader(self, mocker): 153 | """Can request the leader""" 154 | members = {"ce2a822cea30bfca": {"id": "ce2a822cea30bfca", "name": "default"}} 155 | mocker.return_value = members 156 | self._mock_api(200, {"leaderInfo": {"leader": "ce2a822cea30bfca", "followers": {}}}) 157 | self.assertEqual(self.client.leader, members["ce2a822cea30bfca"]) 158 | 159 | def test_set_plain(self): 160 | """Can set a value""" 161 | d = { 162 | "action": "set", 163 | "node": { 164 | "expiration": "2013-09-14T00:56:59.316195568+02:00", 165 | "modifiedIndex": 183, 166 | "key": "/testkey", 167 | "ttl": 19, 168 | "value": "test", 169 | }, 170 | } 171 | 172 | self._mock_api(200, d) 173 | res = self.client.write("/testkey", "test") 174 | self.assertEqual(res, etcd.EtcdResult(**d)) 175 | 176 | def test_update(self): 177 | """Can update a result.""" 178 | d = { 179 | "action": "set", 180 | "node": { 181 | "expiration": "2013-09-14T00:56:59.316195568+02:00", 182 | "modifiedIndex": 6, 183 | "key": "/testkey", 184 | "ttl": 19, 185 | "value": "test", 186 | }, 187 | } 188 | self._mock_api(200, d) 189 | res = self.client.get("/testkey") 190 | res.value = "ciao" 191 | d["node"]["value"] = "ciao" 192 | self._mock_api(200, d) 193 | newres = self.client.update(res) 194 | self.assertEqual(newres.value, "ciao") 195 | 196 | def test_newkey(self): 197 | """Can set a new value""" 198 | d = { 199 | "action": "set", 200 | "node": { 201 | "expiration": "2013-09-14T00:56:59.316195568+02:00", 202 | "modifiedIndex": 183, 203 | "key": "/testkey", 204 | "ttl": 19, 205 | "value": "test", 206 | }, 207 | } 208 | self._mock_api(201, d) 209 | res = self.client.write("/testkey", "test") 210 | d["node"]["newKey"] = True 211 | self.assertEqual(res, etcd.EtcdResult(**d)) 212 | 213 | def test_refresh(self): 214 | """Can refresh a new value""" 215 | d = { 216 | "action": "update", 217 | "node": { 218 | "expiration": "2016-05-31T08:27:54.660337Z", 219 | "modifiedIndex": 183, 220 | "key": "/testkey", 221 | "ttl": 600, 222 | "value": "test", 223 | }, 224 | } 225 | 226 | self._mock_api(200, d) 227 | res = self.client.refresh("/testkey", ttl=600) 228 | self.assertEqual(res, etcd.EtcdResult(**d)) 229 | 230 | def test_not_found_response(self): 231 | """Can handle server not found response""" 232 | self._mock_api(404, "Not found") 233 | self.assertRaises(etcd.EtcdException, self.client.read, "/somebadkey") 234 | 235 | def test_compare_and_swap(self): 236 | """Can set compare-and-swap a value""" 237 | d = { 238 | "action": "compareAndSwap", 239 | "node": { 240 | "expiration": "2013-09-14T00:56:59.316195568+02:00", 241 | "modifiedIndex": 183, 242 | "key": "/testkey", 243 | "ttl": 19, 244 | "value": "test", 245 | }, 246 | } 247 | 248 | self._mock_api(200, d) 249 | res = self.client.write("/testkey", "test", prevValue="test_old") 250 | self.assertEqual(res, etcd.EtcdResult(**d)) 251 | 252 | def test_compare_and_swap_failure(self): 253 | """Exception will be raised if prevValue != value in test_set""" 254 | self._mock_exception(ValueError, "Test Failed : [ 1!=3 ]") 255 | self.assertRaises(ValueError, self.client.write, "/testKey", "test", prevValue="oldbog") 256 | 257 | def test_set_append(self): 258 | """Can append a new key""" 259 | d = { 260 | "action": "create", 261 | "node": { 262 | "createdIndex": 190, 263 | "modifiedIndex": 190, 264 | "key": "/testdir/190", 265 | "value": "test", 266 | }, 267 | } 268 | self._mock_api(201, d) 269 | res = self.client.write("/testdir", "test") 270 | self.assertEqual(res.createdIndex, 190) 271 | 272 | def test_set_dir_with_value(self): 273 | """Creating a directory with a value raises an error.""" 274 | self.assertRaises(etcd.EtcdException, self.client.write, "/bar", "testvalye", dir=True) 275 | 276 | def test_delete(self): 277 | """Can delete a value""" 278 | d = { 279 | "action": "delete", 280 | "node": {"key": "/testkey", "modifiedIndex": 3, "createdIndex": 2}, 281 | } 282 | self._mock_api(200, d) 283 | res = self.client.delete("/testKey") 284 | self.assertEqual(res, etcd.EtcdResult(**d)) 285 | 286 | def test_pop(self): 287 | """Can pop a value""" 288 | d = { 289 | "action": "delete", 290 | "node": {"key": "/testkey", "modifiedIndex": 3, "createdIndex": 2}, 291 | "prevNode": { 292 | "newKey": False, 293 | "createdIndex": None, 294 | "modifiedIndex": 190, 295 | "value": "test", 296 | "expiration": None, 297 | "key": "/testkey", 298 | "ttl": None, 299 | "dir": False, 300 | }, 301 | } 302 | 303 | self._mock_api(200, d) 304 | res = self.client.pop(d["node"]["key"]) 305 | self.assertEqual( 306 | {attr: getattr(res, attr) for attr in dir(res) if attr in etcd.EtcdResult._node_props}, 307 | d["prevNode"], 308 | ) 309 | self.assertEqual(res.value, d["prevNode"]["value"]) 310 | 311 | def test_read(self): 312 | """Can get a value""" 313 | d = { 314 | "action": "get", 315 | "node": {"modifiedIndex": 190, "key": "/testkey", "value": "test"}, 316 | } 317 | self._mock_api(200, d) 318 | res = self.client.read("/testKey") 319 | self.assertEqual(res, etcd.EtcdResult(**d)) 320 | 321 | def test_get_dir(self): 322 | """Can get values in dirs""" 323 | d = { 324 | "action": "get", 325 | "node": { 326 | "modifiedIndex": 190, 327 | "key": "/testkey", 328 | "dir": True, 329 | "nodes": [ 330 | {"key": "/testDir/testKey", "modifiedIndex": 150, "value": "test"}, 331 | { 332 | "key": "/testDir/testKey2", 333 | "modifiedIndex": 190, 334 | "value": "test2", 335 | }, 336 | ], 337 | }, 338 | } 339 | self._mock_api(200, d) 340 | res = self.client.read("/testDir", recursive=True) 341 | self.assertEqual(res, etcd.EtcdResult(**d)) 342 | 343 | def test_not_in(self): 344 | """Can check if key is not in client""" 345 | self._mock_exception(etcd.EtcdKeyNotFound, "Key not Found : /testKey") 346 | self.assertTrue("/testey" not in self.client) 347 | 348 | def test_in(self): 349 | """Can check if key is not in client""" 350 | d = { 351 | "action": "get", 352 | "node": {"modifiedIndex": 190, "key": "/testkey", "value": "test"}, 353 | } 354 | self._mock_api(200, d) 355 | self.assertTrue("/testey" in self.client) 356 | 357 | def test_watch(self): 358 | """Can watch a key""" 359 | d = { 360 | "action": "get", 361 | "node": {"modifiedIndex": 190, "key": "/testkey", "value": "test"}, 362 | } 363 | self._mock_api(200, d) 364 | res = self.client.read("/testkey", wait=True) 365 | self.assertEqual(res, etcd.EtcdResult(**d)) 366 | 367 | def test_watch_index(self): 368 | """Can watch a key starting from the given Index""" 369 | d = { 370 | "action": "get", 371 | "node": {"modifiedIndex": 170, "key": "/testkey", "value": "testold"}, 372 | } 373 | self._mock_api(200, d) 374 | res = self.client.read("/testkey", wait=True, waitIndex=True) 375 | self.assertEqual(res, etcd.EtcdResult(**d)) 376 | 377 | 378 | class TestClientRequest(TestClientApiInterface): 379 | def setUp(self): 380 | self.client = etcd.Client(expected_cluster_id="abcdef1234") 381 | 382 | def _mock_api(self, status, d, cluster_id=None): 383 | resp = self._prepare_response(status, d) 384 | resp.getheader.return_value = cluster_id or "abcdef1234" 385 | self.client.http.request_encode_body = mock.MagicMock(return_value=resp) 386 | self.client.http.request = mock.MagicMock(return_value=resp) 387 | 388 | def _mock_error(self, error_code, msg, cause, method="PUT", fields=None, cluster_id=None): 389 | resp = self._prepare_response( 390 | 500, {"errorCode": error_code, "message": msg, "cause": cause} 391 | ) 392 | resp.getheader.return_value = cluster_id or "abcdef1234" 393 | self.client.http.request_encode_body = mock.create_autospec( 394 | self.client.http.request_encode_body, return_value=resp 395 | ) 396 | self.client.http.request = mock.create_autospec(self.client.http.request, return_value=resp) 397 | 398 | def test_compare_and_swap_failure(self): 399 | """Exception will be raised if prevValue != value in test_set""" 400 | self._mock_error(200, "Test Failed", "[ 1!=3 ]", fields={"prevValue": "oldbog"}) 401 | self.assertRaises(ValueError, self.client.write, "/testKey", "test", prevValue="oldbog") 402 | 403 | def test_watch_timeout(self): 404 | """Exception will be raised if prevValue != value in test_set""" 405 | self.client.http.request = mock.create_autospec( 406 | self.client.http.request, 407 | side_effect=urllib3.exceptions.ReadTimeoutError( 408 | self.client.http, "foo", "Read timed out" 409 | ), 410 | ) 411 | self.assertRaises( 412 | etcd.EtcdWatchTimedOut, 413 | self.client.watch, 414 | "/testKey", 415 | ) 416 | 417 | def test_path_without_trailing_slash(self): 418 | """Exception will be raised if a path without a trailing slash is used""" 419 | self.assertRaises(ValueError, self.client.api_execute, "testpath/bar", self.client._MPUT) 420 | 421 | def test_api_method_not_supported(self): 422 | """Exception will be raised if an unsupported HTTP method is used""" 423 | self.assertRaises(etcd.EtcdException, self.client.api_execute, "/testpath/bar", "TRACE") 424 | 425 | def test_read_cluster_id_changed(self): 426 | """Read timeout set to the default""" 427 | d = { 428 | "action": "set", 429 | "node": { 430 | "expiration": "2013-09-14T00:56:59.316195568+02:00", 431 | "modifiedIndex": 6, 432 | "key": "/testkey", 433 | "ttl": 19, 434 | "value": "test", 435 | }, 436 | } 437 | self._mock_api(200, d, cluster_id="notabcd1234") 438 | self.assertRaises(etcd.EtcdClusterIdChanged, self.client.read, "/testkey") 439 | self.client.read("/testkey") 440 | 441 | def test_read_connection_error(self): 442 | self.client.http.request = mock.create_autospec( 443 | self.client.http.request, side_effect=socket.error() 444 | ) 445 | self.assertRaises(etcd.EtcdConnectionFailed, self.client.read, "/something") 446 | # Direct GET request 447 | self.assertRaises(etcd.EtcdConnectionFailed, self.client.api_execute, "/a", "GET") 448 | 449 | def test_not_in(self): 450 | pass 451 | 452 | def test_in(self): 453 | pass 454 | 455 | def test_update_fails(self): 456 | """Non-atomic updates fail""" 457 | d = { 458 | "action": "set", 459 | "node": { 460 | "expiration": "2013-09-14T00:56:59.316195568+02:00", 461 | "modifiedIndex": 6, 462 | "key": "/testkey", 463 | "ttl": 19, 464 | "value": "test", 465 | }, 466 | } 467 | res = etcd.EtcdResult(**d) 468 | 469 | error = { 470 | "errorCode": 101, 471 | "message": "Compare failed", 472 | "cause": "[ != bar] [7 != 6]", 473 | "index": 6, 474 | } 475 | self._mock_api(412, error) 476 | res.value = "bar" 477 | self.assertRaises(ValueError, self.client.update, res) 478 | -------------------------------------------------------------------------------- /src/etcd/client.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module:: python-etcd 3 | :synopsis: A python etcd client. 4 | 5 | .. moduleauthor:: Jose Plana 6 | 7 | 8 | """ 9 | import logging 10 | 11 | try: 12 | # Python 3 13 | from http.client import HTTPException 14 | except ImportError: 15 | # Python 2 16 | from httplib import HTTPException 17 | import socket 18 | import urllib3 19 | from urllib3.exceptions import HTTPError 20 | from urllib3.exceptions import ReadTimeoutError 21 | import json 22 | import ssl 23 | import dns.resolver 24 | from functools import wraps 25 | import etcd 26 | from dns.resolver import NXDOMAIN 27 | import re 28 | 29 | try: 30 | from urlparse import urlparse 31 | except ImportError: 32 | from urllib.parse import urlparse 33 | 34 | 35 | _log = logging.getLogger(__name__) 36 | 37 | 38 | class Client(object): 39 | 40 | """ 41 | Client for etcd, the distributed log service using raft. 42 | """ 43 | 44 | _MGET = "GET" 45 | _MPUT = "PUT" 46 | _MPOST = "POST" 47 | _MDELETE = "DELETE" 48 | _comparison_conditions = set(("prevValue", "prevIndex", "prevExist", "refresh")) 49 | _read_options = set(("recursive", "wait", "waitIndex", "sorted", "quorum")) 50 | _del_conditions = set(("prevValue", "prevIndex")) 51 | 52 | http = None 53 | 54 | def __init__( 55 | self, 56 | host="127.0.0.1", 57 | port=4001, 58 | srv_domain=None, 59 | version_prefix="/v2", 60 | read_timeout=60, 61 | allow_redirect=True, 62 | protocol="http", 63 | cert=None, 64 | ca_cert=None, 65 | username=None, 66 | password=None, 67 | allow_reconnect=False, 68 | use_proxies=False, 69 | expected_cluster_id=None, 70 | per_host_pool_size=10, 71 | lock_prefix="/_locks", 72 | ): 73 | """ 74 | Initialize the client. 75 | 76 | Args: 77 | host (mixed): 78 | If a string, IP to connect to. 79 | If a tuple ((host, port), (host, port), ...) 80 | 81 | port (int): Port used to connect to etcd. 82 | 83 | srv_domain (str): Domain to search the SRV record for cluster autodiscovery. 84 | 85 | version_prefix (str): Url or version prefix in etcd url (default=/v2). 86 | 87 | read_timeout (int): max seconds to wait for a read. 88 | 89 | allow_redirect (bool): allow the client to connect to other nodes. 90 | 91 | protocol (str): Protocol used to connect to etcd. 92 | 93 | cert (mixed): If a string, the whole ssl client certificate; 94 | if a tuple, the cert and key file names. 95 | 96 | ca_cert (str): The ca certificate. If present it will enable 97 | validation. 98 | 99 | username (str): username for etcd authentication. 100 | 101 | password (str): password for etcd authentication. 102 | 103 | allow_reconnect (bool): allow the client to reconnect to another 104 | etcd server in the cluster in the case the 105 | default one does not respond. 106 | 107 | use_proxies (bool): we are using a list of proxies to which we connect, 108 | and don't want to connect to the original etcd cluster. 109 | 110 | expected_cluster_id (str): If a string, recorded as the expected 111 | UUID of the cluster (rather than 112 | learning it from the first request), 113 | reads will raise EtcdClusterIdChanged 114 | if they receive a response with a 115 | different cluster ID. 116 | per_host_pool_size (int): specifies maximum number of connections to pool 117 | by host. By default this will use up to 10 118 | connections. 119 | lock_prefix (str): Set the key prefix at etcd when client to lock object. 120 | By default this will be use /_locks. 121 | """ 122 | 123 | # If a DNS record is provided, use it to get the hosts list 124 | if srv_domain is not None: 125 | try: 126 | host = self._discover(srv_domain) 127 | except Exception as e: 128 | _log.error("Could not discover the etcd hosts from %s: %s", srv_domain, e) 129 | 130 | self._protocol = protocol 131 | 132 | def uri(protocol, host, port): 133 | return "%s://%s:%d" % (protocol, host, port) 134 | 135 | if not isinstance(host, tuple): 136 | self._machines_cache = [] 137 | self._base_uri = uri(self._protocol, host, port) 138 | else: 139 | if not allow_reconnect: 140 | _log.error("List of hosts incompatible with allow_reconnect.") 141 | raise etcd.EtcdException( 142 | "A list of hosts to connect to was given, but reconnection not allowed?" 143 | ) 144 | self._machines_cache = [uri(self._protocol, *conn) for conn in host] 145 | self._base_uri = self._machines_cache.pop(0) 146 | 147 | self.expected_cluster_id = expected_cluster_id 148 | self.version_prefix = version_prefix 149 | 150 | self._read_timeout = read_timeout 151 | self._allow_redirect = allow_redirect 152 | self._use_proxies = use_proxies 153 | self._allow_reconnect = allow_reconnect 154 | self._lock_prefix = lock_prefix 155 | 156 | # SSL Client certificate support 157 | 158 | kw = {"maxsize": per_host_pool_size} 159 | 160 | if self._read_timeout > 0: 161 | kw["timeout"] = self._read_timeout 162 | 163 | if cert: 164 | if isinstance(cert, tuple): 165 | # Key and cert are separate 166 | kw["cert_file"] = cert[0] 167 | kw["key_file"] = cert[1] 168 | else: 169 | # combined certificate 170 | kw["cert_file"] = cert 171 | 172 | if ca_cert: 173 | kw["ca_certs"] = ca_cert 174 | kw["cert_reqs"] = ssl.CERT_REQUIRED 175 | else: 176 | kw["cert_reqs"] = ssl.CERT_NONE 177 | urllib3.disable_warnings() 178 | 179 | self.username = None 180 | self.password = None 181 | if username and password: 182 | self.username = username 183 | self.password = password 184 | elif username: 185 | _log.warning("Username provided without password, both are required for authentication") 186 | elif password: 187 | _log.warning("Password provided without username, both are required for authentication") 188 | 189 | self.http = urllib3.PoolManager(num_pools=10, **kw) 190 | 191 | _log.debug("New etcd client created for %s", self.base_uri) 192 | 193 | if self._allow_reconnect: 194 | # we need the set of servers in the cluster in order to try 195 | # reconnecting upon error. The cluster members will be 196 | # added to the hosts list you provided. If you are using 197 | # proxies, set all 198 | # 199 | # Beware though: if you input '127.0.0.1' as your host and 200 | # etcd advertises 'localhost', both will be in the 201 | # resulting list. 202 | 203 | # If we're connecting to the original cluster, we can 204 | # extend the list given to the client with what we get 205 | # from self.machines 206 | if not self._use_proxies: 207 | self._machines_cache = list(set(self._machines_cache) | set(self.machines)) 208 | if self._base_uri in self._machines_cache: 209 | self._machines_cache.remove(self._base_uri) 210 | _log.debug("Machines cache initialised to %s", self._machines_cache) 211 | 212 | # Versions set to None. They will be set upon first usage. 213 | self._version = self._cluster_version = None 214 | 215 | def _set_version_info(self): 216 | """ 217 | Sets the version information provided by the server. 218 | """ 219 | # Set the version 220 | data = self.api_execute("/version", self._MGET).data 221 | version_info = json.loads(data.decode("utf-8")) 222 | self._version = version_info["etcdserver"] 223 | self._cluster_version = version_info["etcdcluster"] 224 | 225 | def _discover(self, domain): 226 | srv_names = [ 227 | "_etcd-client-ssl._tcp.{}".format(domain), 228 | "_etcd-client._tcp.{}".format(domain), 229 | "_etcd-ssl._tcp.{}".format(domain), 230 | "_etcd._tcp.{}".format(domain), 231 | ] 232 | found = False 233 | for srv_name in srv_names: 234 | try: 235 | answers = dns.resolver.query(srv_name, "SRV") 236 | if len(answers): 237 | found = True 238 | break 239 | except NXDOMAIN: 240 | continue 241 | 242 | if not found: 243 | raise ValueError("Could not find SRV record for domain {}.".format(domain)) 244 | 245 | if re.search("-ssl", srv_name): 246 | self._protocol = "https" 247 | 248 | hosts = [] 249 | for answer in answers: 250 | hosts.append((answer.target.to_text(omit_final_dot=True), answer.port)) 251 | _log.debug("Found %s", hosts) 252 | if not len(hosts): 253 | raise ValueError("The SRV record is present but no hosts were found") 254 | return tuple(hosts) 255 | 256 | def __del__(self): 257 | """Clean up open connections""" 258 | if self.http is not None: 259 | try: 260 | self.http.clear() 261 | except ReferenceError: 262 | # this may hit an already-cleared weakref 263 | pass 264 | 265 | @property 266 | def base_uri(self): 267 | """URI used by the client to connect to etcd.""" 268 | return self._base_uri 269 | 270 | @property 271 | def host(self): 272 | """Node to connect etcd.""" 273 | return urlparse(self._base_uri).netloc.split(":")[0] 274 | 275 | @property 276 | def port(self): 277 | """Port to connect etcd.""" 278 | return int(urlparse(self._base_uri).netloc.split(":")[1]) 279 | 280 | @property 281 | def protocol(self): 282 | """Protocol used to connect etcd.""" 283 | return self._protocol 284 | 285 | @property 286 | def read_timeout(self): 287 | """Max seconds to wait for a read.""" 288 | return self._read_timeout 289 | 290 | @property 291 | def allow_redirect(self): 292 | """Allow the client to connect to other nodes.""" 293 | return self._allow_redirect 294 | 295 | @property 296 | def lock_prefix(self): 297 | """Get the key prefix at etcd when client to lock object.""" 298 | return self._lock_prefix 299 | 300 | @property 301 | def machines(self): 302 | """ 303 | Members of the cluster. 304 | 305 | Returns: 306 | list. str with all the nodes in the cluster. 307 | 308 | >>> print client.machines 309 | ['http://127.0.0.1:4001', 'http://127.0.0.1:4002'] 310 | """ 311 | # We can't use api_execute here, or it causes a logical loop 312 | try: 313 | uri = self._base_uri + self.version_prefix + "/machines" 314 | response = self.http.request( 315 | self._MGET, 316 | uri, 317 | headers=self._get_headers(), 318 | timeout=self.read_timeout, 319 | redirect=self.allow_redirect, 320 | ) 321 | 322 | machines = [ 323 | node.strip() 324 | for node in self._handle_server_response(response).data.decode("utf-8").split(",") 325 | ] 326 | _log.debug("Retrieved list of machines: %s", machines) 327 | return machines 328 | except (HTTPError, HTTPException, socket.error) as e: 329 | # We can't get the list of machines, if one server is in the 330 | # machines cache, try on it 331 | _log.error( 332 | "Failed to get list of machines from %s%s: %r", 333 | self._base_uri, 334 | self.version_prefix, 335 | e, 336 | ) 337 | if self._machines_cache: 338 | self._base_uri = self._machines_cache.pop(0) 339 | _log.info("Retrying on %s", self._base_uri) 340 | # Call myself 341 | return self.machines 342 | else: 343 | raise etcd.EtcdException( 344 | "Could not get the list of servers, " 345 | "maybe you provided the wrong " 346 | "host(s) to connect to?" 347 | ) 348 | 349 | @property 350 | def members(self): 351 | """ 352 | A more structured view of peers in the cluster. 353 | 354 | Note that while we have an internal DS called _members, accessing the public property will call etcd. 355 | """ 356 | # Empty the members list 357 | self._members = {} 358 | try: 359 | data = self.api_execute(self.version_prefix + "/members", self._MGET).data.decode( 360 | "utf-8" 361 | ) 362 | res = json.loads(data) 363 | for member in res["members"]: 364 | self._members[member["id"]] = member 365 | return self._members 366 | except: 367 | raise etcd.EtcdException( 368 | "Could not get the members list, maybe the cluster has gone away?" 369 | ) 370 | 371 | @property 372 | def leader(self): 373 | """ 374 | Returns: 375 | dict. the leader of the cluster. 376 | 377 | >>> print client.leader 378 | {"id":"ce2a822cea30bfca","name":"default","peerURLs":["http://localhost:2380","http://localhost:7001"],"clientURLs":["http://127.0.0.1:4001"]} 379 | """ 380 | try: 381 | leader = json.loads( 382 | self.api_execute(self.version_prefix + "/stats/self", self._MGET).data.decode( 383 | "utf-8" 384 | ) 385 | ) 386 | return self.members[leader["leaderInfo"]["leader"]] 387 | except Exception as e: 388 | raise etcd.EtcdException("Cannot get leader data: %s" % e) 389 | 390 | @property 391 | def stats(self): 392 | """ 393 | Returns: 394 | dict. the stats of the local server 395 | """ 396 | return self._stats() 397 | 398 | @property 399 | def leader_stats(self): 400 | """ 401 | Returns: 402 | dict. the stats of the leader 403 | """ 404 | return self._stats("leader") 405 | 406 | @property 407 | def store_stats(self): 408 | """ 409 | Returns: 410 | dict. the stats of the kv store 411 | """ 412 | return self._stats("store") 413 | 414 | def _stats(self, what="self"): 415 | """Internal method to access the stats endpoints""" 416 | data = self.api_execute(self.version_prefix + "/stats/" + what, self._MGET).data.decode( 417 | "utf-8" 418 | ) 419 | try: 420 | return json.loads(data) 421 | except (TypeError, ValueError): 422 | raise etcd.EtcdException("Cannot parse json data in the response") 423 | 424 | @property 425 | def version(self): 426 | """ 427 | Version of etcd. 428 | """ 429 | if not self._version: 430 | self._set_version_info() 431 | return self._version 432 | 433 | @property 434 | def cluster_version(self): 435 | """ 436 | Version of the etcd cluster. 437 | """ 438 | if not self._cluster_version: 439 | self._set_version_info() 440 | 441 | return self._cluster_version 442 | 443 | @property 444 | def key_endpoint(self): 445 | """ 446 | REST key endpoint. 447 | """ 448 | return self.version_prefix + "/keys" 449 | 450 | def __contains__(self, key): 451 | """ 452 | Check if a key is available in the cluster. 453 | 454 | >>> print 'key' in client 455 | True 456 | """ 457 | try: 458 | self.get(key) 459 | return True 460 | except etcd.EtcdKeyNotFound: 461 | return False 462 | 463 | def _sanitize_key(self, key): 464 | if not key.startswith("/"): 465 | key = "/{}".format(key) 466 | return key 467 | 468 | def write(self, key, value, ttl=None, dir=False, append=False, **kwdargs): 469 | """ 470 | Writes the value for a key, possibly doing atomic Compare-and-Swap 471 | 472 | Args: 473 | key (str): Key. 474 | 475 | value (object): value to set 476 | 477 | ttl (int): Time in seconds of expiration (optional). 478 | 479 | dir (bool): Set to true if we are writing a directory; default is false. 480 | 481 | append (bool): If true, it will post to append the new value to the dir, creating a sequential key. Defaults to false. 482 | 483 | Other parameters modifying the write method are accepted: 484 | 485 | 486 | prevValue (str): compare key to this value, and swap only if corresponding (optional). 487 | 488 | prevIndex (int): modify key only if actual modifiedIndex matches the provided one (optional). 489 | 490 | prevExist (bool): If false, only create key; if true, only update key. 491 | 492 | refresh (bool): since 2.3.0, If true, only update the ttl, prev key must existed(prevExist=True). 493 | 494 | Returns: 495 | client.EtcdResult 496 | 497 | >>> print client.write('/key', 'newValue', ttl=60, prevExist=False).value 498 | 'newValue' 499 | 500 | """ 501 | _log.debug("Writing %s to key %s ttl=%s dir=%s append=%s", value, key, ttl, dir, append) 502 | key = self._sanitize_key(key) 503 | params = {} 504 | if value is not None: 505 | params["value"] = value 506 | 507 | if ttl is not None: 508 | params["ttl"] = ttl 509 | 510 | if dir: 511 | if value: 512 | raise etcd.EtcdException("Cannot create a directory with a value") 513 | params["dir"] = "true" 514 | 515 | for k, v in kwdargs.items(): 516 | if k in self._comparison_conditions: 517 | if type(v) == bool: 518 | params[k] = v and "true" or "false" 519 | else: 520 | params[k] = v 521 | 522 | method = append and self._MPOST or self._MPUT 523 | if "_endpoint" in kwdargs: 524 | path = kwdargs["_endpoint"] + key 525 | else: 526 | path = self.key_endpoint + key 527 | 528 | response = self.api_execute(path, method, params=params) 529 | return self._result_from_response(response) 530 | 531 | def refresh(self, key, ttl, **kwdargs): 532 | """ 533 | (Since 2.3.0) Refresh the ttl of a key without notifying watchers. 534 | 535 | Keys in etcd can be refreshed without notifying watchers, 536 | this can be achieved by setting the refresh to true when updating a TTL 537 | 538 | You cannot update the value of a key when refreshing it 539 | 540 | @see: https://github.com/coreos/etcd/blob/release-2.3/Documentation/api.md#refreshing-key-ttl 541 | 542 | Args: 543 | key (str): Key. 544 | 545 | ttl (int): Time in seconds of expiration (optional). 546 | 547 | Other parameters modifying the write method are accepted as `EtcdClient.write`. 548 | """ 549 | # overwrite kwdargs' prevExist 550 | kwdargs["prevExist"] = True 551 | return self.write(key=key, value=None, ttl=ttl, refresh=True, **kwdargs) 552 | 553 | def update(self, obj): 554 | """ 555 | Updates the value for a key atomically. Typical usage would be: 556 | 557 | c = etcd.Client() 558 | o = c.read("/somekey") 559 | o.value += 1 560 | c.update(o) 561 | 562 | Args: 563 | obj (etcd.EtcdResult): The object that needs updating. 564 | 565 | """ 566 | _log.debug("Updating %s to %s.", obj.key, obj.value) 567 | kwdargs = {"dir": obj.dir, "ttl": obj.ttl, "prevExist": True} 568 | 569 | if not obj.dir: 570 | # prevIndex on a dir causes a 'not a file' error. d'oh! 571 | kwdargs["prevIndex"] = obj.modifiedIndex 572 | return self.write(obj.key, obj.value, **kwdargs) 573 | 574 | def read(self, key, **kwdargs): 575 | """ 576 | Returns the value of the key 'key'. 577 | 578 | Args: 579 | key (str): Key. 580 | 581 | Recognized kwd args 582 | 583 | recursive (bool): If you should fetch recursively a dir 584 | 585 | wait (bool): If we should wait and return next time the key is changed 586 | 587 | waitIndex (int): The index to fetch results from. 588 | 589 | sorted (bool): Sort the output keys (alphanumerically) 590 | 591 | timeout (int): max seconds to wait for a read. 592 | 593 | Returns: 594 | client.EtcdResult (or an array of client.EtcdResult if a 595 | subtree is queried) 596 | 597 | Raises: 598 | KeyValue: If the key doesn't exists. 599 | 600 | urllib3.exceptions.TimeoutError: If timeout is reached. 601 | 602 | >>> print client.get('/key').value 603 | 'value' 604 | 605 | """ 606 | _log.debug("Issuing read for key %s with args %s", key, kwdargs) 607 | key = self._sanitize_key(key) 608 | 609 | params = {} 610 | for k, v in kwdargs.items(): 611 | if k in self._read_options: 612 | if type(v) == bool: 613 | params[k] = v and "true" or "false" 614 | elif v is not None: 615 | params[k] = v 616 | 617 | timeout = kwdargs.get("timeout", None) 618 | 619 | response = self.api_execute( 620 | self.key_endpoint + key, self._MGET, params=params, timeout=timeout 621 | ) 622 | return self._result_from_response(response) 623 | 624 | def delete(self, key, recursive=None, dir=None, **kwdargs): 625 | """ 626 | Removed a key from etcd. 627 | 628 | Args: 629 | 630 | key (str): Key. 631 | 632 | recursive (bool): if we want to recursively delete a directory, set 633 | it to true 634 | 635 | dir (bool): if we want to delete a directory, set it to true 636 | 637 | prevValue (str): compare key to this value, and swap only if 638 | corresponding (optional). 639 | 640 | prevIndex (int): modify key only if actual modifiedIndex matches the 641 | provided one (optional). 642 | 643 | Returns: 644 | client.EtcdResult 645 | 646 | Raises: 647 | KeyValue: If the key doesn't exists. 648 | 649 | >>> print client.delete('/key').key 650 | '/key' 651 | 652 | """ 653 | _log.debug( 654 | "Deleting %s recursive=%s dir=%s extra args=%s", 655 | key, 656 | recursive, 657 | dir, 658 | kwdargs, 659 | ) 660 | key = self._sanitize_key(key) 661 | 662 | kwds = {} 663 | if recursive is not None: 664 | kwds["recursive"] = recursive and "true" or "false" 665 | if dir is not None: 666 | kwds["dir"] = dir and "true" or "false" 667 | 668 | for k in self._del_conditions: 669 | if k in kwdargs: 670 | kwds[k] = kwdargs[k] 671 | _log.debug("Calculated params = %s", kwds) 672 | 673 | response = self.api_execute(self.key_endpoint + key, self._MDELETE, params=kwds) 674 | return self._result_from_response(response) 675 | 676 | def pop(self, key, recursive=None, dir=None, **kwdargs): 677 | """ 678 | Remove specified key from etcd and return the corresponding value. 679 | 680 | Args: 681 | 682 | key (str): Key. 683 | 684 | recursive (bool): if we want to recursively delete a directory, set 685 | it to true 686 | 687 | dir (bool): if we want to delete a directory, set it to true 688 | 689 | prevValue (str): compare key to this value, and swap only if 690 | corresponding (optional). 691 | 692 | prevIndex (int): modify key only if actual modifiedIndex matches the 693 | provided one (optional). 694 | 695 | Returns: 696 | client.EtcdResult 697 | 698 | Raises: 699 | KeyValue: If the key doesn't exists. 700 | 701 | >>> print client.pop('/key').value 702 | 'value' 703 | 704 | """ 705 | return self.delete(key=key, recursive=recursive, dir=dir, **kwdargs)._prev_node 706 | 707 | # Higher-level methods on top of the basic primitives 708 | def test_and_set(self, key, value, prev_value, ttl=None): 709 | """ 710 | Atomic test & set operation. 711 | It will check if the value of 'key' is 'prev_value', 712 | if the the check is correct will change the value for 'key' to 'value' 713 | if the the check is false an exception will be raised. 714 | 715 | Args: 716 | key (str): Key. 717 | value (object): value to set 718 | prev_value (object): previous value. 719 | ttl (int): Time in seconds of expiration (optional). 720 | 721 | Returns: 722 | client.EtcdResult 723 | 724 | Raises: 725 | ValueError: When the 'prev_value' is not the current value. 726 | 727 | >>> print client.test_and_set('/key', 'new', 'old', ttl=60).value 728 | 'new' 729 | 730 | """ 731 | return self.write(key, value, prevValue=prev_value, ttl=ttl) 732 | 733 | def set(self, key, value, ttl=None): 734 | """ 735 | Compatibility: sets the value of the key 'key' to the value 'value' 736 | 737 | Args: 738 | key (str): Key. 739 | value (object): value to set 740 | ttl (int): Time in seconds of expiration (optional). 741 | 742 | Returns: 743 | client.EtcdResult 744 | 745 | Raises: 746 | etcd.EtcdException: when something weird goes wrong. 747 | 748 | """ 749 | return self.write(key, value, ttl=ttl) 750 | 751 | def get(self, key): 752 | """ 753 | Returns the value of the key 'key'. 754 | 755 | Args: 756 | key (str): Key. 757 | 758 | Returns: 759 | client.EtcdResult 760 | 761 | Raises: 762 | KeyError: If the key doesn't exists. 763 | 764 | >>> print client.get('/key').value 765 | 'value' 766 | 767 | """ 768 | return self.read(key) 769 | 770 | def watch(self, key, index=None, timeout=None, recursive=None): 771 | """ 772 | Blocks until a new event has been received, starting at index 'index' 773 | 774 | Args: 775 | key (str): Key. 776 | 777 | index (int): Index to start from. 778 | 779 | timeout (int): max seconds to wait for a read. 780 | 781 | Returns: 782 | client.EtcdResult 783 | 784 | Raises: 785 | KeyValue: If the key doesn't exist. 786 | 787 | etcd.EtcdWatchTimedOut: If timeout is reached. 788 | 789 | >>> print client.watch('/key').value 790 | 'value' 791 | 792 | """ 793 | _log.debug("About to wait on key %s, index %s", key, index) 794 | if index: 795 | return self.read(key, wait=True, waitIndex=index, timeout=timeout, recursive=recursive) 796 | else: 797 | return self.read(key, wait=True, timeout=timeout, recursive=recursive) 798 | 799 | def eternal_watch(self, key, index=None, recursive=None): 800 | """ 801 | Generator that will yield changes from a key. 802 | Note that this method will block forever until an event is generated. 803 | 804 | Args: 805 | key (str): Key to subcribe to. 806 | index (int): Index from where the changes will be received. 807 | 808 | Yields: 809 | client.EtcdResult 810 | 811 | >>> for event in client.eternal_watch('/subcription_key'): 812 | ... print event.value 813 | ... 814 | value1 815 | value2 816 | 817 | """ 818 | local_index = index 819 | while True: 820 | response = self.watch(key, index=local_index, timeout=0, recursive=recursive) 821 | local_index = response.modifiedIndex + 1 822 | yield response 823 | 824 | def get_lock(self, *args, **kwargs): 825 | raise NotImplementedError("Lock primitives were removed from etcd 2.0") 826 | 827 | @property 828 | def election(self): 829 | raise NotImplementedError("Election primitives were removed from etcd 2.0") 830 | 831 | def _result_from_response(self, response): 832 | """Creates an EtcdResult from json dictionary""" 833 | raw_response = response.data 834 | try: 835 | res = json.loads(raw_response.decode("utf-8")) 836 | except (TypeError, ValueError, UnicodeError) as e: 837 | raise etcd.EtcdException("Server response was not valid JSON: %r" % e) 838 | try: 839 | r = etcd.EtcdResult(**res) 840 | if response.status == 201: 841 | r.newKey = True 842 | r.parse_headers(response) 843 | return r 844 | except Exception as e: 845 | raise etcd.EtcdException("Unable to decode server response: %r" % e) 846 | 847 | def _next_server(self, cause=None): 848 | """Selects the next server in the list, refreshes the server list.""" 849 | _log.debug( 850 | "Selecting next machine in cache. Available machines: %s", 851 | self._machines_cache, 852 | ) 853 | try: 854 | mach = self._machines_cache.pop() 855 | except IndexError: 856 | _log.error("Machines cache is empty, no machines to try.") 857 | raise etcd.EtcdConnectionFailed("No more machines in the cluster", cause=cause) 858 | else: 859 | _log.info("Selected new etcd server %s", mach) 860 | return mach 861 | 862 | def _wrap_request(payload): 863 | @wraps(payload) 864 | def wrapper(self, path, method, params=None, timeout=None): 865 | response = False 866 | 867 | if timeout is None: 868 | timeout = self.read_timeout 869 | 870 | if timeout == 0: 871 | timeout = None 872 | 873 | if not path.startswith("/"): 874 | raise ValueError("Path does not start with /") 875 | 876 | while not response: 877 | some_request_failed = False 878 | try: 879 | response = payload(self, path, method, params=params, timeout=timeout) 880 | # Check the cluster ID hasn't changed under us. We use 881 | # preload_content=False above so we can read the headers 882 | # before we wait for the content of a watch. 883 | self._check_cluster_id(response, path) 884 | # Now force the data to be preloaded in order to trigger any 885 | # IO-related errors in this method rather than when we try to 886 | # access it later. 887 | _ = response.data 888 | # urllib3 doesn't wrap all httplib exceptions and earlier versions 889 | # don't wrap socket errors either. 890 | except (HTTPError, HTTPException, socket.error) as e: 891 | if ( 892 | isinstance(params, dict) 893 | and params.get("wait") == "true" 894 | and isinstance(e, ReadTimeoutError) 895 | ): 896 | _log.debug("Watch timed out.") 897 | raise etcd.EtcdWatchTimedOut("Watch timed out: %r" % e, cause=e) 898 | _log.error("Request to server %s failed: %r", self._base_uri, e) 899 | if self._allow_reconnect: 900 | _log.info("Reconnection allowed, looking for another " "server.") 901 | # _next_server() raises EtcdException if there are no 902 | # machines left to try, breaking out of the loop. 903 | self._base_uri = self._next_server(cause=e) 904 | some_request_failed = True 905 | 906 | # if exception is raised on _ = response.data 907 | # the condition for while loop will be False 908 | # but we should retry 909 | response = False 910 | else: 911 | _log.debug("Reconnection disabled, giving up.") 912 | raise etcd.EtcdConnectionFailed( 913 | "Connection to etcd failed due to %r" % e, cause=e 914 | ) 915 | except etcd.EtcdClusterIdChanged as e: 916 | _log.warning(e) 917 | raise 918 | except: 919 | _log.debug("Unexpected request failure, re-raising.") 920 | raise 921 | 922 | if some_request_failed: 923 | if not self._use_proxies: 924 | # The cluster may have changed since last invocation 925 | self._machines_cache = self.machines 926 | self._machines_cache.remove(self._base_uri) 927 | return self._handle_server_response(response) 928 | 929 | return wrapper 930 | 931 | @_wrap_request 932 | def api_execute(self, path, method, params=None, timeout=None): 933 | """Executes the query.""" 934 | url = self._base_uri + path 935 | 936 | if (method == self._MGET) or (method == self._MDELETE): 937 | return self.http.request( 938 | method, 939 | url, 940 | timeout=timeout, 941 | fields=params, 942 | redirect=self.allow_redirect, 943 | headers=self._get_headers(), 944 | preload_content=False, 945 | ) 946 | 947 | elif (method == self._MPUT) or (method == self._MPOST): 948 | return self.http.request_encode_body( 949 | method, 950 | url, 951 | fields=params, 952 | timeout=timeout, 953 | encode_multipart=False, 954 | redirect=self.allow_redirect, 955 | headers=self._get_headers(), 956 | preload_content=False, 957 | ) 958 | else: 959 | raise etcd.EtcdException("HTTP method {} not supported".format(method)) 960 | 961 | @_wrap_request 962 | def api_execute_json(self, path, method, params=None, timeout=None): 963 | url = self._base_uri + path 964 | json_payload = json.dumps(params) 965 | headers = self._get_headers() 966 | headers["Content-Type"] = "application/json" 967 | return self.http.urlopen( 968 | method, 969 | url, 970 | body=json_payload, 971 | timeout=timeout, 972 | redirect=self.allow_redirect, 973 | headers=headers, 974 | preload_content=False, 975 | ) 976 | 977 | def _check_cluster_id(self, response, path): 978 | cluster_id = response.getheader("x-etcd-cluster-id") 979 | if not cluster_id: 980 | if self.version_prefix in path: 981 | _log.warning("etcd response did not contain a cluster ID") 982 | return 983 | id_changed = self.expected_cluster_id and cluster_id != self.expected_cluster_id 984 | # Update the ID so we only raise the exception once. 985 | old_expected_cluster_id = self.expected_cluster_id 986 | self.expected_cluster_id = cluster_id 987 | if id_changed: 988 | # Defensive: clear the pool so that we connect afresh next 989 | # time. 990 | self.http.clear() 991 | raise etcd.EtcdClusterIdChanged( 992 | "The UUID of the cluster changed from {} to " 993 | "{}.".format(old_expected_cluster_id, cluster_id) 994 | ) 995 | 996 | def _handle_server_response(self, response): 997 | """Handles the server response""" 998 | if response.status in [200, 201]: 999 | return response 1000 | 1001 | else: 1002 | resp = response.data.decode("utf-8") 1003 | 1004 | # throw the appropriate exception 1005 | try: 1006 | r = json.loads(resp) 1007 | r["status"] = response.status 1008 | except (TypeError, ValueError): 1009 | # Bad JSON, make a response locally. 1010 | r = {"message": "Bad response", "cause": str(resp)} 1011 | etcd.EtcdError.handle(r) 1012 | 1013 | def _get_headers(self): 1014 | if self.username and self.password: 1015 | credentials = ":".join((self.username, self.password)) 1016 | return urllib3.make_headers(basic_auth=credentials) 1017 | return {} 1018 | --------------------------------------------------------------------------------