├── .github └── workflows │ ├── ci.yml │ └── release.yml ├── .gitignore ├── .mypy.ini ├── CHANGELOG.rst ├── Dockerfile ├── LICENSE.rst ├── MANIFEST.in ├── Makefile ├── README.rst ├── catalog-info.yaml ├── docs ├── Makefile └── source │ ├── api_documentation.rst │ ├── conf.py │ ├── extending.rst │ └── index.rst ├── poetry.lock ├── pyproject.toml ├── requirements-ci.txt ├── requirements-dev.txt ├── sherlock ├── __init__.py └── lock.py ├── tests ├── __init__.py ├── integration │ ├── __init__.py │ └── test_lock.py ├── test_lock.py └── test_sherlock.py └── tox.ini /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI/CD 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: {} 8 | 9 | jobs: 10 | 11 | lint: 12 | name: Lint 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - uses: actions/checkout@v3 17 | 18 | - name: Install Poetry 19 | run: pipx install poetry 20 | 21 | - name: Setup Python ${{ matrix.py }} 22 | uses: actions/setup-python@v4 23 | with: 24 | python-version: "3.7" 25 | cache: 'poetry' 26 | 27 | - name: Install linting tools 28 | run: poetry install --only dev 29 | 30 | - name: Run Black 31 | run: poetry run black --check sherlock tests 32 | 33 | - name: Run iSort 34 | run: poetry run isort --check sherlock tests 35 | 36 | - name: Run flake8 37 | run: poetry run flake8 sherlock tests 38 | 39 | - name: Run pycodestyle 40 | run: poetry run pycodestyle sherlock tests 41 | 42 | test: 43 | name: Test ${{ matrix.py }} - ${{ matrix.os }} 44 | runs-on: ${{ matrix.os }}-latest 45 | 46 | strategy: 47 | fail-fast: false 48 | matrix: 49 | os: 50 | - Ubuntu 51 | py: 52 | - "3.7" 53 | - "3.8" 54 | - "3.9" 55 | - "3.10" 56 | - "3.11" 57 | 58 | env: 59 | REDIS_HOST: 127.0.0.1 60 | ETCD_HOST: 127.0.0.1 61 | MEMCACHED_HOST: 127.0.0.1 62 | 63 | steps: 64 | - uses: actions/checkout@v3 65 | 66 | - name: Install Poetry 67 | run: pipx install poetry 68 | 69 | - name: Setup Python ${{ matrix.py }} 70 | uses: actions/setup-python@v4 71 | with: 72 | python-version: ${{ matrix.py }} 73 | cache: 'poetry' 74 | 75 | - name: Install `libmemcached-dev` 76 | run: sudo apt-get install -y libmemcached-dev 77 | 78 | - name: Install sherlock 79 | run: poetry install --all-extras 80 | 81 | - name: Run tests 82 | run: poetry run tox 83 | 84 | - uses: codecov/codecov-action@v3 85 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | release: 5 | types: 6 | - published 7 | 8 | jobs: 9 | release: 10 | name: Release to PyPi 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - uses: actions/checkout@v3 15 | 16 | - name: Install Poetry 17 | run: pipx install poetry 18 | 19 | - name: Build 20 | run: poetry build 21 | 22 | - name: Publish 23 | env: 24 | POETRY_PYPI_TOKEN_PYPI: ${{ secrets.POETRY_PYPI_TOKEN_PYPI }} 25 | run: | 26 | poetry config pypi-token.pypi "$POETRY_PYPI_TOKEN_PYPI" 27 | poetry publish 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | bin/ 10 | build/ 11 | develop-eggs/ 12 | dist/ 13 | eggs/ 14 | lib/ 15 | lib64/ 16 | parts/ 17 | sdist/ 18 | var/ 19 | *.egg-info/ 20 | .installed.cfg 21 | *.egg 22 | 23 | # Installer logs 24 | pip-log.txt 25 | pip-delete-this-directory.txt 26 | 27 | # Unit test / coverage reports 28 | .tox/ 29 | .coverage 30 | .cache 31 | nosetests.xml 32 | coverage.xml 33 | 34 | # Translations 35 | *.mo 36 | 37 | # Mr Developer 38 | .mr.developer.cfg 39 | .project 40 | .pydevproject 41 | 42 | # Rope 43 | .ropeproject 44 | 45 | # Django stuff: 46 | *.log 47 | *.pot 48 | 49 | # Sphinx documentation 50 | docs/_build/ 51 | 52 | # Custom 53 | dump.rdb 54 | docs/source/_build 55 | 56 | # Project specific 57 | kubeconfig.yaml -------------------------------------------------------------------------------- /.mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | 3 | [mypy-etcd.*] 4 | ignore_missing_imports = true 5 | 6 | [mypy-kubernetes.*] 7 | ignore_missing_imports = true 8 | 9 | [mypy-pylibmc.*] 10 | ignore_missing_imports = true 11 | -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | .. role:: python(code) 2 | :language: python 3 | 4 | CHANGELOG 5 | ######### 6 | 7 | Development Version 8 | ******************* 9 | 10 | 0.4.1 11 | ***** 12 | 13 | Bug Fixes 14 | ========= 15 | * Fix bug in :python:`RedisLock` where lock would be released when :python:`renew` is called 16 | 17 | 0.4.0 18 | ***** 19 | 20 | Breaking Changes 21 | ================ 22 | 23 | * Drop support for :python:`python<3.7` 24 | 25 | New Features 26 | ============ 27 | * Add :python:`KubernetesLock` backend 28 | * Add :python:`FileLock` backend 29 | * Install backend specific dependencies with extras `#59`_ 30 | * Add python`.renew()` method to all backends `#61`_ 31 | 32 | .. _#59: https://github.com/py-sherlock/sherlock/pull/59 33 | .. _#61: https://github.com/py-sherlock/sherlock/pull/61 34 | 35 | Bug Fixes 36 | ========= 37 | * Use :python:`ARGV` in Redis Lua scripts to add RedisCluster compatibility `#31`_ 38 | * :python:`redis>=2.10.6` client won't work with :python:`sherlock<=0.3.2` `#32`_ 39 | * :python:`timeout=0` doesn't work as expected with :python:`RedisLock` `#60`_ 40 | 41 | .. _#31: https://github.com/vaidik/sherlock/issues/31 42 | .. _#32: https://github.com/vaidik/sherlock/issues/32 43 | .. _#60: https://github.com/py-sherlock/sherlock/pull/60 44 | 45 | 0.3.2 46 | ***** 47 | 48 | Bug Fixes 49 | ========= 50 | * :python:`redis>=2.10.6` client won't work with :python:`sherlock<=0.3.1` `#32`_ 51 | 52 | .. _#32: https://github.com/vaidik/sherlock/issues/32 53 | 54 | 0.3.1 55 | ***** 56 | 57 | Bug Fixes 58 | ========= 59 | * Python 3 support for :python:`sherlock` 60 | 61 | 0.3.0 62 | ***** 63 | 64 | Bug Fixes 65 | ========= 66 | * :python:`sherlock.Lock` should use globally configured client object. 67 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.7 2 | 3 | WORKDIR /sherlock 4 | 5 | # Memcached 6 | RUN apt-get update -y && apt-get install -y libmemcached-dev gcc 7 | RUN pip install pytz ipython ipdb 8 | 9 | COPY requirements-ci.txt /sherlock/requirements-ci.txt 10 | RUN pip install -r /sherlock/requirements-ci.txt && \ 11 | rm /sherlock/requirements-ci.txt 12 | -------------------------------------------------------------------------------- /LICENSE.rst: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Vaidik Kapoor 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst 2 | include CHANGELOG.rst 3 | include LICENSE.rst 4 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Sherlock Makefile 2 | # ~~~~~~~~~~~~~~~~~ 3 | # 4 | # Shortcuts for various tasks. 5 | 6 | documentation: 7 | @(cd docs; make html) 8 | 9 | test: 10 | poetry run tox 11 | 12 | doctest: 13 | @(cd docs/source; sphinx-build -b doctest . _build/doctest) 14 | 15 | readme: 16 | python -c 'import sherlock; print sherlock.__doc__' | sed "s/:mod:\`sherlock\`/Sherlock/g" | sed "s/:.*:\`\(.*\)\`/\`\`\1\`\`/g" > README.rst 17 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | 2 | Sherlock: Distributed Locks with a choice of backend 3 | ==================================================== 4 | 5 | Sherlock is a library that provides easy-to-use distributed inter-process 6 | locks and also allows you to choose a backend of your choice for lock 7 | synchronization. 8 | 9 | |Build Status| |Coverage Status| 10 | 11 | .. |Build Status| image:: https://github.com/py-sherlock/sherlock/actions/workflows/ci.yml/badge.svg 12 | :target: https://github.com/py-sherlock/sherlock/actions?query=workflow%3ACI/CD 13 | 14 | .. |Coverage Status| image:: https://codecov.io/gh/py-sherlock/sherlock/branch/master/graph/badge.svg?token=QJXCZVSAEF 15 | :target: https://codecov.io/gh/py-sherlock/sherlock 16 | 17 | Overview 18 | -------- 19 | 20 | When you are working with resources which are accessed by multiple services or 21 | distributed services, more than often you need some kind of locking mechanism 22 | to make it possible to access some resources at a time. 23 | 24 | Distributed Locks or Mutexes can help you with this. Sherlock provides 25 | the exact same facility, with some extra goodies. It provides an easy-to-use API 26 | that resembles standard library's `threading.Lock` semantics. 27 | 28 | Apart from this, Sherlock gives you the flexibility of using a backend of 29 | your choice for managing locks. 30 | 31 | Sherlock also makes it simple for you to extend Sherlock to use 32 | backends that are not supported. 33 | 34 | Features 35 | ++++++++ 36 | 37 | * API similar to standard library's `threading.Lock`. 38 | * Support for With statement, to cleanly acquire and release locks. 39 | * Backend agnostic: supports File, `Redis`_, `Memcached`_, `Etcd`_, and `Kubernetes`_ as choice of 40 | backends. 41 | * Extendable: can be easily extended to work with any other of backend of 42 | choice by extending base lock class. Read ``extending``. 43 | 44 | .. _Redis: http://redis.io 45 | .. _Memcached: http://memcached.org 46 | .. _Etcd: http://github.com/coreos/etcd 47 | .. _Kubernetes: https://kubernetes.io 48 | 49 | Supported Backends and Client Libraries 50 | +++++++++++++++++++++++++++++++++++++++ 51 | 52 | Following client libraries are supported for every supported backend: 53 | 54 | * File: `filelock `__ 55 | * Redis: `redis-py `__ 56 | * Memcached: `pylibmc `__ 57 | * Etcd: `python-etcd `__ 58 | * Kubernetes: `kubernetes `__ 59 | 60 | As of now, only the above mentioned libraries are supported. Although 61 | Sherlock takes custom client objects so that you can easily provide 62 | settings that you want to use for that backend store, but Sherlock also 63 | checks if the provided client object is an instance of the supported clients 64 | and accepts client objects which pass this check, even if the APIs are the 65 | same. Sherlock might get rid of this issue later, if need be and if 66 | there is a demand for that. 67 | 68 | Installation 69 | ------------ 70 | 71 | Installation is simple. 72 | 73 | .. code:: bash 74 | 75 | pip install sherlock[all] 76 | 77 | .. note:: Sherlock will install all the client libraries for all the 78 | supported backends. 79 | 80 | Basic Usage 81 | ----------- 82 | 83 | Sherlock is simple to use as at the API and semantics level, it tries to 84 | conform to standard library's ``threading.Lock`` APIs. 85 | 86 | .. code-block:: python 87 | 88 | import sherlock 89 | from sherlock import Lock 90 | 91 | # Configure Sherlock's locks to use Redis as the backend, 92 | # never expire locks and retry acquiring an acquired lock after an 93 | # interval of 0.1 second. 94 | sherlock.configure(backend=sherlock.backends.REDIS, 95 | expire=None, 96 | retry_interval=0.1) 97 | 98 | # Note: configuring sherlock to use a backend does not limit you 99 | # another backend at the same time. You can import backend specific locks 100 | # like RedisLock, MCLock and EtcdLock and use them just the same way you 101 | # use a generic lock (see below). In fact, the generic Lock provided by 102 | # sherlock is just a proxy that uses these specific locks under the hood. 103 | 104 | # acquire a lock called my_lock 105 | lock = Lock('my_lock') 106 | 107 | # acquire a blocking lock 108 | lock.acquire() 109 | 110 | # check if the lock has been acquired or not 111 | lock.locked() == True 112 | 113 | # attempt to renew the lock 114 | lock.renew() 115 | 116 | # release the lock 117 | lock.release() 118 | 119 | Support for ``with`` statement 120 | ++++++++++++++++++++++++++++++ 121 | 122 | .. code-block:: python 123 | 124 | # using with statement 125 | with Lock('my_lock') as lock: 126 | # do something constructive with your locked resource here 127 | pass 128 | 129 | Blocking and Non-blocking API 130 | +++++++++++++++++++++++++++++ 131 | 132 | .. code-block:: python 133 | 134 | # acquire non-blocking lock 135 | lock1 = Lock('my_lock') 136 | lock2 = Lock('my_lock') 137 | 138 | # successfully acquire lock1 139 | lock1.acquire() 140 | 141 | # try to acquire lock in a non-blocking way 142 | lock2.acquire(False) == True # returns False 143 | 144 | # try to acquire lock in a blocking way 145 | lock2.acquire() # blocks until lock is acquired to timeout happens 146 | 147 | Using two backends at the same time 148 | +++++++++++++++++++++++++++++++++++ 149 | 150 | Configuring Sherlock to use a backend does not limit you from using 151 | another backend at the same time. You can import backend specific locks like 152 | RedisLock, MCLock and EtcdLock and use them just the same way you use a generic 153 | lock (see below). In fact, the generic Lock provided by Sherlock is just 154 | a proxy that uses these specific locks under the hood. 155 | 156 | .. code-block:: python 157 | 158 | import sherlock 159 | from sherlock import Lock 160 | 161 | # Configure Sherlock's locks to use Redis as the backend 162 | sherlock.configure(backend=sherlock.backends.REDIS) 163 | 164 | # Acquire a lock called my_lock, this lock uses Redis 165 | lock = Lock('my_lock') 166 | 167 | # Now acquire locks in Memcached 168 | from sherlock import MCLock 169 | mclock = MCLock('my_mc_lock') 170 | mclock.acquire() 171 | 172 | Tests 173 | ----- 174 | 175 | To run all the tests (including integration), you have to make sure that all 176 | the databases are running. Make sure all the services are running: 177 | 178 | .. code:: bash 179 | 180 | # memcached 181 | memcached 182 | 183 | # redis-server 184 | redis-server 185 | 186 | # etcd (etcd is probably not available as package, here is the simplest way 187 | # to run it). 188 | wget https://github.com/coreos/etcd/releases/download//etcd--.tar.gz 189 | tar -zxvf etcd--.gz 190 | ./etcd--/etcd 191 | 192 | Run tests like so: 193 | 194 | .. code:: bash 195 | 196 | python setup.py test 197 | 198 | Documentation 199 | ------------- 200 | 201 | Available `here`_. 202 | 203 | .. _here: http://sher-lock.readthedocs.org 204 | 205 | License 206 | ------- 207 | 208 | See `LICENSE`_. 209 | 210 | **In short**: This is an open-source project and exists for anyone to use it 211 | and/or modify it for personal use. Just be nice and attribute the credits 212 | wherever you can. :) 213 | 214 | .. _LICENSE: http://github.com/vaidik/sherlock/blob/master/LICENSE.rst 215 | 216 | Questions? 217 | ---------- 218 | 219 | You are encouraged to ask questions using `issues`_ as that helps everyone and 220 | myself when people with better know-how contribute to the discussion. However, 221 | if you wish to get in touch with me personally, then I can be contacted at 222 | **kapoor.vaidik++github+sherlock@gmail.com**. 223 | 224 | .. _issues: https://github.com/vaidik/sherlock/issues 225 | 226 | Distributed Locking in Other Languages 227 | -------------------------------------- 228 | 229 | * NodeJS - https://github.com/thedeveloper/warlock 230 | 231 | -------------------------------------------------------------------------------- /catalog-info.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: backstage.io/v1alpha1 2 | kind: Component 3 | metadata: 4 | name: sherlock 5 | annotations: 6 | github.com/project-slug: vaidik/sherlock 7 | spec: 8 | type: other 9 | lifecycle: unknown 10 | owner: roadie 11 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 36 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 38 | @echo " text to make text files" 39 | @echo " man to make manual pages" 40 | @echo " texinfo to make Texinfo files" 41 | @echo " info to make Texinfo files and run them through makeinfo" 42 | @echo " gettext to make PO message catalogs" 43 | @echo " changes to make an overview of all changed/added/deprecated items" 44 | @echo " xml to make Docutils-native XML files" 45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 46 | @echo " linkcheck to check all external links for integrity" 47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 48 | 49 | clean: 50 | rm -rf $(BUILDDIR)/* 51 | 52 | html: 53 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 54 | @echo 55 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 56 | 57 | dirhtml: 58 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 59 | @echo 60 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 61 | 62 | singlehtml: 63 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 64 | @echo 65 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 66 | 67 | pickle: 68 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 69 | @echo 70 | @echo "Build finished; now you can process the pickle files." 71 | 72 | json: 73 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 74 | @echo 75 | @echo "Build finished; now you can process the JSON files." 76 | 77 | htmlhelp: 78 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 79 | @echo 80 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 81 | ".hhp project file in $(BUILDDIR)/htmlhelp." 82 | 83 | qthelp: 84 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 85 | @echo 86 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 87 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 88 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Sherlock.qhcp" 89 | @echo "To view the help file:" 90 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Sherlock.qhc" 91 | 92 | devhelp: 93 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 94 | @echo 95 | @echo "Build finished." 96 | @echo "To view the help file:" 97 | @echo "# mkdir -p $$HOME/.local/share/devhelp/Sherlock" 98 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Sherlock" 99 | @echo "# devhelp" 100 | 101 | epub: 102 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 103 | @echo 104 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 105 | 106 | latex: 107 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 108 | @echo 109 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 110 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 111 | "(use \`make latexpdf' here to do that automatically)." 112 | 113 | latexpdf: 114 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 115 | @echo "Running LaTeX files through pdflatex..." 116 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 117 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 118 | 119 | latexpdfja: 120 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 121 | @echo "Running LaTeX files through platex and dvipdfmx..." 122 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 123 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 124 | 125 | text: 126 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 127 | @echo 128 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 129 | 130 | man: 131 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 132 | @echo 133 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 134 | 135 | texinfo: 136 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 137 | @echo 138 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 139 | @echo "Run \`make' in that directory to run these through makeinfo" \ 140 | "(use \`make info' here to do that automatically)." 141 | 142 | info: 143 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 144 | @echo "Running Texinfo files through makeinfo..." 145 | make -C $(BUILDDIR)/texinfo info 146 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 147 | 148 | gettext: 149 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 150 | @echo 151 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 152 | 153 | changes: 154 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 155 | @echo 156 | @echo "The overview file is in $(BUILDDIR)/changes." 157 | 158 | linkcheck: 159 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 160 | @echo 161 | @echo "Link check complete; look for any errors in the above output " \ 162 | "or in $(BUILDDIR)/linkcheck/output.txt." 163 | 164 | doctest: 165 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 166 | @echo "Testing of doctests in the sources finished, look at the " \ 167 | "results in $(BUILDDIR)/doctest/output.txt." 168 | 169 | xml: 170 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 171 | @echo 172 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 173 | 174 | pseudoxml: 175 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 176 | @echo 177 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 178 | -------------------------------------------------------------------------------- /docs/source/api_documentation.rst: -------------------------------------------------------------------------------- 1 | .. :mod:`sherlock` documentation master file, created by 2 | sphinx-quickstart on Wed Jan 22 11:28:21 2014. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | API Documentation 7 | ================= 8 | 9 | Configuration 10 | +++++++++++++ 11 | 12 | :mod:`sherlock` can be globally configured to set a lot of defaults using the 13 | :func:`sherlock.configure` function. The configuration set using 14 | :func:`sherlock.configure` will be used as the default for all the lock 15 | objects. 16 | 17 | .. autofunction:: sherlock.configure 18 | 19 | Understanding the configuration parameters 20 | ------------------------------------------ 21 | 22 | :func:`sherlock.configure` accepts certain configuration patterns which are 23 | individually explained in this section. Some of these configurations are global 24 | configurations and cannot be overriden while others can be overriden at 25 | individual lock levels. 26 | 27 | ``backend`` 28 | ~~~~~~~~~~~ 29 | 30 | The `backend` parameter allows you to set which backend you would like to use 31 | with the :class:`sherlock.Lock` class. When set to a particular backend, 32 | instances of :class:`sherlock.Lock` will use this backend for lock 33 | synchronization. 34 | 35 | Basic Usage: 36 | 37 | >>> import sherlock 38 | >>> sherlock.configure(backend=sherlock.backends.REDIS) 39 | 40 | .. note:: this configuration cannot be overriden at the time of creating a 41 | class object. 42 | 43 | Available Backends 44 | """""""""""""""""" 45 | 46 | To set the `backend` global configuration, you would have to choose one from 47 | the defined backends. The defined backends are: 48 | 49 | * Etcd: :attr:`sherlock.backends.ETCD` 50 | * Memcache: :attr:`sherlock.backends.MEMCACHE` 51 | * Redis: :attr:`sherlock.backends.REDIS` 52 | 53 | ``client`` 54 | ~~~~~~~~~~ 55 | 56 | The `client` parameter allows you to set a custom clien object which 57 | :mod:`sherlock` can use for connecting to the backend store. This gives you the 58 | flexibility to connect to the backend store from the client the way you want. 59 | The provided custom client object must be a valid client object of the 60 | supported client libraries. If the global `backend` has been set, then the 61 | provided custom client object must be an instance of the client library 62 | supported by that backend. If the backend has not been set, then the custom 63 | client object must be an instance of a valid supported client. In this case, 64 | :mod:`sherlock` will set the backend by instrospecting the type of the provided 65 | client object. 66 | 67 | The global default client object set using the `client` parameter will be used 68 | only by :class:`sherlock.Lock` instances. Other :ref:`backend-specific-locks` 69 | will either use the provided client object at the time of instantiating the 70 | lock object of their types or will default to creating a simple client object 71 | by themselves for their backend store, which will assume that their backend 72 | store is running on localhost. 73 | 74 | .. note:: this configuration cannot be overriden at the time of creating a 75 | class object. 76 | 77 | Example: 78 | 79 | >>> import redis 80 | >>> import sherlock 81 | >>> 82 | >>> # Configure just the backend 83 | >>> sherlock.configure(backend=sherlock.backends.REDIS) 84 | >>> 85 | >>> # Configure the global client object. This sets the client for all the 86 | >>> # locks. 87 | >>> sherlock.configure(client=redis.StrictRedis()) 88 | 89 | And when the provided client object does not match the supported library for 90 | the set backend: 91 | 92 | >>> import etcd 93 | >>> import sherlock 94 | >>> 95 | >>> sherlock.configure(backend=sherlock.backends.REDIS) 96 | >>> sherlock.configure(client=etcd.Client()) 97 | Traceback (most recent call last): 98 | File "", line 1, in 99 | File "/Users/vkapoor/Development/sherlock/sherlock/__init__.py", line 148, in configure 100 | _configuration.update(**kwargs) 101 | File "/Users/vkapoor/Development/sherlock/sherlock/__init__.py", line 250, in update 102 | setattr(self, key, val) 103 | File "/Users/vkapoor/Development/sherlock/sherlock/__init__.py", line 214, in client 104 | self.backend['name'])) 105 | ValueError: Only a client of the redis library can be used when using REDIS as the backend store option. 106 | 107 | And when the backend is not configured: 108 | 109 | >>> import redis 110 | >>> import sherlock 111 | >>> 112 | >>> # Congiure just the client, this will configure the backend to 113 | >>> # sherlock.backends.REDIS automatically. 114 | >>> sherlock.configure(client=redis.StrictRedis()) 115 | 116 | And when the client object passed as argument for client parameter is not a 117 | valid client object at all: 118 | 119 | >>> import sherlock 120 | >>> sherlock.configure(client=object()) 121 | Traceback (most recent call last): 122 | File "", line 1, in 123 | File "/Users/vkapoor/Development/sherlock/sherlock/__init__.py", line 148, in configure 124 | _configuration.update(**kwargs) 125 | File "/Users/vkapoor/Development/sherlock/sherlock/__init__.py", line 250, in update 126 | setattr(self, key, val) 127 | File "/Users/vkapoor/Development/sherlock/sherlock/__init__.py", line 221, in client 128 | raise ValueError('The provided object is not a valid client' 129 | ValueError: The provided object is not a valid client object. Client objects can only be instances of redis library's client class, python-etcd library's client class or pylibmc library's client class. 130 | 131 | ``namespace`` 132 | ~~~~~~~~~~~~~ 133 | 134 | The ``namespace`` parameter allows you to configure :mod:`sherlock` to set keys 135 | for synchronizing locks with a namespace so that if you are using the same 136 | datastore for something else, the keys set by locks don't conflict with your 137 | other keys set by your application. 138 | 139 | In case of Redis and Memcached, the name of your locks are prepended with 140 | ``NAMESPACE_`` (where ``NAMESPACE`` is the namespace set by you). In case of 141 | Etcd, a directory with the name same as the ``NAMESPACE`` you provided is 142 | created and the locks are created in that directory. 143 | 144 | By default, :mod:`sherlock` does not namespace the keys set for locks. 145 | 146 | .. note:: this configuration can be overriden at the time of creating a 147 | class object. 148 | 149 | ``expire`` 150 | ~~~~~~~~~~ 151 | 152 | This parameter can be used to set the expiry of locks. When set to :obj:`None`, 153 | the locks will never expire. 154 | 155 | This parameter's value defaults to ``60 seconds``. 156 | 157 | .. note:: this configuration can be overriden at the time of creating a 158 | class object. 159 | 160 | Example: 161 | 162 | >>> import sherlock 163 | >>> import time 164 | >>> 165 | >>> # Configure locks to expire after 2 seconds 166 | >>> sherlock.configure(expire=2) 167 | >>> 168 | >>> lock = sherlock.Lock('my_lock') 169 | >>> 170 | >>> # Acquire the lock 171 | >>> lock.acquire() 172 | True 173 | >>> 174 | >>> # Sleep for 2 seconds to let the lock expire 175 | >>> time.sleep(2) 176 | >>> 177 | >>> # Acquire the lock 178 | >>> lock.acquire() 179 | True 180 | 181 | ``timeout`` 182 | ~~~~~~~~~~~ 183 | 184 | This parameter can be used to set after how much time should :mod:`sherlock` 185 | stop trying to acquire an already acquired lock. 186 | 187 | This parameter's value defaults to ``10 seconds``. 188 | 189 | .. note:: this configuration can be overriden at the time of creating a 190 | class object. 191 | 192 | Example: 193 | 194 | >>> import sherlock 195 | >>> 196 | >>> # Configure locks to timeout after 2 seconds while trying to acquire an 197 | >>> # already acquired lock 198 | >>> sherlock.configure(timeout=2, expire=10) 199 | >>> 200 | >>> lock = sherlock.Lock('my_lock') 201 | >>> 202 | >>> # Acquire the lock 203 | >>> lock.acquire() 204 | True 205 | >>> 206 | >>> # Acquire the lock again and let the timeout elapse 207 | >>> lock.acquire() 208 | Traceback (most recent call last): 209 | File "", line 1, in 210 | File "sherlock/lock.py", line 170, in acquire 211 | 'lock.' % self.timeout) 212 | sherlock.lock.LockTimeoutException: Timeout elapsed after 2 seconds while trying to acquiring lock. 213 | 214 | ``retry_interval`` 215 | ~~~~~~~~~~~~~~~~~~ 216 | 217 | This parameter can be used to set after how much time should :mod:`sherlock` 218 | try to acquire lock after failing to acquire it. For example, a log has already 219 | been acquired, then when another lock object tries to acquire the same lock, 220 | it fails. This parameter sets the time interval for which we should sleep 221 | before retrying to acquire the lock. 222 | 223 | This parameter can be set to 0 to continuously try acquiring the lock. But that 224 | will also mean that you are bombarding your datastore with requests one after 225 | another. 226 | 227 | This parameter's value defaults to ``0.1 seconds (100 milliseconds)``. 228 | 229 | .. note:: this configuration can be overriden at the time of creating a 230 | class object. 231 | 232 | Generic Locks 233 | +++++++++++++ 234 | 235 | :mod:`sherlock` provides generic locks that can be globally configured to use a 236 | specific backend, so that most of your application code does not have to care 237 | about which backend you are using for lock synchronization and makes it easy 238 | to change backend without changing a ton of code. 239 | 240 | .. autoclass:: sherlock.Lock 241 | :members: 242 | :inherited-members: 243 | 244 | .. _backend-specific-locks: 245 | 246 | Backend Specific Locks 247 | ++++++++++++++++++++++ 248 | 249 | :mod:`sherlock` provides backend specific Lock classes as well which can be optionally 250 | used to use different backend than a globally configured backend. These locks 251 | have the same interface and semantics as :class:`sherlock.Lock`. 252 | 253 | Redis based Locks 254 | ----------------- 255 | 256 | .. autoclass:: sherlock.RedisLock 257 | :members: 258 | :inherited-members: 259 | 260 | Etcd based Locks 261 | ---------------- 262 | 263 | .. autoclass:: sherlock.EtcdLock 264 | :members: 265 | :inherited-members: 266 | 267 | Memcached based Locks 268 | --------------------- 269 | 270 | .. autoclass:: sherlock.MCLock 271 | :members: 272 | :inherited-members: 273 | 274 | Indices and tables 275 | ================== 276 | 277 | * :ref:`genindex` 278 | * :ref:`modindex` 279 | * :ref:`search` 280 | 281 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Sherlock documentation build configuration file, created by 4 | # sphinx-quickstart on Wed Jan 22 11:28:21 2014. 5 | # 6 | # This file is execfile()d with the current directory set to its 7 | # containing dir. 8 | # 9 | # Note that not all possible configuration values are present in this 10 | # autogenerated file. 11 | # 12 | # All configuration values have a default; values that are commented out 13 | # serve to show the default. 14 | 15 | import sys 16 | import os 17 | 18 | # If extensions (or modules to document with autodoc) are in another directory, 19 | # add these directories to sys.path here. If the directory is relative to the 20 | # documentation root, use os.path.abspath to make it absolute, like shown here. 21 | #sys.path.insert(0, os.path.abspath('.')) 22 | 23 | # -- General configuration ------------------------------------------------ 24 | 25 | # If your documentation needs a minimal Sphinx version, state it here. 26 | #needs_sphinx = '1.0' 27 | 28 | # Add any Sphinx extension module names here, as strings. They can be 29 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 30 | # ones. 31 | extensions = [ 32 | 'sphinx.ext.autodoc', 33 | 'sphinx.ext.doctest', 34 | 'sphinx.ext.intersphinx', 35 | 'sphinx.ext.todo', 36 | 'sphinx.ext.coverage', 37 | 'sphinx.ext.viewcode', 38 | ] 39 | 40 | # Add any paths that contain templates here, relative to this directory. 41 | templates_path = ['_templates'] 42 | 43 | # The suffix of source filenames. 44 | source_suffix = '.rst' 45 | 46 | # The encoding of source files. 47 | #source_encoding = 'utf-8-sig' 48 | 49 | # The master toctree document. 50 | master_doc = 'index' 51 | 52 | # General information about the project. 53 | project = u'Sherlock' 54 | copyright = u'2014, Vaidik Kapoor' 55 | 56 | # The version info for the project you're documenting, acts as replacement for 57 | # |version| and |release|, also used in various other places throughout the 58 | # built documents. 59 | # 60 | # The short X.Y version. 61 | version = '0.4.1' 62 | # The full version, including alpha/beta/rc tags. 63 | release = '0.4.1' 64 | 65 | # The language for content autogenerated by Sphinx. Refer to documentation 66 | # for a list of supported languages. 67 | #language = None 68 | 69 | # There are two options for replacing |today|: either, you set today to some 70 | # non-false value, then it is used: 71 | #today = '' 72 | # Else, today_fmt is used as the format for a strftime call. 73 | #today_fmt = '%B %d, %Y' 74 | 75 | # List of patterns, relative to source directory, that match files and 76 | # directories to ignore when looking for source files. 77 | exclude_patterns = [] 78 | 79 | # The reST default role (used for this markup: `text`) to use for all 80 | # documents. 81 | #default_role = None 82 | 83 | # If true, '()' will be appended to :func: etc. cross-reference text. 84 | #add_function_parentheses = True 85 | 86 | # If true, the current module name will be prepended to all description 87 | # unit titles (such as .. function::). 88 | #add_module_names = True 89 | 90 | # If true, sectionauthor and moduleauthor directives will be shown in the 91 | # output. They are ignored by default. 92 | #show_authors = False 93 | 94 | # The name of the Pygments (syntax highlighting) style to use. 95 | pygments_style = 'sphinx' 96 | 97 | # A list of ignored prefixes for module index sorting. 98 | #modindex_common_prefix = [] 99 | 100 | # If true, keep warnings as "system message" paragraphs in the built documents. 101 | #keep_warnings = False 102 | 103 | 104 | # -- Options for HTML output ---------------------------------------------- 105 | 106 | # The theme to use for HTML and HTML Help pages. See the documentation for 107 | # a list of builtin themes. 108 | html_theme = 'default' 109 | 110 | # Theme options are theme-specific and customize the look and feel of a theme 111 | # further. For a list of options available for each theme, see the 112 | # documentation. 113 | #html_theme_options = {} 114 | 115 | # Add any paths that contain custom themes here, relative to this directory. 116 | #html_theme_path = [] 117 | 118 | # The name for this set of Sphinx documents. If None, it defaults to 119 | # " v documentation". 120 | #html_title = None 121 | 122 | # A shorter title for the navigation bar. Default is the same as html_title. 123 | #html_short_title = None 124 | 125 | # The name of an image file (relative to this directory) to place at the top 126 | # of the sidebar. 127 | #html_logo = None 128 | 129 | # The name of an image file (within the static path) to use as favicon of the 130 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 131 | # pixels large. 132 | #html_favicon = None 133 | 134 | # Add any paths that contain custom static files (such as style sheets) here, 135 | # relative to this directory. They are copied after the builtin static files, 136 | # so a file named "default.css" will overwrite the builtin "default.css". 137 | html_static_path = ['_static'] 138 | 139 | # Add any extra paths that contain custom files (such as robots.txt or 140 | # .htaccess) here, relative to this directory. These files are copied 141 | # directly to the root of the documentation. 142 | #html_extra_path = [] 143 | 144 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 145 | # using the given strftime format. 146 | #html_last_updated_fmt = '%b %d, %Y' 147 | 148 | # If true, SmartyPants will be used to convert quotes and dashes to 149 | # typographically correct entities. 150 | #html_use_smartypants = True 151 | 152 | # Custom sidebar templates, maps document names to template names. 153 | #html_sidebars = {} 154 | 155 | # Additional templates that should be rendered to pages, maps page names to 156 | # template names. 157 | #html_additional_pages = {} 158 | 159 | # If false, no module index is generated. 160 | #html_domain_indices = True 161 | 162 | # If false, no index is generated. 163 | #html_use_index = True 164 | 165 | # If true, the index is split into individual pages for each letter. 166 | #html_split_index = False 167 | 168 | # If true, links to the reST sources are added to the pages. 169 | #html_show_sourcelink = True 170 | 171 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 172 | #html_show_sphinx = True 173 | 174 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 175 | #html_show_copyright = True 176 | 177 | # If true, an OpenSearch description file will be output, and all pages will 178 | # contain a tag referring to it. The value of this option must be the 179 | # base URL from which the finished HTML is served. 180 | #html_use_opensearch = '' 181 | 182 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 183 | #html_file_suffix = None 184 | 185 | # Output file base name for HTML help builder. 186 | htmlhelp_basename = 'Sherlockdoc' 187 | 188 | 189 | # -- Options for LaTeX output --------------------------------------------- 190 | 191 | latex_elements = { 192 | # The paper size ('letterpaper' or 'a4paper'). 193 | #'papersize': 'letterpaper', 194 | 195 | # The font size ('10pt', '11pt' or '12pt'). 196 | #'pointsize': '10pt', 197 | 198 | # Additional stuff for the LaTeX preamble. 199 | #'preamble': '', 200 | } 201 | 202 | # Grouping the document tree into LaTeX files. List of tuples 203 | # (source start file, target name, title, 204 | # author, documentclass [howto, manual, or own class]). 205 | latex_documents = [ 206 | ('index', 'Sherlock.tex', u'Sherlock Documentation', 207 | u'Vaidik Kapoor', 'manual'), 208 | ] 209 | 210 | # The name of an image file (relative to this directory) to place at the top of 211 | # the title page. 212 | #latex_logo = None 213 | 214 | # For "manual" documents, if this is true, then toplevel headings are parts, 215 | # not chapters. 216 | #latex_use_parts = False 217 | 218 | # If true, show page references after internal links. 219 | #latex_show_pagerefs = False 220 | 221 | # If true, show URL addresses after external links. 222 | #latex_show_urls = False 223 | 224 | # Documents to append as an appendix to all manuals. 225 | #latex_appendices = [] 226 | 227 | # If false, no module index is generated. 228 | #latex_domain_indices = True 229 | 230 | 231 | # -- Options for manual page output --------------------------------------- 232 | 233 | # One entry per manual page. List of tuples 234 | # (source start file, name, description, authors, manual section). 235 | man_pages = [ 236 | ('index', 'sherlock', u'Sherlock Documentation', 237 | [u'Vaidik Kapoor'], 1) 238 | ] 239 | 240 | # If true, show URL addresses after external links. 241 | #man_show_urls = False 242 | 243 | 244 | # -- Options for Texinfo output ------------------------------------------- 245 | 246 | # Grouping the document tree into Texinfo files. List of tuples 247 | # (source start file, target name, title, author, 248 | # dir menu entry, description, category) 249 | texinfo_documents = [ 250 | ('index', 'Sherlock', u'Sherlock Documentation', 251 | u'Vaidik Kapoor', 'Sherlock', 'One line description of project.', 252 | 'Miscellaneous'), 253 | ] 254 | 255 | # Documents to append as an appendix to all manuals. 256 | #texinfo_appendices = [] 257 | 258 | # If false, no module index is generated. 259 | #texinfo_domain_indices = True 260 | 261 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 262 | #texinfo_show_urls = 'footnote' 263 | 264 | # If true, do not generate a @detailmenu in the "Top" node's menu. 265 | #texinfo_no_detailmenu = False 266 | 267 | 268 | # Example configuration for intersphinx: refer to the Python standard library. 269 | intersphinx_mapping = {'http://docs.python.org/': None} 270 | 271 | # -- Custom Configuration ------------------------------------------------ 272 | 273 | # Document __init__ methods as well 274 | autoclass_content = 'both' 275 | 276 | # Use the old sphinx default theme on RTFD 277 | RTD_OLD_THEME = True 278 | 279 | # Mock libraries that cannot be installed on RTFD 280 | class Mock(object): 281 | def __init__(self, *args, **kwargs): 282 | pass 283 | 284 | def __call__(self, *args, **kwargs): 285 | return Mock() 286 | 287 | @classmethod 288 | def __getattr__(cls, name): 289 | if name in ('__file__', '__path__'): 290 | return '/dev/null' 291 | elif name[0] == name[0].upper(): 292 | mockType = type(name, (), {}) 293 | mockType.__module__ = __name__ 294 | return mockType 295 | else: 296 | return Mock() 297 | 298 | MOCK_MODULES = ['etcd', 'pylibmc', 'redis'] 299 | for mod_name in MOCK_MODULES: 300 | sys.modules[mod_name] = Mock() 301 | -------------------------------------------------------------------------------- /docs/source/extending.rst: -------------------------------------------------------------------------------- 1 | .. :mod:`sherlock` documentation master file, created by 2 | sphinx-quickstart on Wed Jan 22 11:28:21 2014. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | .. _extending: 7 | 8 | Extending 9 | ========= 10 | 11 | :mod:`sherlock` can be easily extended to work with any backend. You just have 12 | to register your lock's implementation with :mod:`sherlock` and you will be 13 | able to use your lock with the backend of your choice in your project. 14 | 15 | Registration 16 | ++++++++++++ 17 | 18 | Custom locks can be registered using the following API: 19 | 20 | .. automethod:: sherlock.backends.register 21 | 22 | Example 23 | +++++++ 24 | 25 | Here is an example of implementing a custom lock that uses `Elasticsearch`_ as 26 | backend. 27 | 28 | .. _Elasticsearch: http://elasticsearch.org 29 | 30 | .. note:: You may distributed your custom lock implementation as package if you 31 | please. Just make sure that you add :mod:`sherlock` as a dependency. 32 | 33 | The following code goes in a module called ``sherlock_es.py``. 34 | 35 | .. code:: python 36 | 37 | import elasticsearch 38 | import sherlock 39 | import uuid 40 | 41 | from elasticsearch import Elasticsearch 42 | from sherlock import LockException 43 | 44 | 45 | class ESLock(sherlock.lock.BaseLock): 46 | def __init__(self, lock_name, **kwargs): 47 | super(ESLock, self).__init__(lock_name, **kwargs) 48 | 49 | if self.client is None: 50 | self.client = Elasticsearch(hosts=['localhost:9200']) 51 | 52 | self._owner = None 53 | 54 | def _acquire(self): 55 | owner = uuid.uuid4().hex 56 | 57 | try: 58 | self.client.get(index='sherlock', doc_type='locks', 59 | id=self.lock_name) 60 | except elasticsearch.NotFoundError, err: 61 | self.client.index(index='sherlock', doc_type='locks', 62 | id=self.lock_name, body=dict(owner=owner)) 63 | self._owner = owner 64 | return True 65 | else: 66 | return False 67 | 68 | def _release(self): 69 | if self._owner is None: 70 | raise LockException('Lock was not set by this process.') 71 | 72 | try: 73 | resp = self.client.get(index='sherlock', doc_type='locks', 74 | id=self.lock_name) 75 | if resp['_source']['owner'] == self._owner: 76 | self.client.delete(index='sherlock', doc_type='locks', 77 | id=self.lock_name) 78 | 79 | else: 80 | raise LockException('Lock could not be released because it ' 81 | 'was not acquired by this process.') 82 | except elasticsearch.NotFoundError, err: 83 | raise LockException('Lock could not be released as it has not ' 84 | 'been acquired.') 85 | 86 | @property 87 | def _locked(self): 88 | try: 89 | self.client.get(index='sherlock', doc_type='locks', 90 | id=self.lock_name) 91 | return True 92 | except elasticsearch.NotFoundError, err: 93 | return False 94 | 95 | 96 | # Register the custom lock with sherlock 97 | sherlock.backends.register(name='ES', 98 | lock_class=ESLock, 99 | library='elasticsearch', 100 | client_class=Elasticsearch, 101 | default_args=(), 102 | default_kwargs={ 103 | 'hosts': ['localhost:9200'], 104 | }) 105 | 106 | Our module can be used like so: 107 | 108 | .. code:: python 109 | 110 | import sherlock 111 | import sherlock_es 112 | 113 | # Notice that ES is available as backend now 114 | sherlock.configure(backend=sherlock.backends.ES) 115 | 116 | lock1 = sherlock.Lock('test1') 117 | lock1.acquire() # True 118 | 119 | lock2 = sherlock_es.ESLock('test2') 120 | lock2.acquire() # True 121 | 122 | Indices and tables 123 | ================== 124 | 125 | * :ref:`genindex` 126 | * :ref:`modindex` 127 | * :ref:`search` 128 | 129 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. :mod:`sherlock` documentation master file, created by 2 | sphinx-quickstart on Wed Jan 22 11:28:21 2014. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | .. automodule:: sherlock 7 | 8 | Contents 9 | ======== 10 | 11 | .. toctree:: 12 | :maxdepth: 3 13 | 14 | api_documentation 15 | extending 16 | 17 | Indices and tables 18 | ================== 19 | 20 | * :ref:`genindex` 21 | * :ref:`modindex` 22 | * :ref:`search` 23 | 24 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool] 2 | 3 | [tool.poetry] 4 | name = "sherlock" 5 | version = "0.4.1" 6 | description = "Distributed inter-process locks with a choice of backend" 7 | license = "MIT" 8 | authors = [ 9 | "Vaidik Kapoor ", 10 | ] 11 | maintainers = [ 12 | "Vaidik Kapoor ", 13 | "Judah Rand <17158624+judahrand@users.noreply.github.com>", 14 | ] 15 | readme = "README.rst" 16 | homepage = "https://github.com/py-sherlock/sherlock" 17 | repository = "https://github.com/py-sherlock/sherlock" 18 | keywords = ["locking"] 19 | classifiers = [ 20 | "Development Status :: 2 - Pre-Alpha", 21 | "Intended Audience :: Developers", 22 | "Operating System :: OS Independent", 23 | "License :: OSI Approved :: MIT License", 24 | "Programming Language :: Python :: 3.7", 25 | "Programming Language :: Python :: 3.8", 26 | "Programming Language :: Python :: 3.9", 27 | "Programming Language :: Python :: 3.10", 28 | "Programming Language :: Python :: 3.11", 29 | ] 30 | packages = [ 31 | { include = "sherlock" }, 32 | ] 33 | 34 | [tool.poetry.dependencies] 35 | python = "^3.7" 36 | 37 | filelock = { version = "^3.7.1", optional = true } 38 | kubernetes = { version = "^24.2.0", optional = true } 39 | redis = { version = "^4.3.4", optional = true } 40 | python-etcd = { version = "^0.4.5", optional = true } 41 | pylibmc = { version = "^1.6.1", optional = true } 42 | 43 | [tool.poetry.group.dev.dependencies] 44 | black = "^22.8.0" 45 | flake8 = "^5.0.4" 46 | isort = "^5.10.1" 47 | mypy = "^0.981" 48 | pycodestyle = "^2.9.1" 49 | pytest = "^7.1.2" 50 | pytest-cov = "^3.0.0" 51 | tox = "^3.25.1" 52 | tox-docker = { git = "https://github.com/judahrand/tox-docker.git", branch = "judah" } 53 | tox-gh-actions = "^2.9.1" 54 | tox-poetry = "^0.4.1" 55 | 56 | types-filelock = "^3.2.7" 57 | types-redis = "^4.3.21" 58 | 59 | [tool.poetry.extras] 60 | all = [ 61 | "filelock", 62 | "kubernetes", 63 | "redis", 64 | "etcd", 65 | "pylibmc", 66 | ] 67 | filelock = ["filelock"] 68 | kubernetes = ["kubernetes"] 69 | redis = ["redis"] 70 | etcd = ["python-etcd"] 71 | memcached = ["pylibmc"] 72 | 73 | [tool.isort] 74 | profile = "black" 75 | src_paths = ["sherlock", "tests"] 76 | 77 | [build-system] 78 | requires = ["poetry_core>=1.0.0"] 79 | build-backend = "poetry.core.masonry.api" 80 | -------------------------------------------------------------------------------- /requirements-ci.txt: -------------------------------------------------------------------------------- 1 | pytest 2 | pytest-cov 3 | wheel 4 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | --pre tox>=4.0.0b2,<5.0 2 | # `tox-docker` with support for `privileged` and `command` 3 | # https://github.com/tox-dev/tox-docker/pull/138 4 | # https://github.com/tox-dev/tox-docker/pull/139 5 | git+https://github.com/judahrand/tox-docker.git@judah 6 | tox-gh 7 | -------------------------------------------------------------------------------- /sherlock/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Sherlock: Distributed Locks with a choice of backend 3 | ==================================================== 4 | 5 | :mod:`sherlock` is a library that provides easy-to-use distributed inter-process 6 | locks and also allows you to choose a backend of your choice for lock 7 | synchronization. 8 | 9 | |Build Status| |Coverage Status| 10 | 11 | .. |Build Status| image:: https://travis-ci.org/vaidik/sherlock.png 12 | :target: https://travis-ci.org/vaidik/sherlock/ 13 | .. |Coverage Status| image:: https://coveralls.io/repos/vaidik/incoming/badge.png 14 | :target: https://coveralls.io/r/vaidik/incoming 15 | 16 | Overview 17 | -------- 18 | 19 | When you are working with resources which are accessed by multiple services or 20 | distributed services, more than often you need some kind of locking mechanism 21 | to make it possible to access some resources at a time. 22 | 23 | Distributed Locks or Mutexes can help you with this. :mod:`sherlock` provides 24 | the exact same facility, with some extra goodies. It provides an easy-to-use API 25 | that resembles standard library's `threading.Lock` semantics. 26 | 27 | Apart from this, :mod:`sherlock` gives you the flexibilty of using a backend of 28 | your choice for managing locks. 29 | 30 | :mod:`sherlock` also makes it simple for you to extend :mod:`sherlock` to use 31 | backends that are not supported. 32 | 33 | Features 34 | ++++++++ 35 | 36 | * API similar to standard library's `threading.Lock`. 37 | * Support for With statement, to cleanly acquire and release locks. 38 | * Backend agnostic: supports `Redis`_, `Memcached`_ and `Etcd`_ as choice of 39 | backends. 40 | * Extendable: can be easily extended to work with any other of backend of 41 | choice by extending base lock class. Read :ref:`extending`. 42 | 43 | .. _Redis: http://redis.io 44 | .. _Memcached: http://memcached.org 45 | .. _Etcd: http://github.com/coreos/etcd 46 | 47 | Supported Backends and Client Libraries 48 | +++++++++++++++++++++++++++++++++++++++ 49 | 50 | Following client libraries are supported for every supported backend: 51 | 52 | * Redis: `redis-py`_ 53 | * Memcached: `pylibmc`_ 54 | * Etcd: `python-etcd`_ 55 | 56 | .. _redis-py: http://github.com 57 | .. _pylibmc: http://github.com 58 | .. _python-etcd: https://github.com/jplana/python-etcd 59 | 60 | As of now, only the above mentioned libraries are supported. Although 61 | :mod:`sherlock` takes custom client objects so that you can easily provide 62 | settings that you want to use for that backend store, but :mod:`sherlock` also 63 | checks if the provided client object is an instance of the supported clients 64 | and accepts client objects which pass this check, even if the APIs are the 65 | same. :mod:`sherlock` might get rid of this issue later, if need be and if 66 | there is a demand for that. 67 | 68 | Installation 69 | ------------ 70 | 71 | Installation is simple. 72 | 73 | .. code:: bash 74 | 75 | pip install sherlock 76 | 77 | .. note:: :mod:`sherlock` will install all the client libraries for all the 78 | supported backends. 79 | 80 | Basic Usage 81 | ----------- 82 | 83 | :mod:`sherlock` is simple to use as at the API and semantics level, it tries to 84 | conform to standard library's :mod:`threading.Lock` APIs. 85 | 86 | .. code-block:: python 87 | 88 | import sherlock 89 | from sherlock import Lock 90 | 91 | # Configure :mod:`sherlock`'s locks to use Redis as the backend, 92 | # never expire locks and retry acquiring an acquired lock after an 93 | # interval of 0.1 second. 94 | sherlock.configure(backend=sherlock.backends.REDIS, 95 | expire=None, 96 | retry_interval=0.1) 97 | 98 | # Note: configuring sherlock to use a backend does not limit you 99 | # another backend at the same time. You can import backend specific locks 100 | # like RedisLock, MCLock and EtcdLock and use them just the same way you 101 | # use a generic lock (see below). In fact, the generic Lock provided by 102 | # sherlock is just a proxy that uses these specific locks under the hood. 103 | 104 | # acquire a lock called my_lock 105 | lock = Lock('my_lock') 106 | 107 | # acquire a blocking lock 108 | lock.acquire() 109 | 110 | # check if the lock has been acquired or not 111 | lock.locked() == True 112 | 113 | # release the lock 114 | lock.release() 115 | 116 | Support for ``with`` statement 117 | ++++++++++++++++++++++++++++++ 118 | 119 | .. code-block:: python 120 | 121 | # using with statement 122 | with Lock('my_lock'): 123 | # do something constructive with your locked resource here 124 | pass 125 | 126 | Blocking and Non-blocking API 127 | +++++++++++++++++++++++++++++ 128 | 129 | .. code-block:: python 130 | 131 | # acquire non-blocking lock 132 | lock1 = Lock('my_lock') 133 | lock2 = Lock('my_lock') 134 | 135 | # successfully acquire lock1 136 | lock1.acquire() 137 | 138 | # try to acquire lock in a non-blocking way 139 | lock2.acquire(False) == True # returns False 140 | 141 | # try to acquire lock in a blocking way 142 | lock2.acquire() # blocks until lock is acquired to timeout happens 143 | 144 | Using two backends at the same time 145 | +++++++++++++++++++++++++++++++++++ 146 | 147 | Configuring :mod:`sherlock` to use a backend does not limit you from using 148 | another backend at the same time. You can import backend specific locks like 149 | RedisLock, MCLock and EtcdLock and use them just the same way you use a generic 150 | lock (see below). In fact, the generic Lock provided by :mod:`sherlock` is just 151 | a proxy that uses these specific locks under the hood. 152 | 153 | .. code-block:: python 154 | 155 | import sherlock 156 | from sherlock import Lock 157 | 158 | # Configure :mod:`sherlock`'s locks to use Redis as the backend 159 | sherlock.configure(backend=sherlock.backends.REDIS) 160 | 161 | # Acquire a lock called my_lock, this lock uses Redis 162 | lock = Lock('my_lock') 163 | 164 | # Now acquire locks in Memcached 165 | from sherlock import MCLock 166 | mclock = MCLock('my_mc_lock') 167 | mclock.acquire() 168 | 169 | Tests 170 | ----- 171 | 172 | To run all the tests (including integration), you have to make sure that all 173 | the databases are running. Make sure all the services are running: 174 | 175 | .. code:: bash 176 | 177 | # memcached 178 | memcached 179 | 180 | # redis-server 181 | redis-server 182 | 183 | # etcd (etcd is probably not available as package, here is the simplest way 184 | # to run it). 185 | wget https://github.com/coreos/etcd/releases/download//etcd--.tar.gz 186 | tar -zxvf etcd--.gz 187 | ./etcd--/etcd 188 | 189 | Run tests like so: 190 | 191 | .. code:: bash 192 | 193 | python setup.py test 194 | 195 | Documentation 196 | ------------- 197 | 198 | Available `here`_. 199 | 200 | .. _here: http://sher-lock.readthedocs.org 201 | 202 | Roadmap 203 | ------- 204 | 205 | * Support for `Zookeeper`_ as backend. 206 | * Support for `Gevent`_, `Multithreading`_ and `Multiprocessing`_. 207 | 208 | .. _Zookeeper: http://zookeeper.apache.org/ 209 | .. _Gevent: http://www.gevent.org/ 210 | .. _Multithreading: http://docs.python.org/2/library/multithreading.html 211 | .. _Multiprocessing: http://docs.python.org/2/library/multiprocessing.html 212 | 213 | License 214 | ------- 215 | 216 | See `LICENSE`_. 217 | 218 | **In short**: This is an open-source project and exists in the public domain 219 | for anyone to modify and use it. Just be nice and attribute the credits 220 | wherever you can. :) 221 | 222 | .. _LICENSE: http://github.com/vaidik/sherlock/blob/master/LICENSE.rst 223 | 224 | Distributed Locking in Other Languages 225 | -------------------------------------- 226 | 227 | * NodeJS - https://github.com/thedeveloper/warlock 228 | """ # noqa: disable=E501 229 | 230 | import pathlib 231 | 232 | # Import important Lock classes 233 | from . import lock 234 | from .lock import ( 235 | EtcdLock, 236 | FileLock, 237 | KubernetesLock, 238 | Lock, 239 | LockException, 240 | LockTimeoutException, 241 | MCLock, 242 | RedisLock, 243 | ) 244 | 245 | __all__ = [ 246 | "backends", 247 | "configure", 248 | "Lock", 249 | "LockException", 250 | "LockTimeoutException", 251 | "EtcdLock", 252 | "FileLock", 253 | "KubernetesLock", 254 | "MCLock", 255 | "RedisLock", 256 | ] 257 | 258 | 259 | class _Backends(object): 260 | """ 261 | A simple object that provides a list of available backends. 262 | """ 263 | 264 | _valid_backends = [] 265 | 266 | try: 267 | import redis 268 | 269 | REDIS = { 270 | "name": "REDIS", 271 | "library": "redis", 272 | "client_class": redis.StrictRedis, 273 | "lock_class": "RedisLock", 274 | "default_args": (), 275 | "default_kwargs": {}, 276 | } 277 | _valid_backends.append(REDIS) 278 | except ImportError: 279 | pass 280 | 281 | try: 282 | import etcd 283 | 284 | ETCD = { 285 | "name": "ETCD", 286 | "library": "etcd", 287 | "client_class": etcd.Client, 288 | "lock_class": "EtcdLock", 289 | "default_args": (), 290 | "default_kwargs": {}, 291 | } 292 | _valid_backends.append(ETCD) 293 | except ImportError: 294 | pass 295 | 296 | try: 297 | import pylibmc 298 | 299 | MEMCACHED = { 300 | "name": "MEMCACHED", 301 | "library": "pylibmc", 302 | "client_class": pylibmc.Client, 303 | "lock_class": "MCLock", 304 | "default_args": (["localhost"],), 305 | "default_kwargs": { 306 | "binary": True, 307 | }, 308 | } 309 | _valid_backends.append(MEMCACHED) 310 | except ImportError: 311 | pass 312 | 313 | try: 314 | import kubernetes.client 315 | 316 | KUBERNETES = { 317 | "name": "KUBERNETES", 318 | "library": "kubernetes", 319 | "client_class": kubernetes.client.CoordinationV1Api, 320 | "lock_class": "KubernetesLock", 321 | "default_args": (), 322 | "default_kwargs": {}, 323 | } 324 | _valid_backends.append(KUBERNETES) 325 | except ImportError: 326 | pass 327 | 328 | try: 329 | FILE = { 330 | "name": "FILE", 331 | "library": "pathlib", 332 | "client_class": pathlib.Path, 333 | "lock_class": "FileLock", 334 | "default_args": ("/tmp/sherlock",), 335 | "default_kwargs": {}, 336 | } 337 | _valid_backends.append(FILE) 338 | except ImportError: 339 | pass 340 | 341 | def register( 342 | self, 343 | name, 344 | lock_class, 345 | library, 346 | client_class, 347 | default_args=(), 348 | default_kwargs={}, 349 | ): 350 | """ 351 | Register a custom backend. 352 | 353 | :param str name: Name of the backend by which you would want to refer 354 | this backend in your code. 355 | :param class lock_class: the sub-class of 356 | :class:`sherlock.lock.BaseLock` that you have 357 | implemented. The reference to your implemented 358 | lock class will be used by 359 | :class:`sherlock.Lock` proxy to use your 360 | implemented class when you globally set that 361 | the choice of backend is the one that has been 362 | implemented by you. 363 | :param str library: dependent client library that this implementation 364 | makes use of. 365 | :param client_class: the client class or valid type which you use to 366 | connect the datastore. This is used by the 367 | :func:`configure` function to validate that 368 | the object provided for the `client` 369 | parameter is actually an instance of this class. 370 | :param tuple default_args: default arguments that need to passed to 371 | create an instance of the callable passed to 372 | `client_class` parameter. 373 | :param dict default_kwargs: default keyword arguments that need to 374 | passed to create an instance of the 375 | callable passed to `client_class` 376 | parameter. 377 | 378 | Usage: 379 | 380 | >>> import some_db_client 381 | >>> class MyLock(sherlock.lock.BaseLock): 382 | ... # your implementation comes here 383 | ... pass 384 | >>> 385 | >>> sherlock.configure(name='Mylock', 386 | ... lock_class=MyLock, 387 | ... library='some_db_client', 388 | ... client_class=some_db_client.Client, 389 | ... default_args=('localhost:1234'), 390 | ... default_kwargs=dict(connection_pool=6)) 391 | """ 392 | 393 | if not issubclass(lock_class, lock.BaseLock): 394 | raise ValueError( 395 | "lock_class parameter must be a sub-class of " "sherlock.lock.BaseLock" 396 | ) 397 | setattr( 398 | self, 399 | name, 400 | { 401 | "name": name, 402 | "lock_class": lock_class, 403 | "library": library, 404 | "client_class": client_class, 405 | "default_args": default_args, 406 | "default_kwargs": default_kwargs, 407 | }, 408 | ) 409 | 410 | valid_backends = list(self._valid_backends) 411 | valid_backends.append(getattr(self, name)) 412 | self._valid_backends = tuple(valid_backends) 413 | 414 | @property 415 | def valid_backends(self): 416 | """ 417 | Return a tuple of valid backends. 418 | 419 | :returns: a list of valid supported backends 420 | :rtype: tuple 421 | """ 422 | 423 | return self._valid_backends 424 | 425 | 426 | def configure(**kwargs): 427 | """ 428 | Set basic global configuration for :mod:`sherlock`. 429 | 430 | :param backend: global choice of backend. This backend will be used 431 | for managing locks by :class:`sherlock.Lock` class 432 | objects. 433 | :param client: global client object to use to connect with backend 434 | store. This client object will be used to connect to the 435 | backend store by :class:`sherlock.Lock` class instances. 436 | The client object must be a valid object of the client 437 | library. If the backend has been configured using the 438 | `backend` parameter, the custom client object must belong 439 | to the same library that is supported for that backend. 440 | If the backend has not been set, then the custom client 441 | object must be an instance of a valid supported client. 442 | In that case, :mod:`sherlock` will set the backend by 443 | introspecting the type of provided client object. 444 | :param str namespace: provide global namespace 445 | :param float expire: provide global expiration time. If expicitly set to 446 | `None`, lock will not expire. 447 | :param float timeout: provide global timeout period 448 | :param float retry_interval: provide global retry interval 449 | 450 | Basic Usage: 451 | 452 | >>> import sherlock 453 | >>> from sherlock import Lock 454 | >>> 455 | >>> # Configure sherlock to use Redis as the backend and the timeout for 456 | >>> # acquiring locks equal to 20 seconds. 457 | >>> sherlock.configure(timeout=20, backend=sherlock.backends.REDIS) 458 | >>> 459 | >>> import redis 460 | >>> redis_client = redis.StrictRedis(host='X.X.X.X', port=6379, db=1) 461 | >>> sherlock.configure(client=redis_client) 462 | """ 463 | 464 | _configuration.update(**kwargs) 465 | 466 | 467 | class _Configuration(object): 468 | def __init__(self): 469 | # Choice of backend 470 | self._backend = None 471 | 472 | # Client object to connect with the backend store 473 | self._client = None 474 | 475 | # Namespace to use for setting lock keys in the backend store 476 | self.namespace = None 477 | 478 | # Lock expiration time. If explicitly set to `None`, lock will not 479 | # expire. 480 | self.expire = 60 481 | 482 | # Timeout to acquire lock 483 | self.timeout = 10 484 | 485 | # Retry interval to retry acquiring a lock if previous attempts failed 486 | self.retry_interval = 0.1 487 | 488 | @property 489 | def backend(self): 490 | return self._backend 491 | 492 | @backend.setter 493 | def backend(self, val): 494 | if val not in backends.valid_backends: 495 | backend_names = list( 496 | map( 497 | lambda x: "sherlock.backends.%s" % x["name"], 498 | backends.valid_backends, 499 | ) 500 | ) 501 | error_str = ", ".join(backend_names[:-1]) 502 | backend_names = "%s and %s" % (error_str, backend_names[-1]) 503 | raise ValueError( 504 | "Invalid backend. Valid backends are: " "%s." % backend_names 505 | ) 506 | 507 | self._backend = val 508 | 509 | @property 510 | def client(self): 511 | if self._client is not None: 512 | return self._client 513 | else: 514 | if self.backend is None: 515 | raise ValueError( 516 | "Cannot create a default client object when " 517 | "backend is not configured." 518 | ) 519 | 520 | for backend in backends.valid_backends: 521 | if self.backend == backend: 522 | self.client = self.backend["client_class"]( 523 | *self.backend["default_args"], **self.backend["default_kwargs"] 524 | ) 525 | return self._client 526 | 527 | @client.setter 528 | def client(self, val): 529 | # When backend is set, check client type 530 | if self.backend is not None: 531 | exc_msg = ( 532 | "Only a client of the %s library can be used " 533 | "when using %s as the backend store option." 534 | ) 535 | if isinstance(val, self.backend["client_class"]): 536 | self._client = val 537 | else: 538 | raise ValueError( 539 | exc_msg % (self.backend["library"], self.backend["name"]) 540 | ) 541 | else: 542 | for backend in backends.valid_backends: 543 | if isinstance(val, backend["client_class"]): 544 | self._client = val 545 | self.backend = backend 546 | if self._client is None: 547 | raise ValueError( 548 | "The provided object is not a valid client" 549 | "object. Client objects can only be " 550 | "instances of redis library's client class, " 551 | "python-etcd library's client class or " 552 | "pylibmc library's client class." 553 | ) 554 | 555 | def update(self, **kwargs): 556 | """ 557 | Update configuration. Provide keyword arguments where the keyword 558 | parameter is the configuration and its value (the argument) is the 559 | value you intend to set. 560 | 561 | :param backend: global choice of backend. This backend will be used 562 | for managing locks. 563 | :param client: global client object to use to connect with backend 564 | store. 565 | :param str namespace: optional global namespace to namespace lock keys 566 | for your application in order to avoid conflicts. 567 | :param float expire: set lock expiry time. If explicitly set to `None`, 568 | lock will not expire. 569 | :param float timeout: global timeout for acquiring a lock. 570 | :param float retry_interval: global timeout for retrying to acquire the 571 | lock if previous attempts failed. 572 | """ 573 | 574 | for key, val in kwargs.items(): 575 | if key not in dir(self): 576 | raise AttributeError( 577 | "Invalid configuration. No such " "configuration as %s." % key 578 | ) 579 | setattr(self, key, val) 580 | 581 | 582 | # Create a backends singleton 583 | backends = _Backends() 584 | 585 | # Create a configuration singleton 586 | _configuration = _Configuration() 587 | -------------------------------------------------------------------------------- /sherlock/lock.py: -------------------------------------------------------------------------------- 1 | """ 2 | lock 3 | ~~~~ 4 | 5 | A generic lock. 6 | """ 7 | from __future__ import annotations 8 | 9 | import datetime 10 | import importlib 11 | import json 12 | import pathlib 13 | import re 14 | import time 15 | import typing 16 | import uuid 17 | 18 | if typing.TYPE_CHECKING: 19 | import etcd 20 | import filelock 21 | import kubernetes 22 | 23 | 24 | __all__ = [ 25 | "Lock", 26 | "LockException", 27 | "LockTimeoutException", 28 | "RedisLock", 29 | "EtcdLock", 30 | "MCLock", 31 | "KubernetesLock", 32 | "FileLock", 33 | ] 34 | 35 | 36 | class LockException(Exception): 37 | """ 38 | Generic exception for Locks. 39 | """ 40 | 41 | pass 42 | 43 | 44 | class LockTimeoutException(Exception): 45 | """ 46 | Raised whenever timeout occurs while trying to acquire lock. 47 | """ 48 | 49 | pass 50 | 51 | 52 | class BaseLock(object): 53 | """ 54 | Interface for implementing custom Lock implementations. This class must be 55 | sub-classed in order to implement a custom Lock with custom logic or 56 | different backend or both. 57 | 58 | Basic Usage (an example of our imaginary datastore) 59 | 60 | >>> class MyLock(BaseLock): 61 | ... def __init__(self, lock_name, **kwargs): 62 | ... super(MyLock, self).__init__(lock_name, **kwargs) 63 | ... if self.client is None: 64 | ... self.client = mybackend.Client(host='localhost', port=1234) 65 | ... self._owner = None 66 | ... 67 | ... def _acquire(self): 68 | ... if self.client.get(self.lock_name) is not None: 69 | ... owner = str(uuid.uuid4()) # or anythin you want 70 | ... self.client.set(self.lock_name, owner) 71 | ... self._owner = owner 72 | ... if self.expire is not None: 73 | ... self.client.expire(self.lock_name, self.expire) 74 | ... return True 75 | ... return False 76 | ... 77 | ... def _release(self): 78 | ... if self._owner is not None: 79 | ... lock_val = self.client.get(self.lock_name) 80 | ... if lock_val == self._owner: 81 | ... self.client.delete(self.lock_name) 82 | ... 83 | ... def _locked(self): 84 | ... if self.client.get(self.lock_name) is not None: 85 | ... return True 86 | ... return False 87 | """ 88 | 89 | def __init__(self, lock_name, **kwargs): 90 | """ 91 | :param str lock_name: name of the lock to uniquely identify the lock 92 | between processes. 93 | :param str namespace: Optional namespace to namespace lock keys for 94 | your application in order to avoid conflicts. 95 | :param float expire: set lock expiry time. If explicitly set to `None`, 96 | lock will not expire. 97 | :param float timeout: set timeout to acquire lock 98 | :param float retry_interval: set interval for trying acquiring lock 99 | after the timeout interval has elapsed. 100 | :param client: supported client object for the backend of your choice. 101 | """ 102 | # Lazy to avoid circular 103 | from . import _configuration 104 | 105 | self.lock_name = lock_name 106 | 107 | if kwargs.get("namespace"): 108 | self.namespace = kwargs["namespace"] 109 | else: 110 | self.namespace = _configuration.namespace 111 | 112 | if "expire" in kwargs: 113 | self.expire = kwargs["expire"] 114 | else: 115 | self.expire = _configuration.expire 116 | 117 | if "timeout" in kwargs: 118 | self.timeout = kwargs["timeout"] 119 | else: 120 | self.timeout = _configuration.timeout 121 | 122 | if "retry_interval" in kwargs: 123 | self.retry_interval = kwargs["retry_interval"] 124 | else: 125 | self.retry_interval = _configuration.retry_interval 126 | 127 | if kwargs.get("client"): 128 | self.client = kwargs["client"] 129 | else: 130 | self.client = None 131 | 132 | @property 133 | def _locked(self): 134 | """ 135 | Implementation of method to check if lock has been acquired. Must be 136 | implemented in the sub-class. 137 | 138 | :returns: if the lock is acquired or not 139 | :rtype: bool 140 | """ 141 | 142 | raise NotImplementedError("Must be implemented in the sub-class.") 143 | 144 | def locked(self): 145 | """ 146 | Return if the lock has been acquired or not. 147 | 148 | :returns: True indicating that a lock has been acquired ot a 149 | shared resource is locked. 150 | :rtype: bool 151 | """ 152 | 153 | return self._locked 154 | 155 | def _acquire(self): 156 | """ 157 | Implementation of acquiring a lock in a non-blocking fashion. Must be 158 | implemented in the sub-class. :meth:`acquire` makes use of this 159 | implementation to provide blocking and non-blocking implementations. 160 | 161 | :returns: if the lock was successfully acquired or not 162 | :rtype: bool 163 | """ 164 | 165 | raise NotImplementedError("Must be implemented in the sub-class.") 166 | 167 | def acquire(self, blocking=True): 168 | """ 169 | Acquire a lock, blocking or non-blocking. 170 | 171 | :param bool blocking: acquire a lock in a blocking or non-blocking 172 | fashion. Defaults to True. 173 | :returns: if the lock was successfully acquired or not 174 | :rtype: bool 175 | """ 176 | 177 | if blocking is True: 178 | timeout = self.timeout 179 | while timeout >= 0: 180 | if self._acquire() is not True: 181 | timeout -= self.retry_interval 182 | if timeout > 0: 183 | time.sleep(self.retry_interval) 184 | else: 185 | return True 186 | raise LockTimeoutException( 187 | "Timeout elapsed after %s seconds " 188 | "while trying to acquiring " 189 | "lock." % self.timeout 190 | ) 191 | else: 192 | return self._acquire() 193 | 194 | def _release(self): 195 | """ 196 | Implementation of releasing an acquired lock. Must be implemented in 197 | the sub-class. 198 | """ 199 | 200 | raise NotImplementedError("Must be implemented in the sub-class.") 201 | 202 | def release(self): 203 | """ 204 | Release a lock. 205 | """ 206 | 207 | return self._release() 208 | 209 | def _renew(self) -> bool: 210 | """ 211 | Implementation of renewing an acquired lock. Must be implemented in 212 | the sub-class. 213 | """ 214 | raise NotImplementedError("Must be implemented in the sub-class") 215 | 216 | def renew(self) -> bool: 217 | """ 218 | Renew a lock that is already acquired. 219 | """ 220 | return self._renew() 221 | 222 | def __enter__(self): 223 | self.acquire() 224 | return self 225 | 226 | def __exit__(self, exc_type, exc_value, traceback): 227 | self.release() 228 | 229 | def __del__(self): 230 | try: 231 | self.release() 232 | except LockException: 233 | pass 234 | 235 | 236 | class Lock(BaseLock): 237 | """ 238 | A general lock that inherits global coniguration and provides locks with 239 | the configured backend. 240 | 241 | .. note:: to use :class:`Lock` class, you must configure the global backend 242 | to use a particular backend. If the global backend is not set, 243 | calling any method on instances of :class:`Lock` will throw 244 | exceptions. 245 | 246 | Basic Usage: 247 | 248 | >>> import sherlock 249 | >>> from sherlock import Lock 250 | >>> 251 | >>> sherlock.configure(sherlock.backends.REDIS) 252 | >>> 253 | >>> # Create a lock instance 254 | >>> lock = Lock('my_lock') 255 | >>> 256 | >>> # Acquire a lock in Redis running on localhost 257 | >>> lock.acquire() 258 | True 259 | >>> 260 | >>> # Check if the lock has been acquired 261 | >>> lock.locked() 262 | True 263 | >>> 264 | >>> # Release the acquired lock 265 | >>> lock.release() 266 | >>> 267 | >>> # Check if the lock has been acquired 268 | >>> lock.locked() 269 | False 270 | >>> 271 | >>> import redis 272 | >>> redis_client = redis.StrictRedis(host='X.X.X.X', port=6379, db=2) 273 | >>> sherlock.configure(client=redis_client) 274 | >>> 275 | >>> # Acquire a lock in Redis running on X.X.X.X:6379 276 | >>> lock.acquire() 277 | >>> 278 | >>> lock.locked() 279 | True 280 | >>> 281 | >>> # Acquire a lock using the with_statement 282 | >>> with Lock('my_lock') as lock: 283 | ... # do some stuff with your acquired resource 284 | ... pass 285 | """ 286 | 287 | def __init__(self, lock_name, **kwargs): 288 | """ 289 | :param str lock_name: name of the lock to uniquely identify the lock 290 | between processes. 291 | :param str namespace: Optional namespace to namespace lock keys for 292 | your application in order to avoid conflicts. 293 | :param float expire: set lock expiry time. If explicitly set to `None`, 294 | lock will not expire. 295 | :param float timeout: set timeout to acquire lock 296 | :param float retry_interval: set interval for trying acquiring lock 297 | after the timeout interval has elapsed. 298 | 299 | .. Note:: this Lock object does not accept a custom lock backend store 300 | client object. It instead uses the global custom client 301 | object. 302 | """ 303 | # Lazy to avoid circular 304 | from . import _configuration 305 | 306 | # Raise exception if client keyword argument is found 307 | if "client" in kwargs: 308 | raise TypeError("Lock object does not accept a custom client " "object") 309 | super(Lock, self).__init__(lock_name, **kwargs) 310 | 311 | try: 312 | self.client = _configuration.client 313 | except ValueError: 314 | pass 315 | 316 | if self.client is None: 317 | self._lock_proxy = None 318 | else: 319 | kwargs.update(client=_configuration.client) 320 | try: 321 | self._lock_proxy = globals()[_configuration.backend["lock_class"]]( 322 | lock_name, **kwargs 323 | ) 324 | except KeyError: 325 | self._lock_proxy = _configuration.backend["lock_class"]( 326 | lock_name, **kwargs 327 | ) 328 | 329 | def _acquire(self): 330 | if self._lock_proxy is None: 331 | raise LockException( 332 | "Lock backend has not been configured and " 333 | "lock cannot be acquired or released. " 334 | "Configure lock backend first." 335 | ) 336 | return self._lock_proxy.acquire(False) 337 | 338 | def _release(self): 339 | if self._lock_proxy is None: 340 | raise LockException( 341 | "Lock backend has not been configured and " 342 | "lock cannot be acquired or released. " 343 | "Configure lock backend first." 344 | ) 345 | return self._lock_proxy.release() 346 | 347 | def _renew(self) -> bool: 348 | if self._lock_proxy is None: 349 | raise LockException( 350 | "Lock backend has not been configured and " 351 | "lock cannot be acquired or released. " 352 | "Configure lock backend first." 353 | ) 354 | return self._lock_proxy.renew() 355 | 356 | @property 357 | def _locked(self): 358 | if self._lock_proxy is None: 359 | raise LockException( 360 | "Lock backend has not been configured and " 361 | "lock cannot be acquired or released. " 362 | "Configure lock backend first." 363 | ) 364 | return self._lock_proxy.locked() 365 | 366 | 367 | class RedisLock(BaseLock): 368 | """ 369 | Implementation of lock with Redis as the backend for synchronization. 370 | 371 | Basic Usage: 372 | 373 | >>> import redis 374 | >>> import sherlock 375 | >>> from sherlock import RedisLock 376 | >>> 377 | >>> # Global configuration of defaults 378 | >>> sherlock.configure(expire=120, timeout=20) 379 | >>> 380 | >>> # Create a lock instance 381 | >>> lock = RedisLock('my_lock') 382 | >>> 383 | >>> # Acquire a lock in Redis, global backend and client configuration need 384 | >>> # not be configured since we are using a backend specific lock. 385 | >>> lock.acquire() 386 | True 387 | >>> 388 | >>> # Check if the lock has been acquired 389 | >>> lock.locked() 390 | True 391 | >>> 392 | >>> # Release the acquired lock 393 | >>> lock.release() 394 | >>> 395 | >>> # Check if the lock has been acquired 396 | >>> lock.locked() 397 | False 398 | >>> 399 | >>> # Use this client object 400 | >>> client = redis.StrictRedis() 401 | >>> 402 | >>> # Create a lock instance with custom client object 403 | >>> lock = RedisLock('my_lock', client=client) 404 | >>> 405 | >>> # To override the defaults, just past the configurations as parameters 406 | >>> lock = RedisLock('my_lock', client=client, expire=1, timeout=5) 407 | >>> 408 | >>> # Acquire a lock using the with_statement 409 | >>> with RedisLock('my_lock') as lock: 410 | ... # do some stuff with your acquired resource 411 | ... pass 412 | """ 413 | 414 | _acquire_script = """ 415 | local result = redis.call('SETNX', KEYS[1], ARGV[1]) 416 | if result == 1 and ARGV[2] ~= -1 then 417 | redis.call('EXPIRE', KEYS[1], ARGV[2]) 418 | end 419 | return result 420 | """ 421 | 422 | _release_script = """ 423 | local result = 0 424 | if redis.call('GET', KEYS[1]) == ARGV[1] then 425 | redis.call('DEL', KEYS[1]) 426 | result = 1 427 | end 428 | return result 429 | """ 430 | 431 | _renew_script = """ 432 | local result = 0 433 | if redis.call('GET', KEYS[1]) == ARGV[1] then 434 | if ARGV[2] ~= -1 then 435 | redis.call('EXPIRE', KEYS[1], ARGV[2]) 436 | else 437 | redis.call('PERSIST', KEYS[1]) 438 | end 439 | result = 1 440 | end 441 | return result 442 | """ 443 | 444 | def __init__(self, lock_name, **kwargs): 445 | """ 446 | :param str lock_name: name of the lock to uniquely identify the lock 447 | between processes. 448 | :param str namespace: Optional namespace to namespace lock keys for 449 | your application in order to avoid conflicts. 450 | :param float expire: set lock expiry time. If explicitly set to `None`, 451 | lock will not expire. 452 | :param float timeout: set timeout to acquire lock 453 | :param float retry_interval: set interval for trying acquiring lock 454 | after the timeout interval has elapsed. 455 | :param client: supported client object for the backend of your choice. 456 | """ 457 | try: 458 | import redis 459 | except ImportError as exc: 460 | raise ImportError("Please install `sherlock` with `redis` extras.") from exc 461 | 462 | super(RedisLock, self).__init__(lock_name, **kwargs) 463 | 464 | if self.client is None: 465 | self.client = redis.StrictRedis(host="localhost", port=6379, db=0) 466 | 467 | self._owner = None 468 | 469 | # Register Lua script 470 | self._acquire_func = self.client.register_script(self._acquire_script) 471 | self._release_func = self.client.register_script(self._release_script) 472 | self._renew_func = self.client.register_script(self._renew_script) 473 | 474 | @property 475 | def _key_name(self): 476 | if self.namespace is not None: 477 | key = "%s_%s" % (self.namespace, self.lock_name) 478 | else: 479 | key = self.lock_name 480 | return key 481 | 482 | def _acquire(self): 483 | owner = str(uuid.uuid4()) 484 | if self.expire is None: 485 | expire = -1 486 | else: 487 | expire = self.expire 488 | if self._acquire_func(keys=[self._key_name], args=[owner, expire]) != 1: 489 | return False 490 | self._owner = owner 491 | return True 492 | 493 | def _release(self): 494 | if self._owner is None: 495 | raise LockException("Lock was not set by this process.") 496 | 497 | if self._release_func(keys=[self._key_name], args=[self._owner]) != 1: 498 | raise LockException( 499 | "Lock could not be released because it was " 500 | "not acquired by this instance." 501 | ) 502 | 503 | self._owner = None 504 | 505 | def _renew(self) -> bool: 506 | if self._owner is None: 507 | raise LockException("Lock was not set by this process.") 508 | 509 | if ( 510 | self._renew_func(keys=[self._key_name], args=[self._owner, self.expire]) 511 | != 1 512 | ): 513 | return False 514 | return True 515 | 516 | @property 517 | def _locked(self): 518 | if self.client.get(self._key_name) is None: 519 | return False 520 | return True 521 | 522 | 523 | class EtcdLock(BaseLock): 524 | """ 525 | Implementation of lock with Etcd as the backend for synchronization. 526 | 527 | Basic Usage: 528 | 529 | >>> import etcd 530 | >>> import sherlock 531 | >>> from sherlock import EtcdLock 532 | >>> 533 | >>> # Global configuration of defaults 534 | >>> sherlock.configure(expire=120, timeout=20) 535 | >>> 536 | >>> # Create a lock instance 537 | >>> lock = EtcdLock('my_lock') 538 | >>> 539 | >>> # Acquire a lock in Etcd, global backend and client configuration need 540 | >>> # not be configured since we are using a backend specific lock. 541 | >>> lock.acquire() 542 | True 543 | >>> 544 | >>> # Check if the lock has been acquired 545 | >>> lock.locked() 546 | True 547 | >>> 548 | >>> # Release the acquired lock 549 | >>> lock.release() 550 | >>> 551 | >>> # Check if the lock has been acquired 552 | >>> lock.locked() 553 | False 554 | >>> 555 | >>> # Use this client object 556 | >>> client = etcd.Client() 557 | >>> 558 | >>> # Create a lock instance with custom client object 559 | >>> lock = EtcdLock('my_lock', client=client) 560 | >>> 561 | >>> # To override the defaults, just past the configurations as parameters 562 | >>> lock = EtcdLock('my_lock', client=client, expire=1, timeout=5) 563 | >>> 564 | >>> # Acquire a lock using the with_statement 565 | >>> with EtcdLock('my_lock') as lock: 566 | ... # do some stuff with your acquired resource 567 | ... pass 568 | """ 569 | 570 | def __init__(self, lock_name, **kwargs): 571 | """ 572 | :param str lock_name: name of the lock to uniquely identify the lock 573 | between processes. 574 | :param str namespace: Optional namespace to namespace lock keys for 575 | your application in order to avoid conflicts. 576 | :param float expire: set lock expiry time. If explicitly set to `None`, 577 | lock will not expire. 578 | :param float timeout: set timeout to acquire lock 579 | :param float retry_interval: set interval for trying acquiring lock 580 | after the timeout interval has elapsed. 581 | :param client: supported client object for the backend of your choice. 582 | """ 583 | try: 584 | globals()["etcd"] = importlib.import_module("etcd") 585 | except ImportError as exc: 586 | raise ImportError("Please install `sherlock` with `etcd` extras.") from exc 587 | 588 | super(EtcdLock, self).__init__(lock_name, **kwargs) 589 | 590 | if self.client is None: 591 | self.client = etcd.Client() 592 | 593 | self._owner = None 594 | 595 | @property 596 | def _key_name(self): 597 | if self.namespace is not None: 598 | return "/%s/%s" % (self.namespace, self.lock_name) 599 | else: 600 | return "/%s" % self.lock_name 601 | 602 | def _acquire(self): 603 | owner = str(uuid.uuid4()) 604 | 605 | try: 606 | self.client.write(self._key_name, owner, prevExist=False, ttl=self.expire) 607 | self._owner = owner 608 | except etcd.EtcdAlreadyExist: 609 | return False 610 | else: 611 | return True 612 | 613 | def _release(self): 614 | if self._owner is None: 615 | raise LockException("Lock was not set by this process.") 616 | 617 | try: 618 | self.client.delete(self._key_name, prevValue=self._owner) 619 | self._owner = None 620 | except ValueError: 621 | raise LockException( 622 | "Lock could not be released because it " 623 | "was not acquired by this instance." 624 | ) 625 | except etcd.EtcdKeyNotFound: 626 | raise LockException( 627 | "Lock could not be released as it has not been acquired" 628 | ) 629 | 630 | def _renew(self) -> bool: 631 | if self._owner is None: 632 | raise LockException("Lock was not set by this process.") 633 | 634 | try: 635 | self.client.write( 636 | self._key_name, 637 | None, 638 | ttl=self.expire, 639 | prevValue=self._owner, 640 | prevExist=True, 641 | refresh=True, 642 | ) 643 | return True 644 | except (etcd.EtcdCompareFailed, etcd.EtcdKeyNotFound): 645 | return False 646 | 647 | @property 648 | def _locked(self): 649 | try: 650 | self.client.get(self._key_name) 651 | return True 652 | except etcd.EtcdKeyNotFound: 653 | return False 654 | 655 | 656 | class MCLock(BaseLock): 657 | """ 658 | Implementation of lock with Memcached as the backend for synchronization. 659 | 660 | Basic Usage: 661 | 662 | >>> import pylibmc 663 | >>> import sherlock 664 | >>> from sherlock import MCLock 665 | >>> 666 | >>> # Global configuration of defaults 667 | >>> sherlock.configure(expire=120, timeout=20) 668 | >>> 669 | >>> # Create a lock instance 670 | >>> lock = MCLock('my_lock') 671 | >>> 672 | >>> # Acquire a lock in Memcached, global backend and client configuration 673 | >>> # need not be configured since we are using a backend specific lock. 674 | >>> lock.acquire() 675 | True 676 | >>> 677 | >>> # Check if the lock has been acquired 678 | >>> lock.locked() 679 | True 680 | >>> 681 | >>> # Release the acquired lock 682 | >>> lock.release() 683 | >>> 684 | >>> # Check if the lock has been acquired 685 | >>> lock.locked() 686 | False 687 | >>> 688 | >>> # Use this client object 689 | >>> client = pylibmc.Client(['X.X.X.X'], binary=True) 690 | >>> 691 | >>> # Create a lock instance with custom client object 692 | >>> lock = MCLock('my_lock', client=client) 693 | >>> 694 | >>> # To override the defaults, just past the configurations as parameters 695 | >>> lock = MCLock('my_lock', client=client, expire=1, timeout=5) 696 | >>> 697 | >>> # Acquire a lock using the with_statement 698 | >>> with MCLock('my_lock') as lock: 699 | ... # do some stuff with your acquired resource 700 | ... pass 701 | """ 702 | 703 | def __init__(self, lock_name, **kwargs): 704 | """ 705 | :param str lock_name: name of the lock to uniquely identify the lock 706 | between processes. 707 | :param str namespace: Optional namespace to namespace lock keys for 708 | your application in order to avoid conflicts. 709 | :param float expire: set lock expiry time. If explicitly set to `None`, 710 | lock will not expire. 711 | :param float timeout: set timeout to acquire lock 712 | :param float retry_interval: set interval for trying acquiring lock 713 | after the timeout interval has elapsed. 714 | :param client: supported client object for the backend of your choice. 715 | """ 716 | try: 717 | import pylibmc 718 | except ImportError as exc: 719 | raise ImportError( 720 | "Please install `sherlock` with `memcached` extras." 721 | ) from exc 722 | 723 | super(MCLock, self).__init__(lock_name, **kwargs) 724 | 725 | if self.client is None: 726 | self.client = pylibmc.Client(["localhost"], binary=True) 727 | 728 | self._owner = None 729 | 730 | @property 731 | def _key_name(self): 732 | if self.namespace is not None: 733 | key = "%s_%s" % (self.namespace, self.lock_name) 734 | else: 735 | key = self.lock_name 736 | return key 737 | 738 | def _acquire(self): 739 | owner = str(uuid.uuid4()) 740 | 741 | _args = [self._key_name, str(owner)] 742 | if self.expire is not None: 743 | _args.append(self.expire) 744 | # Set key only if it does not exist 745 | if self.client.add(*_args) is True: 746 | self._owner = owner 747 | return True 748 | else: 749 | return False 750 | 751 | def _release(self): 752 | if self._owner is None: 753 | raise LockException("Lock was not set by this process.") 754 | 755 | resp = self.client.get(self._key_name) 756 | if resp is not None: 757 | if resp == str(self._owner): 758 | self.client.delete(self._key_name) 759 | self._owner = None 760 | else: 761 | raise LockException( 762 | "Lock could not be released because it " 763 | "was been acquired by this instance." 764 | ) 765 | else: 766 | raise LockException( 767 | "Lock could not be released as it has not " "been acquired" 768 | ) 769 | 770 | def _renew(self) -> bool: 771 | if self._owner is None: 772 | raise LockException("Lock was not set by this process.") 773 | 774 | resp = self.client.get(self._key_name) 775 | if resp is not None: 776 | if resp == str(self._owner): 777 | _args = [self._key_name, self._owner] 778 | if self.expire is not None: 779 | _args.append(self.expire) 780 | # Update key with new TTL 781 | self.client.set(*_args) 782 | return True 783 | return False 784 | 785 | @property 786 | def _locked(self): 787 | return True if self.client.get(self._key_name) is not None else False 788 | 789 | 790 | class KubernetesLock(BaseLock): 791 | """ 792 | Implementation of lock with Kubernetes resource as the backend for synchronization. 793 | 794 | Basic Usage: 795 | 796 | >>> import sherlock 797 | >>> from sherlock import KubernetesLock 798 | >>> 799 | >>> # Global configuration of defaults 800 | >>> sherlock.configure(expire=120, timeout=20) 801 | >>> 802 | >>> # Create a lock instance 803 | >>> lock = KubernetesLock('my_lock', 'my_namespace') 804 | >>> 805 | >>> # To acquire a lock in Kubernetes, global backend and client configuration need 806 | >>> # not be configured since we are using a backend specific lock. 807 | >>> lock.acquire() 808 | True 809 | >>> 810 | >>> # Check if the lock has been acquired 811 | >>> lock.locked() 812 | True 813 | >>> 814 | >>> # Release the acquired lock 815 | >>> lock.release() 816 | >>> 817 | >>> # Check if the lock has been acquired 818 | >>> lock.locked() 819 | False 820 | >>> 821 | >>> # To override the defaults, just past the configurations as parameters 822 | >>> lock = KubernetesLock( 823 | ... 'my_lock', 'my_namespace', expire=1, timeout=5, 824 | ... ) 825 | >>> 826 | >>> # Acquire a lock using the with_statement 827 | >>> with KubernetesLock('my_lock', 'my_namespace') as lock: 828 | ... # do some stuff with your acquired resource 829 | ... pass 830 | """ 831 | 832 | def __init__( 833 | self, 834 | lock_name: str, 835 | k8s_namespace: str, 836 | **kwargs, 837 | ) -> None: 838 | """ 839 | :param str lock_name: name of the lock to uniquely identify the lock 840 | between processes. 841 | :param str k8s_namespace: Kubernetes namespace to store the lock in. 842 | :param str namespace: Namespace to namespace lock keys for 843 | your application in order to avoid conflicts. 844 | :param float expire: set lock expiry time. If explicitly set to `None`, 845 | lock will not expire. 846 | :param float timeout: set timeout to acquire lock 847 | :param float retry_interval: set interval for trying acquiring lock 848 | after the timeout interval has elapsed. 849 | :param client: supported client object for the backend of your choice. 850 | """ 851 | try: 852 | globals()["kubernetes"] = importlib.import_module("kubernetes") 853 | except ImportError as exc: 854 | raise ImportError( 855 | "Please install `sherlock` with `kubernetes` extras." 856 | ) from exc 857 | 858 | super().__init__(lock_name, **kwargs) 859 | 860 | self.k8s_namespace = k8s_namespace 861 | 862 | # Verify that all names are compatible with Kubernetes. 863 | rfc_1123_dns_label = re.compile("^(?![0-9]+$)(?!-)[a-zA-Z0-9-]{,63}(? datetime.datetime: 888 | expiry_time = datetime.datetime.min 889 | if ( 890 | lease.spec.renew_time is not None 891 | and lease.spec.lease_duration_seconds is not None 892 | ): 893 | expiry_time = lease.spec.renew_time + datetime.timedelta( 894 | seconds=lease.spec.lease_duration_seconds 895 | ) 896 | elif lease.spec.lease_duration_seconds is None: 897 | expiry_time = datetime.datetime.max 898 | return expiry_time.astimezone(tz=datetime.timezone.utc) 899 | 900 | def _has_expired( 901 | self, lease: kubernetes.client.V1Lease, now: datetime.datetime 902 | ) -> bool: 903 | # Determine whether the Lease has expired. 904 | return now > self._expiry_time(lease) 905 | 906 | def _create_lease( 907 | self, 908 | owner: str, 909 | ) -> typing.Optional[kubernetes.client.V1Lease]: 910 | now = self._now() 911 | try: 912 | return self.client.create_namespaced_lease( 913 | namespace=self.k8s_namespace, 914 | body=kubernetes.client.V1Lease( 915 | metadata=kubernetes.client.V1ObjectMeta( 916 | name=self._key_name, 917 | ), 918 | spec=kubernetes.client.V1LeaseSpec( 919 | holder_identity=owner, 920 | acquire_time=now, 921 | renew_time=now, 922 | lease_duration_seconds=self.expire, 923 | ), 924 | ), 925 | ) 926 | except kubernetes.client.exceptions.ApiException as exc: 927 | if exc.reason == "Conflict": 928 | # Someone has created the Lease before we did. 929 | return None 930 | raise LockException("Failed to create Lock.") from exc 931 | 932 | def _get_lease(self) -> typing.Optional[kubernetes.client.V1Lease]: 933 | try: 934 | return self.client.read_namespaced_lease( 935 | name=self._key_name, 936 | namespace=self.k8s_namespace, 937 | ) 938 | except kubernetes.client.exceptions.ApiException as exc: 939 | if exc.reason == "Not Found": 940 | # Lease did not exist. 941 | return None 942 | raise LockException("Failed to read Lock.") from exc 943 | 944 | def _replace_lease( 945 | self, 946 | lease: kubernetes.client.V1Lease, 947 | ) -> typing.Optional[kubernetes.client.V1Lease]: 948 | try: 949 | return self.client.replace_namespaced_lease( 950 | name=self._key_name, 951 | namespace=self.k8s_namespace, 952 | body=lease, 953 | ) 954 | except kubernetes.client.exceptions.ApiException as exc: 955 | if exc.reason == "Conflict": 956 | return None 957 | raise LockException("Failed to update Lock.") from exc 958 | 959 | def _delete_lease(self, lease: kubernetes.client.V1Lease) -> None: 960 | try: 961 | self.client.delete_namespaced_lease( 962 | name=self._key_name, 963 | namespace=self.k8s_namespace, 964 | body=kubernetes.client.V1DeleteOptions( 965 | preconditions=kubernetes.client.V1Preconditions( 966 | resource_version=lease.metadata.resource_version 967 | ) 968 | ), 969 | ) 970 | except kubernetes.client.exceptions.ApiException as exc: 971 | if exc.reason == "Not Found": 972 | # The Lease has already been acquired by another 973 | # instance which is fine. 974 | return None 975 | raise LockException("Failed to release Lock.") from exc 976 | 977 | def _now(self) -> datetime.datetime: 978 | return datetime.datetime.now(tz=datetime.timezone.utc) 979 | 980 | def _acquire(self) -> bool: 981 | owner = str(uuid.uuid4()) 982 | 983 | # The Lease object contains a `.metadata.resource_version` which 984 | # protects us from race conditions in updating the Lease as described: 985 | # https://blog.atomist.com/kubernetes-apply-replace-patch/. 986 | lease = self._get_lease() 987 | if lease is None: 988 | # Lease does not exist so let's create it in our name. 989 | if self._create_lease(owner) is not None: 990 | # We created the Lease and so hold the Lock. 991 | self._owner = owner 992 | return True 993 | else: 994 | # We failed to create the Lease before someone else. 995 | return False 996 | 997 | now = self._now() 998 | has_expired = self._has_expired(lease, now) 999 | if owner != lease.spec.holder_identity: 1000 | if not has_expired: 1001 | # Someone else holds the lock. 1002 | return False 1003 | 1004 | else: 1005 | # Lock is available for us to take. 1006 | lease.spec.holder_identity = owner 1007 | lease.spec.acquire_time = now 1008 | lease.spec.renew_time = now 1009 | lease.spec.lease_duration_seconds = self.expire 1010 | 1011 | else: 1012 | # Same owner so do not set or modify Lease. 1013 | return False 1014 | 1015 | # The Lease object contains a `.metadata.resource_version` which 1016 | # protects us from race conditions in updating the Lease as described: 1017 | # https://blog.atomist.com/kubernetes-apply-replace-patch/. 1018 | if self._replace_lease(lease) is None: 1019 | # Someone else has modified the Lease so we can't acquire the Lock 1020 | # safely. 1021 | return False 1022 | # We succeeded in replacing the Lease so we now hold the lock. 1023 | self._owner = owner 1024 | return True 1025 | 1026 | def _release(self) -> None: 1027 | if self._owner is None: 1028 | raise LockException("Lock was not set by this process.") 1029 | 1030 | lease = self._get_lease() 1031 | if lease is not None and self._owner == lease.spec.holder_identity: 1032 | # The Lease object contains a `.metadata.resource_version` which 1033 | # protects us from race conditions in deleting the Lease. 1034 | self._delete_lease(lease) 1035 | 1036 | def _renew(self) -> bool: 1037 | if self._owner is None: 1038 | raise LockException("Lock was not set by this process.") 1039 | 1040 | lease = self._get_lease() 1041 | if lease is not None and self._owner == lease.spec.holder_identity: 1042 | now = self._now() 1043 | has_expired = self._has_expired(lease, now) 1044 | 1045 | if has_expired: 1046 | return False 1047 | 1048 | lease.spec.acquire_time = now 1049 | lease.spec.renew_time = now 1050 | lease.spec.lease_duration_seconds = self.expire 1051 | # The Lease object contains a `.metadata.resource_version` which 1052 | # protects us from race conditions in updating the Lease as described: 1053 | # https://blog.atomist.com/kubernetes-apply-replace-patch/. 1054 | if self._replace_lease(lease) is None: 1055 | # Someone else has modified the Lease before we renewed it so it 1056 | # must've expired. 1057 | raise False 1058 | return True 1059 | 1060 | return False 1061 | 1062 | @property 1063 | def _locked(self): 1064 | lease = self._get_lease() 1065 | 1066 | if lease is None: 1067 | # Lease doesn't exist so can't be locked. 1068 | return False 1069 | 1070 | if self._has_expired(lease, self._now()): 1071 | # Lease exists but has expired. 1072 | return False 1073 | # Lease exists and has not expired. 1074 | return True 1075 | 1076 | 1077 | class FileLock(BaseLock): 1078 | """ 1079 | Implementation of lock with the file system as the backend for synchronization. 1080 | 1081 | Basic Usage: 1082 | 1083 | >>> import sherlock 1084 | >>> from sherlock import FileLock 1085 | >>> 1086 | >>> # Global configuration of defaults 1087 | >>> sherlock.configure(expire=120, timeout=20) 1088 | >>> 1089 | >>> # Create a lock instance 1090 | >>> lock = FileLock('my_lock') 1091 | >>> 1092 | >>> # To acquire a lock global backend and client configuration need 1093 | >>> # not be configured since we are using a backend specific lock. 1094 | >>> lock.acquire() 1095 | True 1096 | >>> 1097 | >>> # Check if the lock has been acquired 1098 | >>> lock.locked() 1099 | True 1100 | >>> 1101 | >>> # Release the acquired lock 1102 | >>> lock.release() 1103 | >>> 1104 | >>> # Check if the lock has been acquired 1105 | >>> lock.locked() 1106 | False 1107 | >>> 1108 | >>> # To override the defaults, just past the configurations as parameters 1109 | >>> lock = FileLock( 1110 | ... 'my_lock', expire=1, timeout=5, 1111 | ... ) 1112 | >>> 1113 | >>> # Acquire a lock using the with_statement 1114 | >>> with FileLock('my_lock') as lock: 1115 | ... # do some stuff with your acquired resource 1116 | ... pass 1117 | """ 1118 | 1119 | def __init__(self, lock_name: str, **kwargs) -> None: 1120 | """ 1121 | :param str lock_name: name of the lock to uniquely identify the lock 1122 | between processes. 1123 | :param str namespace: Namespace to namespace lock keys for 1124 | your application in order to avoid conflicts. 1125 | :param float expire: set lock expiry time. If explicitly set to `None`, 1126 | lock will not expire. 1127 | :param float timeout: set timeout to acquire lock 1128 | :param float retry_interval: set interval for trying acquiring lock 1129 | after the timeout interval has elapsed. 1130 | :param client: supported client object for the backend of your choice. 1131 | """ 1132 | try: 1133 | globals()["filelock"] = importlib.import_module("filelock") 1134 | except ImportError as exc: 1135 | raise ImportError( 1136 | "Please install `sherlock` with `filelock` extras." 1137 | ) from exc 1138 | 1139 | super().__init__(lock_name, **kwargs) 1140 | 1141 | if self.client is None: 1142 | self.client = pathlib.Path("/tmp/sherlock") 1143 | self.client.mkdir(parents=True, exist_ok=True) 1144 | 1145 | self._owner: typing.Optional[str] = None 1146 | self._data_file = (self.client / self._key_name).with_suffix(".json") 1147 | self._lock_file = filelock.FileLock( 1148 | (self.client / self._key_name).with_suffix(".lock") 1149 | ) 1150 | 1151 | @property 1152 | def _key_name(self): 1153 | if self.namespace is not None: 1154 | key = "%s_%s" % (self.namespace, self.lock_name) 1155 | else: 1156 | key = self.lock_name 1157 | return key 1158 | 1159 | def _now(self) -> datetime.datetime: 1160 | return datetime.datetime.now(tz=datetime.timezone.utc) 1161 | 1162 | def _expiry_time(self) -> str: 1163 | expiry_time = datetime.datetime.max.astimezone(datetime.timezone.utc) 1164 | if self.expire is not None: 1165 | expiry_time = self._now() + datetime.timedelta(seconds=self.expire) 1166 | return expiry_time.isoformat() 1167 | 1168 | def _has_expired(self, data: dict, now: datetime.datetime) -> bool: 1169 | expiry_time = datetime.datetime.fromisoformat(data["expiry_time"]) 1170 | return now > expiry_time.astimezone(tz=datetime.timezone.utc) 1171 | 1172 | def _acquire(self) -> bool: 1173 | owner = str(uuid.uuid4()) 1174 | 1175 | # Make sure we have unique lock on the file. 1176 | with self._lock_file: 1177 | if self._data_file.exists(): 1178 | data = json.loads(self._data_file.read_text()) 1179 | 1180 | now = self._now() 1181 | has_expired = self._has_expired(data, now) 1182 | if owner != data["owner"]: 1183 | if not has_expired: 1184 | # Someone else holds the lock. 1185 | return False 1186 | 1187 | else: 1188 | # Lock is available for us to take. 1189 | data = {"owner": owner, "expiry_time": self._expiry_time()} 1190 | 1191 | else: 1192 | # Same owner so do not set or modify Lease. 1193 | return False 1194 | else: 1195 | data = {"owner": owner, "expiry_time": self._expiry_time()} 1196 | 1197 | # Write new data back to file. 1198 | self._data_file.touch() 1199 | self._data_file.write_text(json.dumps(data)) 1200 | 1201 | # We succeeded in writing to the file so we now hold the lock. 1202 | self._owner = owner 1203 | return True 1204 | 1205 | def _release(self) -> None: 1206 | if self._owner is None: 1207 | raise LockException("Lock was not set by this process.") 1208 | 1209 | if self._data_file.exists(): 1210 | with self._lock_file: 1211 | data = json.loads(self._data_file.read_text()) 1212 | 1213 | if self._owner == data["owner"]: 1214 | self._data_file.unlink() 1215 | 1216 | def _renew(self) -> bool: 1217 | if self._owner is None: 1218 | raise LockException("Lock was not set by this process.") 1219 | 1220 | if self._data_file.exists(): 1221 | with self._lock_file: 1222 | data = json.loads(self._data_file.read_text()) 1223 | 1224 | now = self._now() 1225 | has_expired = self._has_expired(data, now) 1226 | if self._owner == data["owner"]: 1227 | if has_expired: 1228 | return False 1229 | # Refresh expiry time. 1230 | data["expiry_time"] = self._expiry_time() 1231 | self._data_file.write_text(json.dumps(data)) 1232 | return True 1233 | return False 1234 | 1235 | @property 1236 | def _locked(self): 1237 | if not self._data_file.exists(): 1238 | # File doesn't exist so can't be locked. 1239 | return False 1240 | 1241 | with self._lock_file: 1242 | data = json.loads(self._data_file.read_text()) 1243 | 1244 | if self._has_expired(data, self._now()): 1245 | # File exists but has expired. 1246 | return False 1247 | 1248 | # Lease exists and has not expired. 1249 | return True 1250 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/py-sherlock/sherlock/245ece9adfa843cf8486e401d74391b3f753d950/tests/__init__.py -------------------------------------------------------------------------------- /tests/integration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/py-sherlock/sherlock/245ece9adfa843cf8486e401d74391b3f753d950/tests/integration/__init__.py -------------------------------------------------------------------------------- /tests/integration/test_lock.py: -------------------------------------------------------------------------------- 1 | """ 2 | Integration tests for backend locks. 3 | """ 4 | 5 | import datetime 6 | import json 7 | import os 8 | import pathlib 9 | import time 10 | import unittest 11 | from unittest.mock import patch 12 | 13 | import etcd 14 | import kubernetes.client 15 | import kubernetes.client.exceptions 16 | import kubernetes.config 17 | import pylibmc 18 | import redis 19 | 20 | import sherlock 21 | 22 | 23 | class TestRedisLock(unittest.TestCase): 24 | def setUp(self): 25 | try: 26 | self.client = redis.StrictRedis( 27 | host=os.getenv("REDIS_HOST", "redis"), 28 | ) 29 | except Exception as err: 30 | print(str(err)) 31 | raise Exception( 32 | "You must have Redis server running on localhost " 33 | "to be able to run integration tests." 34 | ) 35 | self.lock_name = "test_lock" 36 | 37 | def test_acquire(self): 38 | lock = sherlock.RedisLock(self.lock_name, client=self.client) 39 | self.assertTrue(lock._acquire()) 40 | self.assertEqual( 41 | self.client.get(self.lock_name).decode("UTF-8"), str(lock._owner) 42 | ) 43 | 44 | def test_acquire_with_namespace(self): 45 | lock = sherlock.RedisLock(self.lock_name, client=self.client, namespace="ns") 46 | self.assertTrue(lock._acquire()) 47 | self.assertEqual( 48 | self.client.get("ns_%s" % self.lock_name).decode("UTF-8"), str(lock._owner) 49 | ) 50 | 51 | def test_acquire_once_only(self): 52 | lock1 = sherlock.RedisLock(self.lock_name, client=self.client) 53 | lock2 = sherlock.RedisLock(self.lock_name, client=self.client) 54 | self.assertTrue(lock1._acquire()) 55 | self.assertFalse(lock2._acquire()) 56 | 57 | def test_acquire_check_expiry(self): 58 | lock = sherlock.RedisLock(self.lock_name, client=self.client, expire=1) 59 | lock.acquire() 60 | time.sleep(2) 61 | self.assertFalse(lock.locked()) 62 | 63 | def test_acquire_check_expire_is_not_set(self): 64 | lock = sherlock.RedisLock(self.lock_name, client=self.client, expire=None) 65 | lock.acquire() 66 | time.sleep(2) 67 | self.assertTrue(self.client.ttl(self.lock_name) < 0) 68 | 69 | def test_release(self): 70 | lock = sherlock.RedisLock(self.lock_name, client=self.client) 71 | lock._acquire() 72 | lock._release() 73 | self.assertEqual(self.client.get(self.lock_name), None) 74 | 75 | def test_release_with_namespace(self): 76 | lock = sherlock.RedisLock(self.lock_name, client=self.client, namespace="ns") 77 | lock._acquire() 78 | lock._release() 79 | self.assertEqual(self.client.get("ns_%s" % self.lock_name), None) 80 | 81 | def test_release_own_only(self): 82 | lock1 = sherlock.RedisLock(self.lock_name, client=self.client) 83 | lock2 = sherlock.RedisLock(self.lock_name, client=self.client) 84 | lock1._acquire() 85 | self.assertRaises(sherlock.LockException, lock2._release) 86 | lock1._release() 87 | 88 | def test_locked(self): 89 | lock = sherlock.RedisLock(self.lock_name, client=self.client) 90 | lock._acquire() 91 | self.assertTrue(lock._locked) 92 | lock._release() 93 | self.assertFalse(lock._locked) 94 | 95 | def test_deleting_lock_object_releases_the_lock(self): 96 | lock = sherlock.lock.RedisLock(self.lock_name, client=self.client) 97 | lock.acquire() 98 | self.assertEqual( 99 | self.client.get(self.lock_name).decode("UTF-8"), str(lock._owner) 100 | ) 101 | 102 | del lock 103 | self.assertEqual(self.client.get(self.lock_name), None) 104 | 105 | def test_renew(self): 106 | lock = sherlock.lock.RedisLock(self.lock_name, client=self.client, expire=3600) 107 | self.assertTrue(lock.acquire()) 108 | lock.renew() 109 | 110 | def test_renew_expired(self): 111 | lock = sherlock.RedisLock(self.lock_name, client=self.client, expire=1) 112 | self.assertTrue(lock.acquire()) 113 | time.sleep(2) 114 | 115 | self.assertFalse(lock.renew()) 116 | 117 | def test_renew_owner_collision(self): 118 | lock_1 = sherlock.RedisLock( 119 | self.lock_name, 120 | client=self.client, 121 | expire=1, 122 | ) 123 | lock_2 = sherlock.RedisLock( 124 | self.lock_name, 125 | client=self.client, 126 | ) 127 | self.assertTrue(lock_1.acquire()) 128 | # Wait for Lock to expire 129 | time.sleep(1) 130 | self.assertTrue(lock_2.acquire()) 131 | 132 | self.assertFalse(lock_1.renew()) 133 | 134 | def test_renew_before_acquire(self): 135 | lock = sherlock.RedisLock(self.lock_name, client=self.client, expire=1) 136 | with self.assertRaisesRegex( 137 | sherlock.LockException, "Lock was not set by this process." 138 | ): 139 | lock.renew() 140 | 141 | def tearDown(self): 142 | self.client.delete(self.lock_name) 143 | self.client.delete("ns_%s" % self.lock_name) 144 | 145 | 146 | class TestEtcdLock(unittest.TestCase): 147 | def setUp(self): 148 | self.client = etcd.Client(host=os.getenv("ETCD_HOST", "etcd")) 149 | self.lock_name = "test_lock" 150 | 151 | def test_acquire(self): 152 | lock = sherlock.EtcdLock(self.lock_name, client=self.client) 153 | self.assertTrue(lock._acquire()) 154 | self.assertEqual(self.client.get(self.lock_name).value, str(lock._owner)) 155 | 156 | def test_acquire_with_namespace(self): 157 | lock = sherlock.EtcdLock(self.lock_name, client=self.client, namespace="ns") 158 | self.assertTrue(lock._acquire()) 159 | self.assertEqual( 160 | self.client.get("/ns/%s" % self.lock_name).value, str(lock._owner) 161 | ) 162 | 163 | def test_acquire_once_only(self): 164 | lock1 = sherlock.EtcdLock(self.lock_name, client=self.client) 165 | lock2 = sherlock.EtcdLock(self.lock_name, client=self.client) 166 | self.assertTrue(lock1._acquire()) 167 | self.assertFalse(lock2._acquire()) 168 | 169 | def test_acquire_check_expiry(self): 170 | lock = sherlock.EtcdLock(self.lock_name, client=self.client, expire=1) 171 | lock.acquire() 172 | time.sleep(2) 173 | self.assertFalse(lock.locked()) 174 | 175 | def test_acquire_check_expire_is_not_set(self): 176 | lock = sherlock.EtcdLock(self.lock_name, client=self.client, expire=None) 177 | lock.acquire() 178 | time.sleep(2) 179 | self.assertEquals(self.client.get(self.lock_name).ttl, None) 180 | 181 | def test_release(self): 182 | lock = sherlock.EtcdLock(self.lock_name, client=self.client) 183 | lock._acquire() 184 | lock._release() 185 | self.assertRaises(etcd.EtcdKeyNotFound, self.client.get, self.lock_name) 186 | 187 | def test_release_with_namespace(self): 188 | lock = sherlock.EtcdLock(self.lock_name, client=self.client, namespace="ns") 189 | lock._acquire() 190 | lock._release() 191 | self.assertRaises( 192 | etcd.EtcdKeyNotFound, self.client.get, "/ns/%s" % self.lock_name 193 | ) 194 | 195 | def test_release_own_only(self): 196 | lock1 = sherlock.EtcdLock(self.lock_name, client=self.client) 197 | lock2 = sherlock.EtcdLock(self.lock_name, client=self.client) 198 | lock1._acquire() 199 | self.assertRaises(sherlock.LockException, lock2._release) 200 | lock1._release() 201 | 202 | def test_locked(self): 203 | lock = sherlock.EtcdLock(self.lock_name, client=self.client) 204 | lock._acquire() 205 | self.assertTrue(lock._locked) 206 | lock._release() 207 | self.assertFalse(lock._locked) 208 | 209 | def test_deleting_lock_object_releases_the_lock(self): 210 | lock = sherlock.lock.EtcdLock(self.lock_name, client=self.client) 211 | lock.acquire() 212 | self.assertEqual(self.client.get(self.lock_name).value, str(lock._owner)) 213 | 214 | del lock 215 | self.assertRaises(etcd.EtcdKeyNotFound, self.client.get, self.lock_name) 216 | 217 | def test_renew(self): 218 | lock = sherlock.lock.EtcdLock(self.lock_name, client=self.client, expire=3600) 219 | self.assertTrue(lock.acquire()) 220 | lock.renew() 221 | 222 | def test_renew_expired(self): 223 | lock = sherlock.EtcdLock(self.lock_name, client=self.client, expire=1) 224 | self.assertTrue(lock.acquire()) 225 | time.sleep(2) 226 | 227 | self.assertFalse(lock.renew()) 228 | 229 | def test_renew_owner_collision(self): 230 | lock_1 = sherlock.EtcdLock( 231 | self.lock_name, 232 | client=self.client, 233 | expire=1, 234 | ) 235 | lock_2 = sherlock.EtcdLock( 236 | self.lock_name, 237 | client=self.client, 238 | ) 239 | self.assertTrue(lock_1.acquire()) 240 | # Wait for Lock to expire 241 | time.sleep(1) 242 | self.assertTrue(lock_2.acquire()) 243 | 244 | self.assertFalse(lock_1.renew()) 245 | 246 | def test_renew_before_acquire(self): 247 | lock = sherlock.EtcdLock(self.lock_name, client=self.client, expire=1) 248 | with self.assertRaisesRegex( 249 | sherlock.LockException, "Lock was not set by this process." 250 | ): 251 | lock.renew() 252 | 253 | def tearDown(self): 254 | try: 255 | self.client.delete(self.lock_name) 256 | except etcd.EtcdKeyNotFound: 257 | pass 258 | try: 259 | self.client.delete("/ns/%s" % self.lock_name) 260 | except etcd.EtcdKeyNotFound: 261 | pass 262 | 263 | 264 | class TestMCLock(unittest.TestCase): 265 | def setUp(self): 266 | self.client = pylibmc.Client( 267 | [os.getenv("MEMCACHED_HOST", "memcached")], 268 | binary=True, 269 | ) 270 | self.lock_name = "test_lock" 271 | 272 | def test_acquire(self): 273 | lock = sherlock.MCLock(self.lock_name, client=self.client) 274 | self.assertTrue(lock._acquire()) 275 | self.assertEqual(self.client.get(self.lock_name), str(lock._owner)) 276 | 277 | def test_acquire_with_namespace(self): 278 | lock = sherlock.MCLock(self.lock_name, client=self.client, namespace="ns") 279 | self.assertTrue(lock._acquire()) 280 | self.assertEqual(self.client.get("ns_%s" % self.lock_name), str(lock._owner)) 281 | 282 | def test_acquire_once_only(self): 283 | lock1 = sherlock.MCLock(self.lock_name, client=self.client) 284 | lock2 = sherlock.MCLock(self.lock_name, client=self.client) 285 | self.assertTrue(lock1._acquire()) 286 | self.assertFalse(lock2._acquire()) 287 | 288 | def test_acquire_check_expiry(self): 289 | lock = sherlock.MCLock(self.lock_name, client=self.client, expire=1) 290 | lock.acquire() 291 | time.sleep(2) 292 | self.assertFalse(lock.locked()) 293 | 294 | def test_release(self): 295 | lock = sherlock.MCLock(self.lock_name, client=self.client) 296 | lock._acquire() 297 | lock._release() 298 | self.assertEqual(self.client.get(self.lock_name), None) 299 | 300 | def test_release_with_namespace(self): 301 | lock = sherlock.MCLock(self.lock_name, client=self.client, namespace="ns") 302 | lock._acquire() 303 | lock._release() 304 | self.assertEqual(self.client.get("ns_%s" % self.lock_name), None) 305 | 306 | def test_release_own_only(self): 307 | lock1 = sherlock.MCLock(self.lock_name, client=self.client) 308 | lock2 = sherlock.MCLock(self.lock_name, client=self.client) 309 | lock1._acquire() 310 | self.assertRaises(sherlock.LockException, lock2._release) 311 | lock1._release() 312 | 313 | def test_locked(self): 314 | lock = sherlock.MCLock(self.lock_name, client=self.client) 315 | lock._acquire() 316 | self.assertTrue(lock._locked) 317 | lock._release() 318 | self.assertFalse(lock._locked) 319 | 320 | def test_deleting_lock_object_releases_the_lock(self): 321 | lock = sherlock.lock.MCLock(self.lock_name, client=self.client) 322 | lock.acquire() 323 | self.assertEqual(self.client.get(self.lock_name), str(lock._owner)) 324 | 325 | del lock 326 | self.assertEqual(self.client.get(self.lock_name), None) 327 | 328 | def test_renew(self): 329 | lock = sherlock.lock.MCLock(self.lock_name, client=self.client, expire=3600) 330 | self.assertTrue(lock.acquire()) 331 | lock.renew() 332 | 333 | def test_renew_expired(self): 334 | lock = sherlock.lock.MCLock(self.lock_name, client=self.client, expire=1) 335 | self.assertTrue(lock.acquire()) 336 | time.sleep(1) 337 | self.assertFalse(lock.renew()) 338 | 339 | def test_renew_owner_collision(self): 340 | lock_1 = sherlock.lock.MCLock( 341 | self.lock_name, 342 | client=self.client, 343 | expire=1, 344 | ) 345 | lock_2 = sherlock.lock.MCLock( 346 | self.lock_name, 347 | client=self.client, 348 | ) 349 | self.assertTrue(lock_1.acquire()) 350 | # Wait for Lock to expire 351 | time.sleep(1) 352 | self.assertTrue(lock_2.acquire()) 353 | 354 | self.assertFalse(lock_1.renew()) 355 | 356 | def test_renew_before_acquire(self): 357 | lock = sherlock.lock.MCLock(self.lock_name, client=self.client, expire=1) 358 | with self.assertRaisesRegex( 359 | sherlock.LockException, "Lock was not set by this process." 360 | ): 361 | lock.renew() 362 | 363 | def tearDown(self): 364 | self.client.delete(self.lock_name) 365 | self.client.delete("ns_%s" % self.lock_name) 366 | 367 | 368 | class TestKubernetesLock(unittest.TestCase): 369 | def setUp(self): 370 | kubernetes.config.load_kube_config( 371 | config_file=os.environ["KUBECONFIG"], 372 | ) 373 | kubernetes.config.load_config 374 | self.client = kubernetes.client.CoordinationV1Api() 375 | self.lock_name = "test-lock" 376 | self.k8s_namespace = "default" 377 | 378 | def test_acquire(self): 379 | lock = sherlock.KubernetesLock(self.lock_name, self.k8s_namespace) 380 | self.assertTrue(lock._acquire()) 381 | lease = self.client.read_namespaced_lease( 382 | name=self.lock_name, 383 | namespace=self.k8s_namespace, 384 | ) 385 | self.assertEqual(lease.spec.holder_identity, str(lock._owner)) 386 | 387 | def test_acquire_with_namespace(self): 388 | lock = sherlock.KubernetesLock( 389 | self.lock_name, 390 | self.k8s_namespace, 391 | namespace="ns", 392 | ) 393 | self.assertTrue(lock._acquire()) 394 | lease = self.client.read_namespaced_lease( 395 | name=f"ns-{self.lock_name}", 396 | namespace=self.k8s_namespace, 397 | ) 398 | self.assertEqual(lease.spec.holder_identity, str(lock._owner)) 399 | 400 | def test_acquire_once_only(self): 401 | lock1 = sherlock.KubernetesLock(self.lock_name, self.k8s_namespace) 402 | lock2 = sherlock.KubernetesLock(self.lock_name, self.k8s_namespace) 403 | self.assertTrue(lock1._acquire()) 404 | self.assertFalse(lock2._acquire()) 405 | 406 | def test_acquire_check_expiry(self): 407 | lock = sherlock.KubernetesLock( 408 | self.lock_name, 409 | self.k8s_namespace, 410 | expire=1, 411 | ) 412 | lock.acquire() 413 | time.sleep(2) 414 | self.assertFalse(lock.locked()) 415 | 416 | def test_acquire_expired_lock(self): 417 | lock = sherlock.KubernetesLock( 418 | self.lock_name, 419 | self.k8s_namespace, 420 | expire=1, 421 | ) 422 | lock.acquire() 423 | lease_1 = self.client.read_namespaced_lease( 424 | name=self.lock_name, 425 | namespace=self.k8s_namespace, 426 | ) 427 | time.sleep(2) 428 | 429 | # We can acquire the Lock again after it 430 | # expires. 431 | self.assertTrue(lock.acquire()) 432 | 433 | lease_2 = self.client.read_namespaced_lease( 434 | name=self.lock_name, 435 | namespace=self.k8s_namespace, 436 | ) 437 | 438 | # New Lease has new owner and new owner is not the same 439 | # as old owner. 440 | self.assertEqual(lease_2.spec.holder_identity, str(lock._owner)) 441 | self.assertNotEqual( 442 | lease_1.spec.holder_identity, 443 | lease_2.spec.holder_identity, 444 | ) 445 | 446 | @patch("sherlock.lock.uuid.uuid4") 447 | def test_acquire_owner_collison(self, mock_uuid4): 448 | mock_uuid4.return_value = b"fake-uuid" 449 | 450 | lock = sherlock.KubernetesLock( 451 | self.lock_name, 452 | self.k8s_namespace, 453 | expire=None, 454 | ) 455 | self.assertTrue(lock.acquire()) 456 | self.assertFalse(lock.acquire(blocking=False)) 457 | 458 | def test_acquire_check_expire_is_not_set(self): 459 | lock = sherlock.KubernetesLock( 460 | self.lock_name, 461 | self.k8s_namespace, 462 | expire=None, 463 | ) 464 | lock.acquire() 465 | time.sleep(2) 466 | lease = self.client.read_namespaced_lease( 467 | name=self.lock_name, 468 | namespace=self.k8s_namespace, 469 | ) 470 | self.assertIsNone(lease.spec.lease_duration_seconds) 471 | self.assertTrue(lock.locked()) 472 | 473 | def test_release(self): 474 | lock = sherlock.KubernetesLock(self.lock_name, self.k8s_namespace) 475 | lock._acquire() 476 | lock._release() 477 | with self.assertRaises(kubernetes.client.exceptions.ApiException) as cm: 478 | self.client.read_namespaced_lease( 479 | name=self.lock_name, 480 | namespace=self.k8s_namespace, 481 | ) 482 | self.assertEqual(cm.exception.reason, "Not Found") 483 | 484 | def test_release_with_namespace(self): 485 | lock = sherlock.KubernetesLock( 486 | self.lock_name, 487 | self.k8s_namespace, 488 | namespace="ns", 489 | ) 490 | lock._acquire() 491 | lock._release() 492 | # self.assertEqual(self.client.get('ns_%s' % self.lock_name), None) 493 | 494 | def test_release_own_only(self): 495 | lock1 = sherlock.KubernetesLock(self.lock_name, self.k8s_namespace) 496 | lock2 = sherlock.KubernetesLock(self.lock_name, self.k8s_namespace) 497 | lock1._acquire() 498 | self.assertRaises(sherlock.LockException, lock2._release) 499 | lock1._release() 500 | 501 | def test_locked(self): 502 | lock = sherlock.KubernetesLock(self.lock_name, self.k8s_namespace) 503 | lock._acquire() 504 | self.assertTrue(lock._locked) 505 | lock._release() 506 | self.assertFalse(lock._locked) 507 | 508 | def test_deleting_lock_object_releases_the_lock(self): 509 | lock = sherlock.KubernetesLock( 510 | self.lock_name, 511 | self.k8s_namespace, 512 | ) 513 | lock.acquire() 514 | lease = self.client.read_namespaced_lease( 515 | name=self.lock_name, 516 | namespace=self.k8s_namespace, 517 | ) 518 | self.assertEqual(lease.spec.holder_identity, str(lock._owner)) 519 | 520 | del lock 521 | with self.assertRaises(kubernetes.client.exceptions.ApiException) as cm: 522 | self.client.read_namespaced_lease( 523 | name=self.lock_name, 524 | namespace=self.k8s_namespace, 525 | ) 526 | self.assertEqual(cm.exception.reason, "Not Found") 527 | 528 | def test_release_lock_that_no_longer_exists(self): 529 | lock_1 = sherlock.KubernetesLock( 530 | self.lock_name, 531 | self.k8s_namespace, 532 | expire=1, 533 | ) 534 | lock_2 = sherlock.KubernetesLock( 535 | self.lock_name, 536 | self.k8s_namespace, 537 | ) 538 | self.assertTrue(lock_1.acquire()) 539 | # Wait for Lock to expire 540 | time.sleep(2) 541 | self.assertTrue(lock_2.acquire()) 542 | lock_2.release() 543 | 544 | # Releasing a Lock has been removed should be fine. 545 | self.assertIsNone(lock_1.release()) 546 | 547 | def test_renew(self): 548 | lock = sherlock.KubernetesLock(self.lock_name, self.k8s_namespace, expire=3600) 549 | self.assertTrue(lock.acquire()) 550 | first_expiry_time = lock._expiry_time(lock._get_lease()) 551 | lock.renew() 552 | second_expiry_time = lock._expiry_time(lock._get_lease()) 553 | self.assertGreater(second_expiry_time, first_expiry_time) 554 | 555 | def test_renew_expired(self): 556 | lock = sherlock.KubernetesLock(self.lock_name, self.k8s_namespace, expire=1) 557 | self.assertTrue(lock.acquire()) 558 | first_expiry_time = lock._expiry_time(lock._get_lease()) 559 | 560 | time.sleep(1) 561 | self.assertFalse(lock.renew()) 562 | 563 | second_expiry_time = lock._expiry_time(lock._get_lease()) 564 | self.assertEqual(second_expiry_time, first_expiry_time) 565 | 566 | def test_renew_owner_collision(self): 567 | lock_1 = sherlock.KubernetesLock( 568 | self.lock_name, 569 | self.k8s_namespace, 570 | expire=1, 571 | ) 572 | lock_2 = sherlock.KubernetesLock( 573 | self.lock_name, 574 | self.k8s_namespace, 575 | ) 576 | self.assertTrue(lock_1.acquire()) 577 | # Wait for Lock to expire 578 | time.sleep(1) 579 | self.assertTrue(lock_2.acquire()) 580 | 581 | self.assertFalse(lock_1.renew()) 582 | 583 | def test_renew_before_acquire(self): 584 | lock = sherlock.KubernetesLock(self.lock_name, self.k8s_namespace, expire=1) 585 | with self.assertRaisesRegex( 586 | sherlock.LockException, "Lock was not set by this process." 587 | ): 588 | lock.renew() 589 | 590 | def tearDown(self): 591 | try: 592 | self.client.delete_namespaced_lease( 593 | name=self.lock_name, 594 | namespace=self.k8s_namespace, 595 | ) 596 | except kubernetes.client.exceptions.ApiException as exc: 597 | if exc.reason != "Not Found": 598 | raise exc 599 | try: 600 | self.client.delete_namespaced_lease( 601 | name=f"ns-{self.lock_name}", 602 | namespace=self.k8s_namespace, 603 | ) 604 | except kubernetes.client.exceptions.ApiException as exc: 605 | if exc.reason != "Not Found": 606 | raise exc 607 | 608 | 609 | class TestFileLock(unittest.TestCase): 610 | def setUp(self): 611 | self.client = pathlib.Path("/tmp/sherlock") 612 | self.lock_name = "test-lock" 613 | self.k8s_namespace = "default" 614 | 615 | def _load_file(self, key_name): 616 | with (self.client / key_name).with_suffix(".json").open("r") as f: 617 | return json.load(f) 618 | 619 | def test_acquire(self): 620 | lock = sherlock.FileLock(self.lock_name, client=self.client) 621 | self.assertTrue(lock._acquire()) 622 | 623 | key = self.lock_name 624 | file = self._load_file(key) 625 | self.assertEqual(file["owner"], str(lock._owner)) 626 | 627 | def test_acquire_with_namespace(self): 628 | lock = sherlock.FileLock( 629 | self.lock_name, 630 | client=self.client, 631 | namespace="ns", 632 | ) 633 | self.assertTrue(lock._acquire()) 634 | key = f"ns_{self.lock_name}" 635 | file = self._load_file(key) 636 | self.assertEqual(file["owner"], str(lock._owner)) 637 | 638 | def test_acquire_once_only(self): 639 | lock1 = sherlock.FileLock( 640 | self.lock_name, 641 | client=self.client, 642 | ) 643 | lock2 = sherlock.FileLock( 644 | self.lock_name, 645 | client=self.client, 646 | ) 647 | self.assertTrue(lock1._acquire()) 648 | self.assertFalse(lock2._acquire()) 649 | 650 | def test_acquire_check_expiry(self): 651 | lock = sherlock.FileLock( 652 | self.lock_name, 653 | client=self.client, 654 | expire=1, 655 | ) 656 | lock.acquire() 657 | time.sleep(2) 658 | self.assertFalse(lock.locked()) 659 | 660 | def test_acquire_locked(self): 661 | lock = sherlock.FileLock( 662 | self.lock_name, 663 | client=self.client, 664 | expire=None, 665 | ) 666 | self.assertTrue(lock.acquire()) 667 | self.assertFalse(lock._acquire()) 668 | 669 | def test_acquire_expired_lock(self): 670 | lock = sherlock.FileLock( 671 | self.lock_name, 672 | client=self.client, 673 | expire=1, 674 | ) 675 | lock.acquire() 676 | file_1 = self._load_file(self.lock_name) 677 | time.sleep(2) 678 | 679 | # We can acquire the Lock again after it 680 | # expires. 681 | self.assertTrue(lock.acquire()) 682 | 683 | file_2 = self._load_file(self.lock_name) 684 | 685 | # New file has new owner and new owner is not the same 686 | # as old owner. 687 | self.assertEqual(file_2["owner"], str(lock._owner)) 688 | self.assertNotEqual(file_1["owner"], file_2["owner"]) 689 | 690 | @patch("sherlock.lock.uuid.uuid4") 691 | def test_acquire_owner_collison(self, mock_uuid4): 692 | mock_uuid4.return_value = b"fake-uuid" 693 | 694 | lock = sherlock.FileLock( 695 | self.lock_name, 696 | client=self.client, 697 | expire=None, 698 | ) 699 | self.assertTrue(lock.acquire()) 700 | self.assertFalse(lock.acquire(blocking=False)) 701 | 702 | def test_acquire_check_expire_is_not_set(self): 703 | lock = sherlock.FileLock( 704 | self.lock_name, 705 | client=self.client, 706 | expire=None, 707 | ) 708 | lock.acquire() 709 | time.sleep(2) 710 | file = self._load_file(self.lock_name) 711 | self.assertEqual( 712 | file["expiry_time"], 713 | datetime.datetime.max.astimezone(datetime.timezone.utc).isoformat(), 714 | ) 715 | self.assertTrue(lock.locked()) 716 | 717 | def test_release(self): 718 | lock = sherlock.FileLock(self.lock_name, client=self.client) 719 | lock._acquire() 720 | lock._release() 721 | self.assertFalse(lock._data_file.exists()) 722 | 723 | def test_release_with_namespace(self): 724 | lock = sherlock.FileLock( 725 | self.lock_name, 726 | client=self.client, 727 | namespace="ns", 728 | ) 729 | lock._acquire() 730 | lock._release() 731 | self.assertFalse(lock._data_file.exists()) 732 | 733 | def test_release_own_only(self): 734 | lock1 = sherlock.FileLock(self.lock_name, client=self.client) 735 | lock2 = sherlock.FileLock(self.lock_name, client=self.client) 736 | lock1._acquire() 737 | self.assertRaises(sherlock.LockException, lock2._release) 738 | lock1._release() 739 | 740 | def test_locked(self): 741 | lock = sherlock.FileLock(self.lock_name, client=self.client) 742 | lock._acquire() 743 | self.assertTrue(lock._locked) 744 | lock._release() 745 | self.assertFalse(lock._locked) 746 | 747 | def test_deleting_lock_object_releases_the_lock(self): 748 | lock = sherlock.FileLock(self.lock_name, client=self.client) 749 | 750 | lock.acquire() 751 | file = self._load_file(self.lock_name) 752 | self.assertEqual(file["owner"], str(lock._owner)) 753 | 754 | data_file = lock._data_file 755 | del lock 756 | self.assertFalse(data_file.exists()) 757 | 758 | def test_release_lock_that_no_longer_exists(self): 759 | lock_1 = sherlock.FileLock( 760 | self.lock_name, 761 | client=self.client, 762 | expire=1, 763 | ) 764 | lock_2 = sherlock.FileLock( 765 | self.lock_name, 766 | client=self.client, 767 | ) 768 | self.assertTrue(lock_1.acquire()) 769 | # Wait for Lock to expire 770 | time.sleep(2) 771 | self.assertTrue(lock_2.acquire()) 772 | lock_2.release() 773 | 774 | # Releasing a Lock has been removed should be fine. 775 | self.assertIsNone(lock_1.release()) 776 | 777 | def test_renew(self): 778 | lock = sherlock.FileLock(self.lock_name, client=self.client, expire=3600) 779 | self.assertTrue(lock.acquire()) 780 | first_expiry_time = datetime.datetime.fromisoformat( 781 | self._load_file(self.lock_name)["expiry_time"] 782 | ) 783 | lock.renew() 784 | second_expiry_time = datetime.datetime.fromisoformat( 785 | self._load_file(self.lock_name)["expiry_time"] 786 | ) 787 | self.assertGreater(second_expiry_time, first_expiry_time) 788 | 789 | def test_renew_expired(self): 790 | lock = sherlock.FileLock(self.lock_name, client=self.client, expire=1) 791 | self.assertTrue(lock.acquire()) 792 | first_expiry_time = datetime.datetime.fromisoformat( 793 | self._load_file(self.lock_name)["expiry_time"] 794 | ) 795 | 796 | time.sleep(1) 797 | self.assertFalse(lock.renew()) 798 | 799 | second_expiry_time = datetime.datetime.fromisoformat( 800 | self._load_file(self.lock_name)["expiry_time"] 801 | ) 802 | self.assertEqual(second_expiry_time, first_expiry_time) 803 | 804 | def test_renew_owner_collision(self): 805 | lock_1 = sherlock.FileLock( 806 | self.lock_name, 807 | client=self.client, 808 | expire=1, 809 | ) 810 | lock_2 = sherlock.FileLock( 811 | self.lock_name, 812 | client=self.client, 813 | ) 814 | self.assertTrue(lock_1.acquire()) 815 | # Wait for Lock to expire 816 | time.sleep(1) 817 | self.assertTrue(lock_2.acquire()) 818 | 819 | self.assertFalse(lock_1.renew()) 820 | 821 | def test_renew_before_acquire(self): 822 | lock = sherlock.FileLock(self.lock_name, client=self.client, expire=1) 823 | with self.assertRaisesRegex( 824 | sherlock.LockException, "Lock was not set by this process." 825 | ): 826 | lock.renew() 827 | 828 | def tearDown(self): 829 | for file in self.client.iterdir(): 830 | file.unlink() 831 | -------------------------------------------------------------------------------- /tests/test_lock.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests for all sorts of locks. 3 | """ 4 | 5 | import datetime 6 | import pathlib 7 | import tempfile 8 | import unittest 9 | from importlib import reload 10 | from unittest.mock import Mock, patch 11 | 12 | import etcd 13 | import kubernetes.client 14 | import kubernetes.client.exceptions 15 | 16 | import sherlock 17 | 18 | 19 | class TestBaseLock(unittest.TestCase): 20 | def test_init_uses_global_defaults(self): 21 | sherlock.configure(namespace="new_namespace") 22 | lock = sherlock.lock.BaseLock("lockname") 23 | self.assertEqual(lock.namespace, "new_namespace") 24 | 25 | def test_init_does_not_use_global_default_for_client_obj(self): 26 | client_obj = etcd.Client() 27 | sherlock.configure(client=client_obj) 28 | lock = sherlock.lock.BaseLock("lockname") 29 | self.assertNotEqual(lock.client, client_obj) 30 | 31 | def test__locked_raises_not_implemented_error(self): 32 | def _test(): 33 | sherlock.lock.BaseLock("")._locked 34 | 35 | self.assertRaises(NotImplementedError, _test) 36 | 37 | def test_locked_raises_not_implemented_error(self): 38 | self.assertRaises(NotImplementedError, sherlock.lock.BaseLock("").locked) 39 | 40 | def test__acquire_raises_not_implemented_error(self): 41 | self.assertRaises(NotImplementedError, sherlock.lock.BaseLock("")._acquire) 42 | 43 | def test_acquire_raises_not_implemented_error(self): 44 | self.assertRaises(NotImplementedError, sherlock.lock.BaseLock("").acquire) 45 | 46 | def test__release_raises_not_implemented_error(self): 47 | self.assertRaises(NotImplementedError, sherlock.lock.BaseLock("")._release) 48 | 49 | def test_release_raises_not_implemented_error(self): 50 | self.assertRaises(NotImplementedError, sherlock.lock.BaseLock("").release) 51 | 52 | def test_acquire_acquires_blocking_lock(self): 53 | lock = sherlock.lock.BaseLock("") 54 | lock._acquire = Mock(return_value=True) 55 | self.assertTrue(lock.acquire()) 56 | 57 | def test_acquire_acquires_non_blocking_lock(self): 58 | lock = sherlock.lock.BaseLock("123") 59 | lock._acquire = Mock(return_value=True) 60 | self.assertTrue(lock.acquire()) 61 | 62 | def test_acquire_obeys_timeout(self): 63 | lock = sherlock.lock.BaseLock("123", timeout=1) 64 | lock._acquire = Mock(return_value=False) 65 | self.assertRaises(sherlock.LockTimeoutException, lock.acquire) 66 | 67 | def test_acquire_obeys_retry_interval(self): 68 | lock = sherlock.lock.BaseLock("123", timeout=0.5, retry_interval=0.1) 69 | lock._acquire = Mock(return_value=False) 70 | try: 71 | lock.acquire() 72 | except sherlock.LockTimeoutException: 73 | pass 74 | self.assertEqual(lock._acquire.call_count, 6) 75 | 76 | def test_deleting_lock_object_releases_the_lock(self): 77 | lock = sherlock.lock.BaseLock("123") 78 | release_func = Mock() 79 | lock.release = release_func 80 | del lock 81 | self.assertTrue(release_func.called) 82 | 83 | 84 | class TestLock(unittest.TestCase): 85 | def setUp(self): 86 | reload(sherlock) 87 | reload(sherlock.lock) 88 | 89 | def test_lock_does_not_accept_custom_client_object(self): 90 | self.assertRaises(TypeError, sherlock.lock.Lock, client=None) 91 | 92 | def test_lock_does_not_create_proxy_when_backend_is_not_set(self): 93 | sherlock._configuration._backend = None 94 | sherlock._configuration._client = None 95 | lock = sherlock.lock.Lock("") 96 | self.assertEqual(lock._lock_proxy, None) 97 | 98 | self.assertRaises(sherlock.lock.LockException, lock.acquire) 99 | self.assertRaises(sherlock.lock.LockException, lock.release) 100 | self.assertRaises(sherlock.lock.LockException, lock.locked) 101 | 102 | def test_lock_creates_proxy_when_backend_is_set(self): 103 | sherlock._configuration.backend = sherlock.backends.ETCD 104 | lock = sherlock.lock.Lock("") 105 | self.assertTrue(isinstance(lock._lock_proxy, sherlock.lock.EtcdLock)) 106 | 107 | def test_lock_uses_proxys_methods(self): 108 | sherlock.lock.RedisLock._acquire = Mock(return_value=True) 109 | sherlock.lock.RedisLock._release = Mock() 110 | sherlock.lock.RedisLock.locked = Mock(return_value=False) 111 | 112 | sherlock._configuration.backend = sherlock.backends.REDIS 113 | lock = sherlock.lock.Lock("") 114 | 115 | lock.acquire() 116 | self.assertTrue(sherlock.lock.RedisLock._acquire.called) 117 | 118 | lock.release() 119 | self.assertTrue(sherlock.lock.RedisLock._release.called) 120 | 121 | lock.locked() 122 | self.assertTrue(sherlock.lock.RedisLock.locked.called) 123 | 124 | def test_lock_sets_client_object_on_lock_proxy_when_globally_configured(self): 125 | client = etcd.Client(host="8.8.8.8") 126 | sherlock.configure(client=client) 127 | lock = sherlock.lock.Lock("lock") 128 | self.assertEqual(lock._lock_proxy.client, client) 129 | 130 | 131 | class TestRedisLock(unittest.TestCase): 132 | def setUp(self): 133 | reload(sherlock) 134 | reload(sherlock.lock) 135 | 136 | def test_valid_key_names_are_generated_when_namespace_not_set(self): 137 | name = "lock" 138 | lock = sherlock.lock.RedisLock(name) 139 | self.assertEqual(lock._key_name, name) 140 | 141 | def test_valid_key_names_are_generated_when_namespace_is_set(self): 142 | name = "lock" 143 | lock = sherlock.lock.RedisLock(name, namespace="local_namespace") 144 | self.assertEqual(lock._key_name, "local_namespace_%s" % name) 145 | 146 | sherlock.configure(namespace="global_namespace") 147 | lock = sherlock.lock.RedisLock(name) 148 | self.assertEqual(lock._key_name, "global_namespace_%s" % name) 149 | 150 | 151 | class TestEtcdLock(unittest.TestCase): 152 | def setUp(self): 153 | reload(sherlock) 154 | reload(sherlock.lock) 155 | 156 | def test_valid_key_names_are_generated_when_namespace_not_set(self): 157 | name = "lock" 158 | lock = sherlock.lock.EtcdLock(name) 159 | self.assertEqual(lock._key_name, "/" + name) 160 | 161 | def test_valid_key_names_are_generated_when_namespace_is_set(self): 162 | name = "lock" 163 | lock = sherlock.lock.EtcdLock(name, namespace="local_namespace") 164 | self.assertEqual(lock._key_name, "/local_namespace/%s" % name) 165 | 166 | sherlock.configure(namespace="global_namespace") 167 | lock = sherlock.lock.EtcdLock(name) 168 | self.assertEqual(lock._key_name, "/global_namespace/%s" % name) 169 | 170 | 171 | class TestMCLock(unittest.TestCase): 172 | def setUp(self): 173 | reload(sherlock) 174 | reload(sherlock.lock) 175 | 176 | def test_valid_key_names_are_generated_when_namespace_not_set(self): 177 | name = "lock" 178 | lock = sherlock.lock.MCLock(name) 179 | self.assertEqual(lock._key_name, name) 180 | 181 | def test_valid_key_names_are_generated_when_namespace_is_set(self): 182 | name = "lock" 183 | lock = sherlock.lock.MCLock(name, namespace="local_namespace") 184 | self.assertEqual(lock._key_name, "local_namespace_%s" % name) 185 | 186 | sherlock.configure(namespace="global_namespace") 187 | lock = sherlock.lock.MCLock(name) 188 | self.assertEqual(lock._key_name, "global_namespace_%s" % name) 189 | 190 | 191 | class TestKubernetesLock(unittest.TestCase): 192 | def setUp(self): 193 | reload(sherlock) 194 | reload(sherlock.lock) 195 | 196 | def test_valid_key_names_are_generated_when_namespace_not_set(self): 197 | name = "lock" 198 | k8s_namespace = "default" 199 | lock = sherlock.lock.KubernetesLock(name, k8s_namespace, client=Mock()) 200 | self.assertEqual(lock._key_name, name) 201 | 202 | def test_valid_key_names_are_generated_when_namespace_is_set(self): 203 | name = "lock" 204 | k8s_namespace = "default" 205 | lock = sherlock.lock.KubernetesLock( 206 | name, 207 | k8s_namespace, 208 | client=Mock(), 209 | namespace="local-namespace", 210 | ) 211 | self.assertEqual(lock._key_name, "local-namespace-%s" % name) 212 | 213 | sherlock.configure(namespace="global-namespace") 214 | lock = sherlock.lock.KubernetesLock(name, k8s_namespace, client=Mock()) 215 | self.assertEqual(lock._key_name, "global-namespace-%s" % name) 216 | 217 | def test_exception_raised_when_invalid_names_set(self): 218 | test_cases = [ 219 | ( 220 | "lock_name", 221 | "my-k8s-namespace", 222 | "my-namespace", 223 | "lock_name must conform to RFC1123's definition of a DNS label for KubernetesLock", # noqa: disable=501 224 | ), 225 | ( 226 | "lock-name", 227 | "my_k8s_namespace", 228 | "my-namespace", 229 | "k8s_namespace must conform to RFC1123's definition of a DNS label for KubernetesLock", # noqa: disable=501 230 | ), 231 | ( 232 | "lock-name", 233 | "my-k8s-namespace", 234 | "my_namespace", 235 | "namespace must conform to RFC1123's definition of a DNS label for KubernetesLock", # noqa: disable=501 236 | ), 237 | ] 238 | for lock_name, k8s_namespace, namespace, err_msg in test_cases: 239 | with self.assertRaises(ValueError) as cm: 240 | sherlock.lock.KubernetesLock( 241 | lock_name, 242 | k8s_namespace, 243 | client=Mock(), 244 | namespace=namespace, 245 | ) 246 | self.assertEqual(cm.exception.args[0], err_msg) 247 | 248 | sherlock.configure(namespace=namespace) 249 | with self.assertRaises(ValueError) as cm: 250 | sherlock.lock.KubernetesLock( 251 | lock_name, 252 | k8s_namespace, 253 | client=Mock(), 254 | namespace=namespace, 255 | ) 256 | self.assertEqual(cm.exception.args[0], err_msg) 257 | 258 | @patch("kubernetes.client.CoordinationV1Api") 259 | def test_acquire_create_race_condition(self, mock_client): 260 | name = "lock" 261 | k8s_namespace = "default" 262 | 263 | # Mock the client to reproduce the scenario where the Lease 264 | # does not exist when read but does when created. 265 | mock_client.read_namespaced_lease.side_effect = ( 266 | kubernetes.client.exceptions.ApiException(reason="Not Found") 267 | ) 268 | mock_client.create_namespaced_lease.side_effect = ( 269 | kubernetes.client.exceptions.ApiException(reason="Conflict") 270 | ) 271 | lock = sherlock.lock.KubernetesLock( 272 | name, 273 | k8s_namespace, 274 | client=mock_client, 275 | ) 276 | self.assertFalse(lock._acquire()) 277 | 278 | @patch("kubernetes.client.CoordinationV1Api") 279 | def test_acquire_create_failed(self, mock_client): 280 | name = "lock" 281 | k8s_namespace = "default" 282 | 283 | # Mock the client to reproduce the scenario where the Lease 284 | # does not exist and we fail to create it for some other reason. 285 | mock_client.read_namespaced_lease.side_effect = ( 286 | kubernetes.client.exceptions.ApiException(reason="Not Found") 287 | ) 288 | mock_client.create_namespaced_lease.side_effect = ( 289 | kubernetes.client.exceptions.ApiException(reason="Not Conflict") 290 | ) 291 | lock = sherlock.lock.KubernetesLock( 292 | name, 293 | k8s_namespace, 294 | client=mock_client, 295 | ) 296 | self.assertRaisesRegex( 297 | sherlock.lock.LockException, 298 | "Failed to create Lock.", 299 | lock._acquire, 300 | ) 301 | 302 | @patch("kubernetes.client.CoordinationV1Api") 303 | def test_acquire_get_failed(self, mock_client): 304 | name = "lock" 305 | k8s_namespace = "default" 306 | 307 | # Mock the client to reproduce the scenario where we fail to read the Lease 308 | # for some other reason other than it doesn't exist. 309 | mock_client.read_namespaced_lease.side_effect = ( 310 | kubernetes.client.exceptions.ApiException(reason="Unexpected") 311 | ) 312 | lock = sherlock.lock.KubernetesLock( 313 | name, 314 | k8s_namespace, 315 | client=mock_client, 316 | ) 317 | self.assertRaisesRegex( 318 | sherlock.lock.LockException, 319 | "Failed to read Lock.", 320 | lock._acquire, 321 | ) 322 | 323 | @patch("kubernetes.client.CoordinationV1Api") 324 | def test_acquire_replaced_race_condition(self, mock_client): 325 | name = "lock" 326 | k8s_namespace = "default" 327 | 328 | lock = sherlock.lock.KubernetesLock( 329 | name, 330 | k8s_namespace, 331 | client=mock_client, 332 | ) 333 | 334 | now = lock._now() - datetime.timedelta(seconds=10) 335 | lease = kubernetes.client.V1Lease( 336 | metadata=kubernetes.client.V1ObjectMeta(name=name, namespace=k8s_namespace), 337 | spec=kubernetes.client.V1LeaseSpec( 338 | acquire_time=now, 339 | holder_identity="test-identity", 340 | lease_duration_seconds=1, 341 | renew_time=now, 342 | ), 343 | ) 344 | 345 | # Mock the client to reproduce the scenario where we try to acquire 346 | # the Lock but someone beats us to it. 347 | mock_client.read_namespaced_lease.return_value = lease 348 | mock_client.replace_namespaced_lease.side_effect = ( 349 | kubernetes.client.exceptions.ApiException(reason="Conflict") 350 | ) 351 | 352 | self.assertFalse(lock._acquire()) 353 | 354 | @patch("kubernetes.client.CoordinationV1Api") 355 | def test_acquire_replaced_failed(self, mock_client): 356 | name = "lock" 357 | k8s_namespace = "default" 358 | 359 | lock = sherlock.lock.KubernetesLock( 360 | name, 361 | k8s_namespace, 362 | client=mock_client, 363 | ) 364 | 365 | now = lock._now() - datetime.timedelta(seconds=10) 366 | lease = kubernetes.client.V1Lease( 367 | metadata=kubernetes.client.V1ObjectMeta(name=name, namespace=k8s_namespace), 368 | spec=kubernetes.client.V1LeaseSpec( 369 | acquire_time=now, 370 | holder_identity="test-identity", 371 | lease_duration_seconds=1, 372 | renew_time=now, 373 | ), 374 | ) 375 | 376 | # Mock the client to reproduce the scenario where we try to acquire 377 | # the Lock but fail to replace the Lease for an unexpected reason. 378 | mock_client.read_namespaced_lease.return_value = lease 379 | mock_client.replace_namespaced_lease.side_effect = ( 380 | kubernetes.client.exceptions.ApiException(reason="Unexpected") 381 | ) 382 | 383 | self.assertRaisesRegex( 384 | sherlock.lock.LockException, 385 | "Failed to update Lock.", 386 | lock._acquire, 387 | ) 388 | 389 | @patch("kubernetes.client.CoordinationV1Api") 390 | def test_release_delete_race_condition(self, mock_client): 391 | name = "lock" 392 | k8s_namespace = "default" 393 | 394 | lock = sherlock.lock.KubernetesLock( 395 | name, 396 | k8s_namespace, 397 | client=mock_client, 398 | ) 399 | lock._owner = "test-identity" 400 | 401 | now = lock._now() - datetime.timedelta(seconds=10) 402 | lease = kubernetes.client.V1Lease( 403 | metadata=kubernetes.client.V1ObjectMeta(name=name, namespace=k8s_namespace), 404 | spec=kubernetes.client.V1LeaseSpec( 405 | acquire_time=now, 406 | holder_identity=lock._owner, 407 | lease_duration_seconds=1, 408 | renew_time=now, 409 | ), 410 | ) 411 | 412 | # Mock the client to reproduce the scenario where we try to release 413 | # the Lock but someone acquires it before we get a chance. 414 | mock_client.read_namespaced_lease.return_value = lease 415 | mock_client.delete_namespaced_lease.side_effect = ( 416 | kubernetes.client.exceptions.ApiException(reason="Not Found") 417 | ) 418 | 419 | # This should return without issue. 420 | self.assertIsNone(lock.release()) 421 | 422 | @patch("kubernetes.client.CoordinationV1Api") 423 | def test_release_delete_failed(self, mock_client): 424 | name = "lock" 425 | k8s_namespace = "default" 426 | 427 | lock = sherlock.lock.KubernetesLock( 428 | name, 429 | k8s_namespace, 430 | client=mock_client, 431 | ) 432 | lock._owner = "test-identity" 433 | 434 | now = lock._now() - datetime.timedelta(seconds=10) 435 | lease = kubernetes.client.V1Lease( 436 | metadata=kubernetes.client.V1ObjectMeta(name=name, namespace=k8s_namespace), 437 | spec=kubernetes.client.V1LeaseSpec( 438 | acquire_time=now, 439 | holder_identity=lock._owner, 440 | lease_duration_seconds=1, 441 | renew_time=now, 442 | ), 443 | ) 444 | 445 | # Mock the client to reproduce the scenario where we try to release 446 | # the Lock but fail to delete the Lease for an unexpected reason. 447 | mock_client.read_namespaced_lease.return_value = lease 448 | mock_client.delete_namespaced_lease.side_effect = ( 449 | kubernetes.client.exceptions.ApiException(reason="Unexpected") 450 | ) 451 | 452 | self.assertRaisesRegex( 453 | sherlock.lock.LockException, 454 | "Failed to release Lock.", 455 | lock.release, 456 | ) 457 | 458 | 459 | class TestFileLock(unittest.TestCase): 460 | def setUp(self): 461 | reload(sherlock) 462 | reload(sherlock.lock) 463 | 464 | def test_valid_key_names_are_generated_when_namespace_not_set(self): 465 | name = "lock" 466 | 467 | with tempfile.TemporaryDirectory() as tmpdir: 468 | lock = sherlock.lock.FileLock(name, client=pathlib.Path(tmpdir)) 469 | 470 | self.assertEqual(lock._key_name, name) 471 | 472 | def test_valid_key_names_are_generated_when_namespace_is_set(self): 473 | name = "lock" 474 | 475 | with tempfile.TemporaryDirectory() as tmpdir: 476 | lock = sherlock.lock.FileLock( 477 | name, 478 | client=pathlib.Path(tmpdir), 479 | namespace="local_namespace", 480 | ) 481 | 482 | self.assertEqual(lock._key_name, "local_namespace_%s" % name) 483 | 484 | sherlock.configure(namespace="global_namespace") 485 | with tempfile.TemporaryDirectory() as tmpdir: 486 | lock = sherlock.lock.FileLock(name, client=pathlib.Path(tmpdir)) 487 | self.assertEqual(lock._key_name, "global_namespace_%s" % name) 488 | -------------------------------------------------------------------------------- /tests/test_sherlock.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests for some basic package's root level functionality. 3 | """ 4 | 5 | import unittest 6 | from importlib import reload 7 | from unittest.mock import Mock 8 | 9 | import etcd 10 | 11 | import sherlock 12 | from sherlock import _Configuration 13 | 14 | 15 | class TestConfiguration(unittest.TestCase): 16 | def setUp(self): 17 | reload(sherlock) 18 | self.configure = _Configuration() 19 | 20 | def test_update_settings_raises_error_when_updating_invalid_config(self): 21 | # Raises error when trying to update invalid setting 22 | self.assertRaises(AttributeError, self.configure.update, invalid_arg="val") 23 | 24 | def test_updates_valid_settings(self): 25 | # Updates valid setting 26 | self.configure.update(namespace="something") 27 | self.assertEqual(self.configure.namespace, "something") 28 | 29 | def test_backend_gets_backend_when_backend_is_not_set(self): 30 | # When backend is not set 31 | self.assertEqual(self.configure._backend, None) 32 | self.assertEqual(self.configure._backend, self.configure.backend) 33 | self.assertEqual(self.configure._backend, None) 34 | 35 | def test_backend_gets_backend_when_backend_is_set(self): 36 | # When backend is set 37 | self.configure.backend = sherlock.backends.ETCD 38 | self.assertEqual(self.configure._backend, self.configure.backend) 39 | self.assertEqual(self.configure._backend, sherlock.backends.ETCD) 40 | 41 | def test_backend_raises_error_on_setting_invalid_backend(self): 42 | def _test(): 43 | # Set some unexpected value 44 | self.configure.backend = 0 45 | 46 | self.assertRaises(ValueError, _test) 47 | 48 | def test_backend_sets_backend_value(self): 49 | self.configure.backend = sherlock.backends.ETCD 50 | self.assertEqual(self.configure._backend, sherlock.backends.ETCD) 51 | 52 | def test_client_returns_the_set_client_object(self): 53 | client = Mock() 54 | self.configure._client = client 55 | self.assertEqual(self.configure.client, self.configure._client) 56 | self.assertEqual(self.configure._client, client) 57 | 58 | def test_client_raises_error_when_backend_is_not_set(self): 59 | # Make sure backend is set to None 60 | self.assertEqual(self.configure.backend, None) 61 | 62 | def _test(): 63 | self.configure.client 64 | 65 | self.assertRaises(ValueError, _test) 66 | 67 | def test_client_returns_client_when_not_set_but_backend_is_set(self): 68 | self.configure.backend = sherlock.backends.ETCD 69 | self.assertTrue(isinstance(self.configure.client, etcd.Client)) 70 | 71 | def test_client_sets_valid_client_obj_only_when_backend_set(self): 72 | # When backend is set and client object is invalid 73 | self.configure.backend = sherlock.backends.ETCD 74 | 75 | def _test(): 76 | self.configure.client = None 77 | 78 | self.assertRaises(ValueError, _test) 79 | 80 | # When backend is set and client object is valid 81 | self.configure.client = etcd.Client() 82 | 83 | def test_client_sets_valid_client_obj_only_when_backend_not_set(self): 84 | # When backend is not set and client library is available and client is 85 | # valid 86 | self.configure._backend = None 87 | self.assertEqual(self.configure.backend, None) 88 | client_obj = etcd.Client() 89 | self.configure.client = client_obj 90 | self.assertEqual(self.configure.client, client_obj) 91 | self.assertTrue(isinstance(self.configure.client, etcd.Client)) 92 | 93 | # When backend is not set and client library is available and client is 94 | # invalid 95 | self.configure._backend = None 96 | self.configure._client = None 97 | self.assertEqual(self.configure.backend, None) 98 | client_obj = "Random" 99 | 100 | def _test(): 101 | self.configure.client = client_obj 102 | 103 | self.assertRaises(ValueError, _test) 104 | 105 | # When backend is not set and client library is available and client is 106 | # valid 107 | self.configure._backend = None 108 | self.configure._client = None 109 | self.assertEqual(self.configure.backend, None) 110 | client_obj = etcd.Client() 111 | self.configure.client = client_obj 112 | 113 | 114 | def testConfigure(): 115 | """ 116 | Test the library configure function. 117 | """ 118 | 119 | sherlock.configure(namespace="namespace") 120 | assert sherlock._configuration.namespace == "namespace" 121 | 122 | 123 | class TestBackends(unittest.TestCase): 124 | def setUp(self): 125 | reload(sherlock) 126 | 127 | def test_valid_backends(self): 128 | self.assertEqual( 129 | sherlock.backends.valid_backends, sherlock.backends._valid_backends 130 | ) 131 | 132 | def test_register_raises_exception_when_lock_class_invalid(self): 133 | self.assertRaises( 134 | ValueError, sherlock.backends.register, "MyLock", object, "some_lib", object 135 | ) 136 | 137 | def test_register_registers_custom_backend(self): 138 | class MyLock(sherlock.lock.BaseLock): 139 | pass 140 | 141 | name = "MyLock" 142 | lock_class = MyLock 143 | library = "some_lib" 144 | client_class = object 145 | args = (1, 2, 3) 146 | kwargs = dict(somekey="someval") 147 | sherlock.backends.register( 148 | name=name, 149 | lock_class=lock_class, 150 | library=library, 151 | client_class=client_class, 152 | default_args=args, 153 | default_kwargs=kwargs, 154 | ) 155 | 156 | self.assertTrue(isinstance(sherlock.backends.MyLock, dict)) 157 | self.assertEqual(sherlock.backends.MyLock["name"], name) 158 | self.assertEqual(sherlock.backends.MyLock["lock_class"], lock_class) 159 | self.assertEqual(sherlock.backends.MyLock["library"], library) 160 | self.assertEqual(sherlock.backends.MyLock["client_class"], client_class) 161 | self.assertEqual(sherlock.backends.MyLock["default_args"], args) 162 | self.assertEqual(sherlock.backends.MyLock["default_kwargs"], kwargs) 163 | 164 | self.assertTrue(sherlock.backends.MyLock in sherlock.backends.valid_backends) 165 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = 3 | py37, 4 | py38, 5 | py39, 6 | py310, 7 | py311 8 | isolated_build = true 9 | 10 | [testenv] 11 | setenv = 12 | ETCD_HOST = 127.0.0.1 13 | KUBECONFIG = {toxinidir}/kubeconfig.yaml 14 | MEMCACHED_HOST = 127.0.0.1 15 | REDIS_HOST = 127.0.0.1 16 | commands = 17 | poetry install --all-extras --verbose 18 | poetry run pytest \ 19 | --cov {toxinidir}/sherlock \ 20 | --cov-report term \ 21 | --cov-report xml:{toxinidir}/coverage.xml \ 22 | {toxinidir}/tests 23 | docker = 24 | etcd 25 | redis 26 | memcached 27 | kubernetes 28 | whitelist_externals = poetry 29 | 30 | [tox:.package] 31 | basepython = python3 32 | 33 | [docker:etcd] 34 | image = quay.io/coreos/etcd 35 | environment = 36 | ETCD_LISTEN_CLIENT_URLS=http://0.0.0.0:2379,http://0.0.0.0:4001 37 | ETCD_ADVERTISE_CLIENT_URLS=http://0.0.0.0:2379,http://0.0.0.0:4001 38 | ports = 39 | 2379:2379/tcp 40 | 4001:4001/tcp 41 | 42 | [docker:redis] 43 | image = redis 44 | ports = 6379:6379/tcp 45 | 46 | [docker:memcached] 47 | image = memcached 48 | ports = 11211:11211/tcp 49 | 50 | [docker:kubernetes] 51 | image = rancher/k3s 52 | command = server 53 | privileged = true 54 | environment = 55 | K3S_TOKEN=13521885930989 56 | K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml 57 | K3S_KUBECONFIG_MODE=666 58 | volumes = 59 | bind:rw:{toxinidir}:/output 60 | ports = 6443:6443/tcp 61 | healthcheck_cmd = kubectl get --raw='/readyz?verbose' 62 | healthcheck_timeout = 120 63 | healthcheck_retries = 999 64 | healthcheck_interval = 5 65 | healthcheck_start_period = 0 66 | 67 | [gh-actions] 68 | python = 69 | 3.7: py37 70 | 3.8: py38 71 | 3.9: py39 72 | 3.10: py310 73 | 3.11: py311 74 | fail_on_no_env = True 75 | 76 | [flake8] 77 | max-line-length = 88 78 | extend-ignore = "E203" 79 | 80 | [pycodestyle] 81 | max-line-length = 88 82 | --------------------------------------------------------------------------------