├── .github └── workflows │ ├── build_docs.yml │ └── run_tests.yml ├── .gitignore ├── .readthedocs.yaml ├── CHANGELOG.md ├── LICENSE ├── MANIFEST.in ├── README.md ├── docs ├── CHANGELOG.md ├── api │ ├── inter_process.md │ └── inter_thread.md ├── guide │ ├── glossary.md │ ├── index.md │ ├── inter_process.md │ └── inter_thread.md ├── img │ ├── favicon-192x192.png │ └── favicon-32x32.png └── index.md ├── fasteners ├── __init__.py ├── _utils.py ├── lock.py ├── process_lock.py ├── process_mechanism.py ├── pywin32 │ ├── __init__.py │ ├── pywintypes.py │ ├── win32con.py │ └── win32file.py └── version.py ├── mkdocs.yml ├── publish.md ├── pyproject.toml ├── requirements-docs.txt ├── requirements-pkg.txt ├── requirements-test.txt ├── setup.cfg └── tests ├── __init__.py ├── test_decorators.py ├── test_eventlet.py ├── test_helpers.py ├── test_lock.py ├── test_process_lock.py └── test_reader_writer_lock.py /.github/workflows/build_docs.yml: -------------------------------------------------------------------------------- 1 | name: GitHub Pages 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | 8 | jobs: 9 | deploy: 10 | runs-on: ubuntu-20.04 11 | concurrency: 12 | group: ${{ github.workflow }}-${{ github.ref }} 13 | steps: 14 | - uses: actions/checkout@v2 15 | 16 | - name: Setup Python 17 | uses: actions/setup-python@v2 18 | with: 19 | python-version: '3.8' 20 | 21 | - name: Upgrade pip 22 | run: | 23 | # install pip=>20.1 to use "pip cache dir" 24 | python3 -m pip install --upgrade pip 25 | 26 | - name: Get pip cache dir 27 | id: pip-cache 28 | run: echo "::set-output name=dir::$(pip cache dir)" 29 | 30 | - name: Cache dependencies 31 | uses: actions/cache@v2 32 | with: 33 | path: ${{ steps.pip-cache.outputs.dir }} 34 | key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} 35 | restore-keys: | 36 | ${{ runner.os }}-pip- 37 | 38 | - name: Install dependencies 39 | run: python3 -m pip install -r ./requirements-docs.txt 40 | 41 | - name: Install itself 42 | run: python3 -m pip install -e . 43 | 44 | - run: mkdocs build 45 | 46 | - name: Deploy 47 | uses: peaceiris/actions-gh-pages@v3 48 | if: ${{ github.ref == 'refs/heads/main' }} 49 | with: 50 | github_token: ${{ secrets.GITHUB_TOKEN }} 51 | publish_dir: ./site -------------------------------------------------------------------------------- /.github/workflows/run_tests.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a variety of Python versions 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: run tests 5 | 6 | on: 7 | push: 8 | branches: [ main ] 9 | pull_request: 10 | branches: [ main ] 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ${{ matrix.os }} 16 | strategy: 17 | matrix: 18 | os: [ubuntu-latest, macos-latest, windows-latest] 19 | python-version: [3.8, 3.9, '3.10', '3.11', 'pypy3.9', 'pypy3.10'] 20 | 21 | steps: 22 | - uses: actions/checkout@v2 23 | - name: Set up Python ${{ matrix.python-version }} 24 | uses: actions/setup-python@v4 25 | with: 26 | python-version: ${{ matrix.python-version }} 27 | - name: Install dependencies 28 | run: | 29 | python -m pip install --upgrade pip 30 | python -m pip install . 31 | python -m pip install -r requirements-test.txt 32 | - name: Test with pytest 33 | run: | 34 | pytest tests/ 35 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.egg-info 2 | __pycache__ 3 | dist/ 4 | build/ 5 | site/ 6 | venv/ 7 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # Read the Docs configuration file 2 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 3 | version: 2 4 | 5 | build: 6 | os: ubuntu-22.04 7 | tools: 8 | python: "3.11" 9 | 10 | mkdocs: 11 | configuration: mkdocs.yml 12 | fail_on_warning: false 13 | 14 | python: 15 | install: 16 | - requirements: requirements-docs.txt 17 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # ChangeLog 2 | 3 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) 4 | 5 | ## [Unreleased] 6 | 7 | ## [0.19] 8 | - Add `.acquire_read_lock`, `.release_read_lock`, `.acquire_write_lock`, and 9 | `.release_write_lock` methods to the inter thread `ReaderWriterLock` as was 10 | promised in the README. 11 | - Remove support for python 3.7 and pypy 3.7. It should still work, but is no 12 | longer tested. 13 | - Add support for pypy 3.10 and python 3.11 14 | 15 | ## [0.18] 16 | - Reshuffle the process lock code and properly document it. 17 | - Revamp the docs and switch from sphinx to mkdocs 18 | - Remove difficult to use tread lock features from docs 19 | - Bring back support for eventlet `spawn_n` 20 | - Remove support for python3.6. It should still work, but is no longer tested. 21 | 22 | ## [0.17.3]: 23 | - Allow writer to become a reader in thread ReaderWriter lock 24 | 25 | ## [0.17.2]: 26 | - Remove unnecessary setuptools pin 27 | 28 | ## [0.17.1]: 29 | - Switch to the modern python package build infrastructure 30 | 31 | ## [0.17]: [NEVER RELEASED] 32 | - Remove support for python 3.5 and earlier, including 2.7 33 | - Add support for python 3.9 and 3.10 34 | - Fix a conflict with django lock 35 | - Add `__version__` and `__all__` attributes 36 | - Fix a failure to parse README as utf-8 37 | - Move from nosetest to pytest and cleanup testing infrastructure 38 | 39 | ## [0.16.3]: 40 | - Fix a failure to parse README as utf-8 on python2 41 | 42 | ## [0.16.2]: 43 | - Fix a failure to parse README as utf-8 44 | 45 | ## [0.16.1]: [YANKED] 46 | 47 | ## [0.16]: 48 | - Move from travis and appveyor to github actions 49 | - Add interprocess reader writer lock 50 | - Improve README 51 | - remove unused eventlet import 52 | - use stdlib monotonic instead of external for python >= 3.4 53 | 54 | ## [0.15]: 55 | - Add testing for additional python versions 56 | - Remove python 2.6 support 57 | - Remove eventlet dependency and use 58 | threading.current_thread instead 59 | 60 | ## [0.14]: 61 | - Allow providing a custom exception logger to 'locked' decorator 62 | - Allow providing a custom logger to process lock class 63 | - Fix issue #12 64 | 65 | ## [0.13]: 66 | - Fix 'ensure_tree' check on freebsd 67 | 68 | ## [0.12]: 69 | - Use a tiny retry util helper class for performing process locking retries. 70 | 71 | ## [0.11]: 72 | - Directly use monotonic.monotonic. 73 | - Use BLATHER level for previously INFO/DEBUG statements. 74 | 75 | ## [0.10]: 76 | - Add LICENSE in generated source tarballs 77 | - Add a version.py file that can be used to extract the current version. 78 | 79 | ## [0.9]: 80 | - Allow providing a non-standard (eventlet or other condition class) to the 81 | r/w lock for cases where it is useful to do so. 82 | - Instead of having the r/w lock take a find eventlet keyword argument, allow 83 | for it to be provided a function that will be later called to get the 84 | current thread. This allows for the current *hack* to be easily removed 85 | by users (if they so desire). 86 | 87 | ## [0.8]: 88 | - Add fastener logo (from openclipart). 89 | - Ensure r/w writer -> reader -> writer lock acquisition. 90 | - Attempt to use the monotonic pypi module if its installed for monotonically 91 | increasing time on python versions where this is not built-in. 92 | 93 | ## [0.7]: 94 | - Add helpful `locked` decorator that can lock a method using a found 95 | attribute (a lock object or list of lock objects) in the instance the method 96 | is attached to. 97 | - Expose top level `try_lock` function. 98 | 99 | ## [0.6]: 100 | - Allow the sleep function to be provided (so that various alternatives other 101 | than time.sleep can be used), ie eventlet.sleep (or other). 102 | - Remove dependency on oslo.utils (replace with small utility code that 103 | achieves the same effect). 104 | 105 | ## [0.5]: 106 | - Make it possible to provide an acquisition timeout to the interprocess lock 107 | (which when acquisition can not complete in the desired time will return 108 | false). 109 | 110 | ## [0.4]: 111 | - Have the interprocess lock acquire take a blocking keyword argument 112 | (defaulting to true) that can avoid blocking trying to acquire the lock 113 | 114 | ## [0.3]: 115 | - Renamed from 'shared_lock' to 'fasteners' 116 | 117 | ## [0.2.1] 118 | - Fix delay not working as expected 119 | 120 | ## [0.2]: 121 | - Add a interprocess lock 122 | 123 | ## [0.1]: 124 | - Add travis yaml file 125 | - Initial commit/import 126 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | 177 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include CHANGELOG.md 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Fasteners 2 | ========= 3 | 4 | [![Documentation status](https://readthedocs.org/projects/fasteners/badge/?version=latest)](https://readthedocs.org/projects/fasteners/?badge=latest) 5 | [![Downloads](https://img.shields.io/pypi/dm/fasteners.svg)](https://pypi.python.org/pypi/fasteners/) 6 | [![Latest version](https://img.shields.io/pypi/v/fasteners.svg)](https://pypi.python.org/pypi/fasteners/) 7 | 8 | Cross-platform locks for threads and processes. 9 | 10 | 🔩 Install 11 | ---------- 12 | 13 | ``` 14 | pip install fasteners 15 | ``` 16 | 17 | 🔩 Usage 18 | -------- 19 | Lock for processes has the same API as the 20 | [threading.Lock](https://docs.python.org/3/library/threading.html#threading.Lock) 21 | for threads: 22 | ```python 23 | import fasteners 24 | import threading 25 | 26 | lock = threading.Lock() # for threads 27 | lock = fasteners.InterProcessLock('path/to/lock.file') # for processes 28 | 29 | with lock: 30 | ... # exclusive access 31 | 32 | # or alternatively 33 | 34 | lock.acquire() 35 | ... # exclusive access 36 | lock.release() 37 | ``` 38 | 39 | Reader Writer lock has a similar API, which is the same for threads or processes: 40 | 41 | ```python 42 | import fasteners 43 | 44 | rw_lock = fasteners.ReaderWriterLock() # for threads 45 | rw_lock = fasteners.InterProcessReaderWriterLock('path/to/lock.file') # for processes 46 | 47 | with rw_lock.write_lock(): 48 | ... # write access 49 | 50 | with rw_lock.read_lock(): 51 | ... # read access 52 | 53 | # or alternatively 54 | 55 | rw_lock.acquire_read_lock() 56 | ... # read access 57 | rw_lock.release_read_lock() 58 | 59 | rw_lock.acquire_write_lock() 60 | ... # write access 61 | rw_lock.release_write_lock() 62 | ``` 63 | 64 | 🔩 Overview 65 | ----------- 66 | 67 | Python standard library provides a lock for threads (both a reentrant one, and a 68 | non-reentrant one, see below). Fasteners extends this, and provides a lock for 69 | processes, as well as Reader Writer locks for both threads and processes. 70 | Definitions of terms used in this overview can be found in the 71 | [glossary](https://fasteners.readthedocs.io/en/latest/guide/glossary/). 72 | 73 | The specifics of the locks are as follows: 74 | 75 | ### Process locks 76 | 77 | The `fasteners.InterProcessLock` uses [fcntl](https://man7.org/linux/man-pages/man2/fcntl.2.html) on Unix-like systems and 78 | msvc [_locking](https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/locking?view=msvc-160) on Windows. 79 | As a result, if used cross-platform it guarantees an intersection of their features: 80 | 81 | | lock | reentrant | mandatory | 82 | |------|-----------|-----------| 83 | | fcntl | ✘ | ✘ | 84 | | _locking | ✔ | ✔ | 85 | | fasteners.InterProcessLock | ✘ | ✘ | 86 | 87 | 88 | The `fasteners.InterProcessReaderWriterLock` also uses fcntl on Unix-like systems and 89 | [LockFileEx](https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-lockfileex) on Windows. Their 90 | features are as follows: 91 | 92 | | lock | reentrant | mandatory | upgradable | preference | 93 | |------|-----------|-----------|------------|------------| 94 | | fcntl | ✘ | ✘ | ✔ | reader | 95 | | LockFileEx | ✔ | ✔ | ✘ | reader | 96 | | fasteners.InterProcessReaderWriterLock | ✘ | ✘ | ✘ | reader | 97 | 98 | 99 | ### Thread locks 100 | 101 | Fasteners does not provide a simple thread lock, but for the sake of comparison note that the `threading` module 102 | provides both a reentrant and non-reentrant locks: 103 | 104 | | lock | reentrant | mandatory | 105 | |------|-----------|-----------| 106 | | threading.Lock | ✘ | ✘ | 107 | | threading.RLock | ✔ | ✘ | 108 | 109 | 110 | The `fasteners.ReaderWriterLock` at the moment is as follows: 111 | 112 | | lock | reentrant | mandatory | upgradable | preference | 113 | |------|-----------|-----------|-------------|------------| 114 | | fasteners.ReaderWriterLock | ✔ | ✘ | ✘ | writer | 115 | 116 | If your threads are created by some other means than the standard library `threading` 117 | module (for example `eventlet`), you may need to provide the corresponding thread 118 | identification and synchronisation functions to the `ReaderWriterLock`. -------------------------------------------------------------------------------- /docs/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ../CHANGELOG.md -------------------------------------------------------------------------------- /docs/api/inter_process.md: -------------------------------------------------------------------------------- 1 | # Process Lock API 2 | 3 | ::: fasteners.process_lock.InterProcessLock 4 | 5 | ::: fasteners.process_lock.InterProcessReaderWriterLock 6 | 7 | ## Decorators 8 | 9 | ::: fasteners.process_lock.interprocess_locked 10 | rendering: 11 | heading_level: 3 12 | 13 | ::: fasteners.process_lock.interprocess_read_locked 14 | rendering: 15 | heading_level: 3 16 | 17 | ::: fasteners.process_lock.interprocess_write_locked 18 | rendering: 19 | heading_level: 3 -------------------------------------------------------------------------------- /docs/api/inter_thread.md: -------------------------------------------------------------------------------- 1 | # Thread lock API 2 | 3 | ::: fasteners.lock.ReaderWriterLock 4 | -------------------------------------------------------------------------------- /docs/guide/glossary.md: -------------------------------------------------------------------------------- 1 | # Glossary 2 | 3 | To learn more about the various aspects of locks, check the wikipedia pages for 4 | [locks](https://en.wikipedia.org/wiki/Lock_(computer_science)) and 5 | [readers writer locks](https://en.wikipedia.org/wiki/Readers%E2%80%93writer_lock) 6 | Here we briefly mention the main notions used in the documentation. 7 | 8 | * **Lock** - a mechanism that prevents two or more threads or processes from running the same code at the same time. 9 | * **Readers writer lock** - a mechanism that prevents two or more threads from having write (or write and read) access, 10 | while allowing multiple readers. 11 | * **Reentrant lock** - a lock that can be acquired (and then released) multiple times, as in: 12 | 13 | ```python 14 | with lock: 15 | with lock: 16 | ... # some code 17 | ``` 18 | 19 | * **Mandatory lock** (as opposed to advisory lock) - a lock that is enforced by the operating system, rather than 20 | by the cooperation between threads or processes 21 | * **Upgradable readers writer lock** - a readers writer lock that can be upgraded from reader to writer (or downgraded 22 | from writer to reader) without losing the lock that is already held, as in: 23 | ```python 24 | with rw_lock.read_lock(): 25 | ... # read access 26 | with rw_lock.write_lock(): 27 | ... # write access 28 | ... # read access 29 | ``` 30 | * **Readers writer lock preference** - describes the behaviour when multiple threads or processes are waiting for 31 | access. Some of the patterns are: 32 | * **Reader preference** - If lock is held by readers, then new readers will get immediate access. This can result 33 | in writers waiting forever (writer starvation). 34 | * **Writer preference** - If writer is waiting for a lock, then all the new readers (and writers) will be queued 35 | after it. This can result in readers waiting forever (reader starvation). 36 | * **Phase fair** - Lock that alternates between readers and writers. 37 | 38 | -------------------------------------------------------------------------------- /docs/guide/index.md: -------------------------------------------------------------------------------- 1 | # User Guide 2 | 3 | ## Basic Exclusive Lock Usage 4 | 5 | Exclusive Lock for independent processes has the same API as the 6 | [threading.Lock](https://docs.python.org/3/library/threading.html#threading.Lock) 7 | for threads: 8 | ```python 9 | import fasteners 10 | import threading 11 | 12 | lock = threading.Lock() # for threads 13 | lock = fasteners.InterProcessLock('path/to/lock.file') # for processes 14 | 15 | with lock: 16 | ... # exclusive access 17 | 18 | # or alternatively 19 | 20 | lock.acquire() 21 | ... # exclusive access 22 | lock.release() 23 | ``` 24 | 25 | ## Basic Reader Writer lock usage 26 | 27 | Reader Writer lock has a similar API, which is the same for threads or processes: 28 | 29 | ```python 30 | import fasteners 31 | 32 | # for threads 33 | rw_lock = fasteners.ReaderWriterLock() 34 | # for processes 35 | rw_lock = fasteners.InterProcessReaderWriterLock('path/to/lock.file') 36 | 37 | with rw_lock.write_lock(): 38 | ... # write access 39 | 40 | with rw_lock.read_lock(): 41 | ... # read access 42 | 43 | # or alternatively, for processes only: 44 | 45 | rw_lock.acquire_read_lock() 46 | ... # read access 47 | rw_lock.release_read_lock() 48 | 49 | rw_lock.acquire_write_lock() 50 | ... # write access 51 | rw_lock.release_write_lock() 52 | ``` 53 | 54 | ## Advanced usage 55 | 56 | For more details and options see [Process lock details](inter_process.md) and [Thread lock details](inter_thread.md). 57 | 58 | -------------------------------------------------------------------------------- /docs/guide/inter_process.md: -------------------------------------------------------------------------------- 1 | # Inter process locks 2 | 3 | Fasteners inter-process locks are cross-platform and are released automatically 4 | if the process crashes. They are based on the platform specific locking 5 | mechanisms: 6 | 7 | * fcntl for posix (Linux and OSX) 8 | * LockFileEx (via pywin32) and \_locking (via msvcrt) for Windows 9 | 10 | ## Difference from `multiprocessing.Lock` 11 | 12 | Python standard library [multiprocessing.Lock] functions when the processes are 13 | launched by a single main process, who is responsible for managing the 14 | synchronization. `fasteners` locks use the operating system mechanisms for 15 | synchronization management, and hence work between processes that were launched 16 | independently. 17 | 18 | ## Timeouts 19 | 20 | `fasteners` locks support timeouts, that can be used as follows: 21 | 22 | ```python 23 | import fasteners 24 | 25 | lock = fasteners.InterProcessLock('path/to/lock.file') 26 | 27 | lock.acquire(timeout=10) 28 | ... # exclusive access 29 | lock.release() 30 | ``` 31 | 32 | Equivalently for readers writer lock: 33 | 34 | 35 | ```python 36 | import fasteners 37 | 38 | lock = fasteners.InterProcessReaderWriterLock('path/to/lock.file') 39 | 40 | lock.acquire_read_lock(timeout=10) 41 | ... # exclusive access 42 | lock.release_read_lock() 43 | 44 | lock.acquire_write_lock(timeout=10) 45 | ... # exclusive access 46 | lock.release_write_lock() 47 | ``` 48 | 49 | ## Decorators 50 | 51 | For extra sugar, a function that always needs exclusive / read / write access 52 | can be decorated using one of the provided decorators. Note that they do not 53 | expose the timeout parameter, and always block until the lock is acquired. 54 | 55 | ```python 56 | import fasteners 57 | 58 | 59 | @fasteners.interprocess_read_locked 60 | def read_file(): 61 | ... 62 | 63 | @fasteners.interprocess_write_locked 64 | def write_file(): 65 | ... 66 | 67 | @fasteners.interprocess_locked 68 | def do_something_exclusive(): 69 | ... 70 | ``` 71 | 72 | ## (Lack of) Features 73 | 74 | The intersection of fcntl and LockFileEx features is quite small, hence you 75 | should always assume that: 76 | 77 | * Locks are advisory. They do not prevent the modification of the locked file 78 | by other processes. 79 | 80 | * Locks can be unintentionally released by simply opening and closing the file 81 | descriptor, so lock files must be accessed only using provided abstractions. 82 | 83 | * Locks are not [reentrant]. An attempt to acquire a lock multiple times can 84 | result in a deadlock or a crash upon a release of the lock. 85 | 86 | * Reader writer locks are not [upgradeable]. An attempt to get a reader's lock 87 | while holding a writer's lock (or vice versa) can result in a deadlock or a 88 | crash upon a release of the lock. 89 | 90 | * There are no guarantees regarding usage by multiple threads in a 91 | single process. The locks work only between processes. 92 | 93 | ## Resources 94 | 95 | To learn more about the complications of locking on different platforms we 96 | recommend the following resources: 97 | 98 | * [File locking in Linux (blog post)](https://gavv.github.io/articles/file-locks/) 99 | 100 | * [On the Brokenness of File Locking (blog post)](http://0pointer.de/blog/projects/locking.html) 101 | 102 | * [Everything you never wanted to know about file locking (blog post)](https://chris.improbable.org/2010/12/16/everything-you-never-wanted-to-know-about-file-locking/) 103 | 104 | * [Record Locking (course notes)](http://poincare.matf.bg.ac.rs/~ivana/courses/ps/sistemi_knjige/pomocno/apue/APUE/0201433079/ch14lev1sec3.html) 105 | 106 | * [Windows NT Files -- Locking (pywin32 docs)](http://timgolden.me.uk/pywin32-docs/Windows_NT_Files_.2d.2d_Locking.html) 107 | 108 | * [_locking (Windows Dev Center)](https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/locking?view=vs-2019) 109 | 110 | * [LockFileEx function (Windows Dev Center)](https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-lockfileex) 111 | 112 | [upgradeable]: https://en.wikipedia.org/wiki/Readers%E2%80%93writer_lock#Upgradable_RW_lock> 113 | [reentrant]: https://en.wikipedia.org/wiki/Reentrant_mutex 114 | [multiprocessing.Lock]: https://docs.python.org/3/library/multiprocessing.html#multiprocessing.Lock -------------------------------------------------------------------------------- /docs/guide/inter_thread.md: -------------------------------------------------------------------------------- 1 | # Inter thread locks 2 | 3 | Fasteners inter-thread locks were build specifically for the needs of 4 | `oslo.concurrency` project and thus have a rather peculiar API. We do not 5 | recommend using it fully, and it is hence not documented (but maintained until 6 | the end of time). 7 | 8 | Instead, we recommend limiting the use of fasteners inter-thread readers writer 9 | lock to the basic API: 10 | 11 | ```python 12 | import fasteners 13 | 14 | # for threads 15 | rw_lock = fasteners.ReaderWriterLock() 16 | 17 | with rw_lock.write_lock(): 18 | ... # write access 19 | 20 | with rw_lock.read_lock(): 21 | ... # read access 22 | ``` 23 | 24 | ## (Lack of) Features 25 | 26 | Fasteners inter-thread readers writer lock is 27 | 28 | * not [upgradeable]. An attempt to get a reader's lock while holding a writer's 29 | lock (or vice versa) will raise an exception. 30 | 31 | * [reentrant] (!). You can acquire (and correspondingly release) the lock 32 | multiple times. 33 | 34 | * has writer preference. Readers will queue after writers and pending writers. 35 | 36 | [upgradeable]: https://en.wikipedia.org/wiki/Readers%E2%80%93writer_lock#Upgradable_RW_lock> 37 | 38 | [reentrant]: https://en.wikipedia.org/wiki/Reentrant_mutex 39 | 40 | ## Different thread creation mechanisms 41 | 42 | If your threads are created by some other means than the standard library `threading` 43 | module, you may need to provide corresponding thread identification and synchronisation 44 | functions to the `ReaderWriterLock`. 45 | 46 | ### Eventlet 47 | 48 | In particular, in case of `eventlet` threads, you should monkey_patch the stdlib threads with `eventlet.monkey_patch(tread=True)` 49 | and initialise the `ReaderWriterLock` as `ReaderWriterLock(current_thread_functor=eventlet.getcurrent)`. 50 | 51 | -------------------------------------------------------------------------------- /docs/img/favicon-192x192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/harlowja/fasteners/06c3f06cab4e135b8d921932019a231c180eb9f4/docs/img/favicon-192x192.png -------------------------------------------------------------------------------- /docs/img/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/harlowja/fasteners/06c3f06cab4e135b8d921932019a231c180eb9f4/docs/img/favicon-32x32.png -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | # Fasteners 2 | 3 | Python standard library provides an Exclusive Lock for threads and Exclusive Lock for processes 4 | spawned by `multiprocessing` module. `fasteners` provides additional three synchronization primitives: 5 | 6 | * Exclusive Lock for independent processes 7 | * Readers Writer Lock for independent processes 8 | * Readers Writer Lock for threads 9 | 10 | ## Installation 11 | 12 | ``` 13 | pip install fasteners 14 | ``` 15 | 16 | ## Usage 17 | 18 | See [User Guide](guide/index.md) for usage tips and examples and [Reference](api/inter_process.md) for detailed API. 19 | 20 | ## Similar libraries 21 | 22 | [`portarlocker`](https://github.com/WoLpH/portalocker): readers writer lock and semaphore for 23 | independent processes, exclusive lock based on redis. 24 | 25 | [`py-filelock`](https://github.com/tox-dev/py-filelock): exclusive lock for independent processes. 26 | 27 | [`pyReaderWriterLock`](https://github.com/elarivie/pyReaderWriterLock): inter-thread readers writer 28 | locks, optionally downgradable, with various priorities (reader, writer, fair). -------------------------------------------------------------------------------- /fasteners/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. 4 | # 5 | # All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 8 | # not use this file except in compliance with the License. You may obtain 9 | # a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 16 | # License for the specific language governing permissions and limitations 17 | # under the License. 18 | 19 | # promote helpers to this module namespace 20 | 21 | from __future__ import absolute_import 22 | 23 | from fasteners.lock import locked 24 | from fasteners.lock import read_locked 25 | from fasteners.lock import ReaderWriterLock 26 | from fasteners.lock import try_lock 27 | from fasteners.lock import write_locked 28 | from fasteners.process_lock import interprocess_locked 29 | from fasteners.process_lock import interprocess_read_locked 30 | from fasteners.process_lock import interprocess_write_locked 31 | from fasteners.process_lock import InterProcessLock 32 | from fasteners.process_lock import InterProcessReaderWriterLock 33 | 34 | from fasteners.version import _VERSION as __version__ 35 | 36 | __all__ = [ 37 | '__version__', 38 | 'locked', 39 | 'read_locked', 40 | 'ReaderWriterLock', 41 | 'try_lock', 42 | 'write_locked', 43 | 'interprocess_locked', 44 | 'interprocess_read_locked', 45 | 'interprocess_write_locked', 46 | 'InterProcessLock', 47 | 'InterProcessReaderWriterLock', 48 | ] 49 | -------------------------------------------------------------------------------- /fasteners/_utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Copyright (C) 2015 Yahoo! Inc. All Rights Reserved. 4 | # 5 | # All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 8 | # not use this file except in compliance with the License. You may obtain 9 | # a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 16 | # License for the specific language governing permissions and limitations 17 | # under the License. 18 | 19 | import logging 20 | import os 21 | import time 22 | 23 | # log level for low-level debugging 24 | BLATHER = 5 25 | 26 | LOG = logging.getLogger(__name__) 27 | 28 | 29 | def canonicalize_path(path): 30 | """Canonicalizes a potential path. 31 | 32 | Returns a binary string encoded into filesystem encoding. 33 | """ 34 | if isinstance(path, bytes): 35 | return path 36 | if isinstance(path, str): 37 | return os.fsencode(path) 38 | else: 39 | return canonicalize_path(str(path)) 40 | 41 | 42 | def pick_first_not_none(*values): 43 | """Returns first of values that is *not* None (or None if all are/were).""" 44 | for val in values: 45 | if val is not None: 46 | return val 47 | return None 48 | 49 | 50 | class LockStack(object): 51 | """Simple lock stack to get and release many locks. 52 | 53 | An instance of this should **not** be used by many threads at the 54 | same time, as the stack that is maintained will be corrupted and 55 | invalid if that is attempted. 56 | """ 57 | 58 | def __init__(self, logger=None): 59 | self._stack = [] 60 | self._logger = pick_first_not_none(logger, LOG) 61 | 62 | def acquire_lock(self, lock): 63 | gotten = lock.acquire() 64 | if gotten: 65 | self._stack.append(lock) 66 | return gotten 67 | 68 | def __enter__(self): 69 | return self 70 | 71 | def __exit__(self, exc_type, exc_value, exc_tb): 72 | am_left = len(self._stack) 73 | tot_am = am_left 74 | while self._stack: 75 | lock = self._stack.pop() 76 | try: 77 | lock.release() 78 | except Exception: 79 | self._logger.exception("Failed releasing lock %s from lock" 80 | " stack with %s locks", am_left, tot_am) 81 | am_left -= 1 82 | 83 | 84 | class RetryAgain(Exception): 85 | """Exception to signal to retry helper to try again.""" 86 | 87 | 88 | class Retry(object): 89 | """A little retry helper object.""" 90 | 91 | def __init__(self, delay, max_delay, 92 | sleep_func=time.sleep, watch=None): 93 | self.delay = delay 94 | self.attempts = 0 95 | self.max_delay = max_delay 96 | self.sleep_func = sleep_func 97 | self.watch = watch 98 | 99 | def __call__(self, fn, *args, **kwargs): 100 | while True: 101 | self.attempts += 1 102 | try: 103 | return fn(*args, **kwargs) 104 | except RetryAgain: 105 | maybe_delay = self.attempts * self.delay 106 | if maybe_delay < self.max_delay: 107 | actual_delay = maybe_delay 108 | else: 109 | actual_delay = self.max_delay 110 | actual_delay = max(0.0, actual_delay) 111 | if self.watch is not None: 112 | leftover = self.watch.leftover() 113 | if leftover is not None and leftover < actual_delay: 114 | actual_delay = leftover 115 | self.sleep_func(actual_delay) 116 | 117 | 118 | class StopWatch(object): 119 | """A really basic stop watch.""" 120 | 121 | def __init__(self, duration=None): 122 | self.duration = duration 123 | self.started_at = None 124 | self.stopped_at = None 125 | 126 | def leftover(self): 127 | if self.duration is None: 128 | return None 129 | return max(0.0, self.duration - self.elapsed()) 130 | 131 | def elapsed(self): 132 | if self.stopped_at is not None: 133 | end_time = self.stopped_at 134 | else: 135 | end_time = time.monotonic() 136 | return max(0.0, end_time - self.started_at) 137 | 138 | def __enter__(self): 139 | self.start() 140 | return self 141 | 142 | def __exit__(self, exc_type, exc_value, exc_tb): 143 | self.stopped_at = time.monotonic() 144 | 145 | def start(self): 146 | self.started_at = time.monotonic() 147 | self.stopped_at = None 148 | 149 | def expired(self): 150 | if self.duration is None: 151 | return False 152 | else: 153 | return self.elapsed() > self.duration 154 | -------------------------------------------------------------------------------- /fasteners/lock.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. 4 | # Copyright 2011 OpenStack Foundation. 5 | # 6 | # All Rights Reserved. 7 | # 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 9 | # not use this file except in compliance with the License. You may obtain 10 | # a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | # License for the specific language governing permissions and limitations 18 | # under the License. 19 | 20 | import collections 21 | import contextlib 22 | import functools 23 | import threading 24 | from typing import Optional 25 | 26 | from fasteners import _utils 27 | 28 | 29 | class ReaderWriterLock(object): 30 | """An inter-thread readers writer lock.""" 31 | 32 | WRITER = 'w' #: Writer owner type/string constant. 33 | READER = 'r' #: Reader owner type/string constant. 34 | 35 | def __init__(self, 36 | condition_cls=threading.Condition, 37 | current_thread_functor=threading.current_thread): 38 | """ 39 | Args: 40 | condition_cls: 41 | Optional custom `Condition` primitive used for synchronization. 42 | current_thread_functor: 43 | Optional function that returns the identity of the thread in case 44 | threads are not properly identified by threading.current_thread 45 | """ 46 | self._writer = None 47 | self._writer_entries = 0 48 | self._pending_writers = collections.deque() 49 | self._readers = {} 50 | self._cond = condition_cls() 51 | self._current_thread = current_thread_functor 52 | 53 | @property 54 | def has_pending_writers(self) -> bool: 55 | """Check if there pending writers 56 | 57 | Returns: 58 | Whether there are pending writers. 59 | """ 60 | return bool(self._pending_writers) 61 | 62 | def is_writer(self, check_pending: bool = True) -> bool: 63 | """Check if caller is a writer (optionally pending writer). 64 | 65 | Args: 66 | check_pending: 67 | Whether to check for pending writer status. 68 | 69 | Returns: 70 | Whether the caller is the active (or optionally pending) writer. 71 | """ 72 | me = self._current_thread() 73 | if self._writer == me: 74 | return True 75 | if check_pending: 76 | return me in self._pending_writers 77 | else: 78 | return False 79 | 80 | def is_reader(self) -> bool: 81 | """Check if caller is a reader. 82 | 83 | Returns: 84 | Whether the caller is an active reader. 85 | """ 86 | me = self._current_thread() 87 | return me in self._readers 88 | 89 | @property 90 | def owner(self) -> Optional[str]: 91 | """Caller ownership (if any) of the lock 92 | 93 | Returns: 94 | `'w'` if caller is a writer, `'r'` if caller is a reader, None otherwise. 95 | """ 96 | """Returns whether the lock is locked by a writer or reader.""" 97 | if self._writer is not None: 98 | return self.WRITER 99 | if self._readers: 100 | return self.READER 101 | return None 102 | 103 | def acquire_read_lock(self): 104 | """Acquire a read lock. 105 | 106 | Will wait until no active or pending writers. 107 | 108 | Raises: 109 | RuntimeError: if a pending writer tries to acquire a read lock. 110 | """ 111 | me = self._current_thread() 112 | self._acquire_read_lock(me) 113 | 114 | def release_read_lock(self): 115 | """Release a read lock. 116 | 117 | Raises: 118 | RuntimeError: if the current thread does not own a read lock. 119 | """ 120 | me = self._current_thread() 121 | self._release_read_lock(me) 122 | 123 | def _acquire_read_lock(self, me): 124 | if me in self._pending_writers: 125 | raise RuntimeError("Writer %s can not acquire a read lock" 126 | " while waiting for the write lock" 127 | % me) 128 | with self._cond: 129 | while True: 130 | # No active writer, or we are the writer; 131 | # Also no pending writers; 132 | # we are good to become a reader. 133 | if self._writer is None or self._writer == me: 134 | if me in self._readers: 135 | # ok to get a lock if current thread already has one 136 | self._readers[me] = self._readers[me] + 1 137 | break 138 | elif (self._writer == me) or not self.has_pending_writers: 139 | self._readers[me] = 1 140 | break 141 | # An active or pending writer; guess we have to wait. 142 | self._cond.wait() 143 | 144 | def _release_read_lock(self, me, raise_on_not_owned=True): 145 | # I am no longer a reader, remove *one* occurrence of myself. 146 | # If the current thread acquired two read locks, then it will 147 | # still have to remove that other read lock; this allows for 148 | # basic reentrancy to be possible. 149 | with self._cond: 150 | try: 151 | me_instances = self._readers[me] 152 | if me_instances > 1: 153 | self._readers[me] = me_instances - 1 154 | else: 155 | self._readers.pop(me) 156 | except KeyError: 157 | if raise_on_not_owned: 158 | raise RuntimeError(f"Thread {me} does not own a read lock") 159 | self._cond.notify_all() 160 | 161 | @contextlib.contextmanager 162 | def read_lock(self): 163 | """Context manager that grants a read lock. 164 | 165 | Will wait until no active or pending writers. 166 | 167 | Raises: 168 | RuntimeError: if a pending writer tries to acquire a read lock. 169 | """ 170 | me = self._current_thread() 171 | self._acquire_read_lock(me) 172 | try: 173 | yield self 174 | finally: 175 | self._release_read_lock(me, raise_on_not_owned=False) 176 | 177 | def _acquire_write_lock(self, me): 178 | if self.is_reader(): 179 | raise RuntimeError("Reader %s to writer privilege" 180 | " escalation not allowed" % me) 181 | 182 | with self._cond: 183 | self._pending_writers.append(me) 184 | while True: 185 | # No readers, and no active writer, am I next?? 186 | if len(self._readers) == 0 and self._writer is None: 187 | if self._pending_writers[0] == me: 188 | self._writer = self._pending_writers.popleft() 189 | self._writer_entries = 1 190 | break 191 | self._cond.wait() 192 | 193 | def _release_write_lock(self, me, raise_on_not_owned=True): 194 | with self._cond: 195 | self._writer = None 196 | self._writer_entries = 0 197 | self._cond.notify_all() 198 | 199 | def acquire_write_lock(self): 200 | """Acquire a write lock. 201 | 202 | Will wait until no active readers. Blocks readers after acquiring. 203 | 204 | Guaranteed for locks to be processed in fair order (FIFO). 205 | 206 | Raises: 207 | RuntimeError: if an active reader attempts to acquire a lock. 208 | """ 209 | me = self._current_thread() 210 | if self._writer == me: 211 | self._writer_entries += 1 212 | else: 213 | self._acquire_write_lock(me) 214 | 215 | def release_write_lock(self): 216 | """Release a write lock. 217 | 218 | Raises: 219 | RuntimeError: if the current thread does not own a write lock. 220 | """ 221 | me = self._current_thread() 222 | if self._writer == me: 223 | self._writer_entries -= 1 224 | if self._writer_entries == 0: 225 | self._release_write_lock(me) 226 | else: 227 | raise RuntimeError(f"Thread {me} does not own a write lock") 228 | 229 | @contextlib.contextmanager 230 | def write_lock(self): 231 | """Context manager that grants a write lock. 232 | 233 | Will wait until no active readers. Blocks readers after acquiring. 234 | 235 | Guaranteed for locks to be processed in fair order (FIFO). 236 | 237 | Raises: 238 | RuntimeError: if an active reader attempts to acquire a lock. 239 | """ 240 | me = self._current_thread() 241 | if self.is_writer(check_pending=False): 242 | self._writer_entries += 1 243 | try: 244 | yield self 245 | finally: 246 | self._writer_entries -= 1 247 | else: 248 | self._acquire_write_lock(me) 249 | try: 250 | yield self 251 | finally: 252 | self._release_write_lock(me) 253 | 254 | 255 | def locked(*args, **kwargs): 256 | """A locking **method** decorator. 257 | 258 | It will look for a provided attribute (typically a lock or a list 259 | of locks) on the first argument of the function decorated (typically this 260 | is the 'self' object) and before executing the decorated function it 261 | activates the given lock or list of locks as a context manager, 262 | automatically releasing that lock on exit. 263 | 264 | NOTE(harlowja): if no attribute name is provided then by default the 265 | attribute named '_lock' is looked for (this attribute is expected to be 266 | the lock/list of locks object/s) in the instance object this decorator 267 | is attached to. 268 | 269 | NOTE(harlowja): a custom logger (which will be used if lock release 270 | failures happen) can be provided by passing a logger instance for keyword 271 | argument ``logger``. 272 | 273 | NOTE(paulius): This function is DEPRECATED and will be kept until the end 274 | of time. It is potentially used by oslo, but too specific to be recommended 275 | for other projects 276 | """ 277 | 278 | def decorator(f): 279 | attr_name = kwargs.get('lock', '_lock') 280 | logger = kwargs.get('logger') 281 | 282 | @functools.wraps(f) 283 | def wrapper(self, *args, **kwargs): 284 | attr_value = getattr(self, attr_name) 285 | if isinstance(attr_value, (tuple, list)): 286 | with _utils.LockStack(logger=logger) as stack: 287 | for i, lock in enumerate(attr_value): 288 | if not stack.acquire_lock(lock): 289 | raise threading.ThreadError("Unable to acquire" 290 | " lock %s" % (i + 1)) 291 | return f(self, *args, **kwargs) 292 | else: 293 | lock = attr_value 294 | with lock: 295 | return f(self, *args, **kwargs) 296 | 297 | return wrapper 298 | 299 | # This is needed to handle when the decorator has args or the decorator 300 | # doesn't have args, python is rather weird here... 301 | if kwargs or not args: 302 | return decorator 303 | else: 304 | if len(args) == 1: 305 | return decorator(args[0]) 306 | else: 307 | return decorator 308 | 309 | 310 | def read_locked(*args, **kwargs): 311 | """Acquires & releases a read lock around call into decorated method. 312 | 313 | NOTE(harlowja): if no attribute name is provided then by default the 314 | attribute named '_lock' is looked for (this attribute is expected to be 315 | a :py:class:`.ReaderWriterLock`) in the instance object this decorator 316 | is attached to. 317 | 318 | NOTE(paulius): This function is DEPRECATED and will be kept until the end 319 | of time. It is potentially used by oslo, but too specific to be recommended 320 | for other projects 321 | """ 322 | 323 | def decorator(f): 324 | attr_name = kwargs.get('lock', '_lock') 325 | 326 | @functools.wraps(f) 327 | def wrapper(self, *args, **kwargs): 328 | rw_lock = getattr(self, attr_name) 329 | with rw_lock.read_lock(): 330 | return f(self, *args, **kwargs) 331 | 332 | return wrapper 333 | 334 | # This is needed to handle when the decorator has args or the decorator 335 | # doesn't have args, python is rather weird here... 336 | if kwargs or not args: 337 | return decorator 338 | else: 339 | if len(args) == 1: 340 | return decorator(args[0]) 341 | else: 342 | return decorator 343 | 344 | 345 | def write_locked(*args, **kwargs): 346 | """Acquires & releases a write lock around call into decorated method. 347 | 348 | NOTE(harlowja): if no attribute name is provided then by default the 349 | attribute named '_lock' is looked for (this attribute is expected to be 350 | a :py:class:`.ReaderWriterLock` object) in the instance object this 351 | decorator is attached to. 352 | 353 | NOTE(paulius): This function is DEPRECATED and will be kept until the end 354 | of time. It is potentially used by oslo, but too specific to be recommended 355 | for other projects 356 | """ 357 | 358 | def decorator(f): 359 | attr_name = kwargs.get('lock', '_lock') 360 | 361 | @functools.wraps(f) 362 | def wrapper(self, *args, **kwargs): 363 | rw_lock = getattr(self, attr_name) 364 | with rw_lock.write_lock(): 365 | return f(self, *args, **kwargs) 366 | 367 | return wrapper 368 | 369 | # This is needed to handle when the decorator has args or the decorator 370 | # doesn't have args, python is rather weird here... 371 | if kwargs or not args: 372 | return decorator 373 | else: 374 | if len(args) == 1: 375 | return decorator(args[0]) 376 | else: 377 | return decorator 378 | 379 | 380 | @contextlib.contextmanager 381 | def try_lock(lock: threading.Lock) -> bool: 382 | """Context manager that attempts to acquire a lock without a timeout, and 383 | releases it on exit (if acquired). 384 | 385 | Args: 386 | lock: 387 | A lock to try to acquire. 388 | 389 | Returns: 390 | Whether the lock was acquired. 391 | 392 | # NOTE(harlowja): the keyword argument for 'blocking' does not work 393 | # in py2.x and only is fixed in py3.x (this adjustment is documented 394 | # and/or debated in http://bugs.python.org/issue10789); so we'll just 395 | # stick to the format that works in both (oddly the keyword argument 396 | # works in py2.x but only with reentrant locks). 397 | 398 | NOTE(paulius): This function is DEPRECATED and will be kept until the end 399 | of time. It is potentially used by oslo, but too specific to be recommended 400 | for other projects 401 | """ 402 | was_locked = lock.acquire(False) 403 | try: 404 | yield was_locked 405 | finally: 406 | if was_locked: 407 | lock.release() 408 | -------------------------------------------------------------------------------- /fasteners/process_lock.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Copyright 2011 OpenStack Foundation. 4 | # All Rights Reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 7 | # not use this file except in compliance with the License. You may obtain 8 | # a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 15 | # License for the specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from contextlib import contextmanager 19 | import errno 20 | import functools 21 | import logging 22 | import os 23 | from pathlib import Path 24 | import threading 25 | import time 26 | from typing import Callable 27 | from typing import Optional 28 | from typing import Union 29 | 30 | from fasteners import _utils 31 | from fasteners.process_mechanism import _interprocess_mechanism 32 | from fasteners.process_mechanism import _interprocess_reader_writer_mechanism 33 | 34 | LOG = logging.getLogger(__name__) 35 | 36 | 37 | def _ensure_tree(path): 38 | """Create a directory (and any ancestor directories required). 39 | 40 | :param path: Directory to create 41 | """ 42 | try: 43 | os.makedirs(path) 44 | except OSError as e: 45 | if e.errno == errno.EEXIST: 46 | if not os.path.isdir(path): 47 | raise 48 | else: 49 | return False 50 | elif e.errno == errno.EISDIR: 51 | return False 52 | else: 53 | raise 54 | else: 55 | return True 56 | 57 | 58 | class InterProcessLock: 59 | """An interprocess lock.""" 60 | 61 | MAX_DELAY = 0.1 # For backwards compatibility 62 | DELAY_INCREMENT = 0.01 # For backwards compatibility 63 | 64 | def __init__(self, 65 | path: Union[Path, str], 66 | sleep_func: Callable[[float], None] = time.sleep, 67 | logger: Optional[logging.Logger] = None): 68 | """ 69 | args: 70 | path: 71 | Path to the file that will be used for locking. 72 | sleep_func: 73 | Optional function to use for sleeping. 74 | logger: 75 | Optional logger to use for logging. 76 | """ 77 | self.lockfile = None 78 | self.path = _utils.canonicalize_path(path) 79 | self.acquired = False 80 | self.sleep_func = sleep_func 81 | self.logger = _utils.pick_first_not_none(logger, LOG) 82 | 83 | def _try_acquire(self, blocking, watch): 84 | try: 85 | self.trylock() 86 | except IOError as e: 87 | if e.errno in (errno.EACCES, errno.EAGAIN): 88 | if not blocking or watch.expired(): 89 | return False 90 | else: 91 | raise _utils.RetryAgain() 92 | else: 93 | raise threading.ThreadError("Unable to acquire lock on" 94 | " `%(path)s` due to" 95 | " %(exception)s" % 96 | { 97 | 'path': self.path, 98 | 'exception': e, 99 | }) 100 | else: 101 | return True 102 | 103 | def _do_open(self): 104 | basedir = os.path.dirname(self.path) 105 | if basedir: 106 | made_basedir = _ensure_tree(basedir) 107 | if made_basedir: 108 | self.logger.log(_utils.BLATHER, 109 | 'Created lock base path `%s`', basedir) 110 | # Open in append mode so we don't overwrite any potential contents of 111 | # the target file. This eliminates the possibility of an attacker 112 | # creating a symlink to an important file in our lock path. 113 | if self.lockfile is None or self.lockfile.closed: 114 | self.lockfile = open(self.path, 'a') 115 | 116 | def acquire(self, 117 | blocking: bool = True, 118 | delay: float = 0.01, 119 | max_delay: float = 0.1, 120 | timeout: Optional[float] = None) -> bool: 121 | """Attempt to acquire the lock. 122 | 123 | Args: 124 | blocking: 125 | Whether to wait to try to acquire the lock. 126 | delay: 127 | When `blocking`, starting delay as well as the delay increment 128 | (in seconds). 129 | max_delay: 130 | When `blocking` the maximum delay in between attempts to 131 | acquire (in seconds). 132 | timeout: 133 | When `blocking`, maximal waiting time (in seconds). 134 | 135 | Returns: 136 | whether or not the acquisition succeeded 137 | """ 138 | if delay < 0: 139 | raise ValueError("Delay must be greater than or equal to zero") 140 | if timeout is not None and timeout < 0: 141 | raise ValueError("Timeout must be greater than or equal to zero") 142 | if delay >= max_delay: 143 | max_delay = delay 144 | self._do_open() 145 | watch = _utils.StopWatch(duration=timeout) 146 | r = _utils.Retry(delay, max_delay, 147 | sleep_func=self.sleep_func, watch=watch) 148 | with watch: 149 | gotten = r(self._try_acquire, blocking, watch) 150 | if not gotten: 151 | return False 152 | else: 153 | self.acquired = True 154 | self.logger.log(_utils.BLATHER, 155 | "Acquired file lock `%s` after waiting %0.3fs [%s" 156 | " attempts were required]", self.path, 157 | watch.elapsed(), r.attempts) 158 | return True 159 | 160 | def _do_close(self): 161 | if self.lockfile is not None: 162 | self.lockfile.close() 163 | self.lockfile = None 164 | 165 | def __enter__(self): 166 | gotten = self.acquire() 167 | if not gotten: 168 | # This shouldn't happen, but just in case... 169 | raise threading.ThreadError("Unable to acquire a file lock" 170 | " on `%s` (when used as a" 171 | " context manager)" % self.path) 172 | return self 173 | 174 | def release(self): 175 | """Release the previously acquired lock.""" 176 | if not self.acquired: 177 | raise threading.ThreadError("Unable to release an unaquired lock") 178 | try: 179 | self.unlock() 180 | except Exception as e: 181 | msg = "Could not unlock the acquired lock opened on `%s`", self.path 182 | self.logger.exception(msg) 183 | raise threading.ThreadError(msg) from e 184 | else: 185 | self.acquired = False 186 | try: 187 | self._do_close() 188 | except IOError: 189 | self.logger.exception("Could not close the file handle" 190 | " opened on `%s`", self.path) 191 | else: 192 | self.logger.log(_utils.BLATHER, 193 | "Unlocked and closed file lock open on" 194 | " `%s`", self.path) 195 | 196 | def __exit__(self, exc_type, exc_val, exc_tb): 197 | self.release() 198 | 199 | def exists(self): 200 | return os.path.exists(self.path) 201 | 202 | def trylock(self): 203 | _interprocess_mechanism.trylock(self.lockfile) 204 | 205 | def unlock(self): 206 | _interprocess_mechanism.unlock(self.lockfile) 207 | 208 | 209 | class InterProcessReaderWriterLock: 210 | """An interprocess readers writer lock.""" 211 | 212 | MAX_DELAY = 0.1 # for backwards compatibility 213 | DELAY_INCREMENT = 0.01 # for backwards compatibility 214 | 215 | def __init__(self, 216 | path: Union[Path, str], 217 | sleep_func: Callable[[float], None] = time.sleep, 218 | logger: Optional[logging.Logger] = None): 219 | """ 220 | Args: 221 | path: 222 | Path to the file that will be used for locking. 223 | sleep_func: 224 | Optional function to use for sleeping. 225 | logger: 226 | Optional logger to use for logging. 227 | """ 228 | self.lockfile = None 229 | self.path = _utils.canonicalize_path(path) 230 | self.sleep_func = sleep_func 231 | self.logger = _utils.pick_first_not_none(logger, LOG) 232 | 233 | @contextmanager 234 | def read_lock(self, delay=0.01, max_delay=0.1): 235 | """Context manager that grans a read lock""" 236 | 237 | self.acquire_read_lock(blocking=True, delay=delay, 238 | max_delay=max_delay, timeout=None) 239 | try: 240 | yield 241 | finally: 242 | self.release_read_lock() 243 | 244 | @contextmanager 245 | def write_lock(self, delay=0.01, max_delay=0.1): 246 | """Context manager that grans a write lock""" 247 | 248 | gotten = self.acquire_write_lock(blocking=True, delay=delay, 249 | max_delay=max_delay, timeout=None) 250 | 251 | if not gotten: 252 | # This shouldn't happen, but just in case... 253 | raise threading.ThreadError("Unable to acquire a file lock" 254 | " on `%s` (when used as a" 255 | " context manager)" % self.path) 256 | try: 257 | yield 258 | finally: 259 | self.release_write_lock() 260 | 261 | def _try_acquire(self, blocking, watch, exclusive): 262 | try: 263 | gotten = _interprocess_reader_writer_mechanism.trylock(self.lockfile, exclusive) 264 | except Exception as e: 265 | raise threading.ThreadError( 266 | "Unable to acquire lock on {} due to {}!".format(self.path, e)) 267 | 268 | if gotten: 269 | return True 270 | 271 | if not blocking or watch.expired(): 272 | return False 273 | 274 | raise _utils.RetryAgain() 275 | 276 | def _do_open(self): 277 | basedir = os.path.dirname(self.path) 278 | if basedir: 279 | made_basedir = _ensure_tree(basedir) 280 | if made_basedir: 281 | self.logger.log(_utils.BLATHER, 282 | 'Created lock base path `%s`', basedir) 283 | if self.lockfile is None: 284 | self.lockfile = _interprocess_reader_writer_mechanism.get_handle(self.path) 285 | 286 | def acquire_read_lock(self, 287 | blocking: bool = True, 288 | delay: float = 0.01, 289 | max_delay: float = 0.1, 290 | timeout: float = None) -> bool: 291 | """Attempt to acquire a reader's lock. 292 | 293 | Args: 294 | blocking: 295 | Whether to wait to try to acquire the lock. 296 | delay: 297 | When `blocking`, starting delay as well as the delay increment 298 | (in seconds). 299 | max_delay: 300 | When `blocking` the maximum delay in between attempts to 301 | acquire (in seconds). 302 | timeout: 303 | When `blocking`, maximal waiting time (in seconds). 304 | 305 | Returns: 306 | whether or not the acquisition succeeded 307 | """ 308 | return self._acquire(blocking, delay, max_delay, timeout, exclusive=False) 309 | 310 | def acquire_write_lock(self, 311 | blocking: bool = True, 312 | delay: float = 0.01, 313 | max_delay: float = 0.1, 314 | timeout: float = None) -> bool: 315 | """Attempt to acquire a writer's lock. 316 | 317 | Args: 318 | blocking: 319 | Whether to wait to try to acquire the lock. 320 | delay: 321 | When `blocking`, starting delay as well as the delay increment 322 | (in seconds). 323 | max_delay: 324 | When `blocking` the maximum delay in between attempts to 325 | acquire (in seconds). 326 | timeout: 327 | When `blocking`, maximal waiting time (in seconds). 328 | 329 | Returns: 330 | whether or not the acquisition succeeded 331 | """ 332 | return self._acquire(blocking, delay, max_delay, timeout, exclusive=True) 333 | 334 | def _acquire(self, blocking=True, 335 | delay=0.01, max_delay=0.1, 336 | timeout=None, exclusive=True): 337 | 338 | if delay < 0: 339 | raise ValueError("Delay must be greater than or equal to zero") 340 | if timeout is not None and timeout < 0: 341 | raise ValueError("Timeout must be greater than or equal to zero") 342 | if delay >= max_delay: 343 | max_delay = delay 344 | self._do_open() 345 | watch = _utils.StopWatch(duration=timeout) 346 | r = _utils.Retry(delay, max_delay, 347 | sleep_func=self.sleep_func, watch=watch) 348 | with watch: 349 | gotten = r(self._try_acquire, blocking, watch, exclusive) 350 | if not gotten: 351 | return False 352 | else: 353 | self.logger.log(_utils.BLATHER, 354 | "Acquired file lock `%s` after waiting %0.3fs [%s" 355 | " attempts were required]", self.path, 356 | watch.elapsed(), r.attempts) 357 | return True 358 | 359 | def _do_close(self): 360 | if self.lockfile is not None: 361 | _interprocess_reader_writer_mechanism.close_handle(self.lockfile) 362 | self.lockfile = None 363 | 364 | def release_write_lock(self): 365 | """Release the writer's lock.""" 366 | try: 367 | _interprocess_reader_writer_mechanism.unlock(self.lockfile) 368 | except IOError: 369 | self.logger.exception("Could not unlock the acquired lock opened" 370 | " on `%s`", self.path) 371 | else: 372 | try: 373 | self._do_close() 374 | except IOError: 375 | self.logger.exception("Could not close the file handle" 376 | " opened on `%s`", self.path) 377 | else: 378 | self.logger.log(_utils.BLATHER, 379 | "Unlocked and closed file lock open on" 380 | " `%s`", self.path) 381 | 382 | def release_read_lock(self): 383 | """Release the reader's lock.""" 384 | try: 385 | _interprocess_reader_writer_mechanism.unlock(self.lockfile) 386 | except IOError: 387 | self.logger.exception("Could not unlock the acquired lock opened" 388 | " on `%s`", self.path) 389 | else: 390 | try: 391 | self._do_close() 392 | except IOError: 393 | self.logger.exception("Could not close the file handle" 394 | " opened on `%s`", self.path) 395 | else: 396 | self.logger.log(_utils.BLATHER, 397 | "Unlocked and closed file lock open on" 398 | " `%s`", self.path) 399 | 400 | 401 | def interprocess_write_locked(path: Union[Path, str]): 402 | """Acquires & releases an interprocess **write** lock around the call into 403 | the decorated function 404 | 405 | Args: 406 | path: Path to the file used for locking. 407 | """ 408 | lock = InterProcessReaderWriterLock(path) 409 | 410 | def decorator(f): 411 | @functools.wraps(f) 412 | def wrapper(*args, **kwargs): 413 | with lock.write_lock(): 414 | return f(*args, **kwargs) 415 | 416 | return wrapper 417 | 418 | return decorator 419 | 420 | 421 | def interprocess_read_locked(path: Union[Path, str]): 422 | """Acquires & releases an interprocess **read** lock around the call into 423 | the decorated function 424 | 425 | Args: 426 | path: Path to the file used for locking. 427 | """ 428 | lock = InterProcessReaderWriterLock(path) 429 | 430 | def decorator(f): 431 | @functools.wraps(f) 432 | def wrapper(*args, **kwargs): 433 | with lock.read_lock(): 434 | return f(*args, **kwargs) 435 | 436 | return wrapper 437 | 438 | return decorator 439 | 440 | 441 | def interprocess_locked(path: Union[Path, str]): 442 | """Acquires & releases an interprocess lock around the call to the 443 | decorated function. 444 | 445 | Args: 446 | path: Path to the file used for locking. 447 | """ 448 | lock = InterProcessLock(path) 449 | 450 | def decorator(f): 451 | @functools.wraps(f) 452 | def wrapper(*args, **kwargs): 453 | with lock: 454 | return f(*args, **kwargs) 455 | 456 | return wrapper 457 | 458 | return decorator 459 | -------------------------------------------------------------------------------- /fasteners/process_mechanism.py: -------------------------------------------------------------------------------- 1 | from abc import ABC 2 | from abc import abstractmethod 3 | import errno 4 | import os 5 | 6 | 7 | class _InterProcessReaderWriterLockMechanism(ABC): 8 | 9 | @staticmethod 10 | @abstractmethod 11 | def trylock(lockfile, exclusive): 12 | ... 13 | 14 | @staticmethod 15 | @abstractmethod 16 | def unlock(lockfile): 17 | ... 18 | 19 | @staticmethod 20 | @abstractmethod 21 | def get_handle(path): 22 | ... 23 | 24 | @staticmethod 25 | @abstractmethod 26 | def close_handle(lockfile): 27 | ... 28 | 29 | 30 | class _InterProcessMechanism(ABC): 31 | @staticmethod 32 | @abstractmethod 33 | def trylock(lockfile): 34 | ... 35 | 36 | @staticmethod 37 | @abstractmethod 38 | def unlock(lockfile): 39 | ... 40 | 41 | 42 | class _WindowsInterProcessMechanism(_InterProcessMechanism): 43 | """Interprocess lock implementation that works on windows systems.""" 44 | 45 | @staticmethod 46 | def trylock(lockfile): 47 | fileno = lockfile.fileno() 48 | msvcrt.locking(fileno, msvcrt.LK_NBLCK, 1) 49 | 50 | @staticmethod 51 | def unlock(lockfile): 52 | fileno = lockfile.fileno() 53 | msvcrt.locking(fileno, msvcrt.LK_UNLCK, 1) 54 | 55 | 56 | class _FcntlInterProcessMechanism(_InterProcessMechanism): 57 | """Interprocess lock implementation that works on posix systems.""" 58 | 59 | @staticmethod 60 | def trylock(lockfile): 61 | fcntl.lockf(lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB) 62 | 63 | @staticmethod 64 | def unlock(lockfile): 65 | fcntl.lockf(lockfile, fcntl.LOCK_UN) 66 | 67 | 68 | class _WindowsInterProcessReaderWriterLockMechanism(_InterProcessReaderWriterLockMechanism): 69 | """Interprocess readers writer lock implementation that works on windows 70 | systems.""" 71 | 72 | @staticmethod 73 | def trylock(lockfile, exclusive): 74 | 75 | if exclusive: 76 | flags = win32con.LOCKFILE_FAIL_IMMEDIATELY | win32con.LOCKFILE_EXCLUSIVE_LOCK 77 | else: 78 | flags = win32con.LOCKFILE_FAIL_IMMEDIATELY 79 | 80 | handle = msvcrt.get_osfhandle(lockfile.fileno()) 81 | ok = win32file.LockFileEx(handle, flags, 0, 1, 0, win32file.pointer(pywintypes.OVERLAPPED())) 82 | if ok: 83 | return True 84 | else: 85 | last_error = win32file.GetLastError() 86 | if last_error == win32file.ERROR_LOCK_VIOLATION: 87 | return False 88 | else: 89 | raise OSError(last_error) 90 | 91 | @staticmethod 92 | def unlock(lockfile): 93 | handle = msvcrt.get_osfhandle(lockfile.fileno()) 94 | ok = win32file.UnlockFileEx(handle, 0, 1, 0, win32file.pointer(pywintypes.OVERLAPPED())) 95 | if not ok: 96 | raise OSError(win32file.GetLastError()) 97 | 98 | @staticmethod 99 | def get_handle(path): 100 | return open(path, 'a+') 101 | 102 | @staticmethod 103 | def close_handle(lockfile): 104 | lockfile.close() 105 | 106 | 107 | class _FcntlInterProcessReaderWriterLockMechanism(_InterProcessReaderWriterLockMechanism): 108 | """Interprocess readers writer lock implementation that works on posix 109 | systems.""" 110 | 111 | @staticmethod 112 | def trylock(lockfile, exclusive): 113 | 114 | if exclusive: 115 | flags = fcntl.LOCK_EX | fcntl.LOCK_NB 116 | else: 117 | flags = fcntl.LOCK_SH | fcntl.LOCK_NB 118 | 119 | try: 120 | fcntl.lockf(lockfile, flags) 121 | return True 122 | except (IOError, OSError) as e: 123 | if e.errno in (errno.EACCES, errno.EAGAIN): 124 | return False 125 | else: 126 | raise e 127 | 128 | @staticmethod 129 | def unlock(lockfile): 130 | fcntl.lockf(lockfile, fcntl.LOCK_UN) 131 | 132 | @staticmethod 133 | def get_handle(path): 134 | return open(path, 'a+') 135 | 136 | @staticmethod 137 | def close_handle(lockfile): 138 | lockfile.close() 139 | 140 | 141 | if os.name == 'nt': 142 | import msvcrt 143 | import fasteners.pywin32.pywintypes as pywintypes 144 | import fasteners.pywin32.win32con as win32con 145 | import fasteners.pywin32.win32file as win32file 146 | 147 | _interprocess_reader_writer_mechanism = _WindowsInterProcessReaderWriterLockMechanism() 148 | _interprocess_mechanism = _WindowsInterProcessMechanism() 149 | 150 | else: 151 | import fcntl 152 | 153 | _interprocess_reader_writer_mechanism = _FcntlInterProcessReaderWriterLockMechanism() 154 | _interprocess_mechanism = _FcntlInterProcessMechanism() 155 | -------------------------------------------------------------------------------- /fasteners/pywin32/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Exposes the minimal amount of code to use Win32 native file locking. We only 3 | need two APIs, so this is far lighter weight than pulling in all of pywin32. 4 | """ 5 | 6 | -------------------------------------------------------------------------------- /fasteners/pywin32/pywintypes.py: -------------------------------------------------------------------------------- 1 | from ctypes import c_void_p 2 | from ctypes import Structure 3 | from ctypes import Union 4 | from ctypes.wintypes import DWORD 5 | from ctypes.wintypes import HANDLE 6 | 7 | # Definitions for OVERLAPPED. 8 | # Refer: https://docs.microsoft.com/en-us/windows/win32/api/minwinbase/ns-minwinbase-overlapped 9 | 10 | 11 | class _DummyStruct(Structure): 12 | _fields_ = [ 13 | ('Offset', DWORD), 14 | ('OffsetHigh', DWORD), 15 | ] 16 | 17 | 18 | class _DummyUnion(Union): 19 | _fields_ = [ 20 | ('_offsets', _DummyStruct), 21 | ('Pointer', c_void_p), 22 | ] 23 | 24 | 25 | class OVERLAPPED(Structure): 26 | _fields_ = [ 27 | ('Internal', c_void_p), 28 | ('InternalHigh', c_void_p), 29 | ('_offset_or_ptr', _DummyUnion), 30 | ('hEvent', HANDLE), 31 | ] 32 | -------------------------------------------------------------------------------- /fasteners/pywin32/win32con.py: -------------------------------------------------------------------------------- 1 | LOCKFILE_EXCLUSIVE_LOCK = 0x02 2 | LOCKFILE_FAIL_IMMEDIATELY = 0x01 3 | -------------------------------------------------------------------------------- /fasteners/pywin32/win32file.py: -------------------------------------------------------------------------------- 1 | from ctypes import POINTER 2 | from ctypes import pointer 3 | from ctypes import WinDLL 4 | from ctypes.wintypes import BOOL 5 | from ctypes.wintypes import DWORD 6 | from ctypes.wintypes import HANDLE 7 | 8 | from fasteners.pywin32.pywintypes import OVERLAPPED 9 | 10 | kernel32 = WinDLL('kernel32', use_last_error=True) 11 | _ = pointer 12 | 13 | # Refer: https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-lockfileex 14 | LockFileEx = kernel32.LockFileEx 15 | LockFileEx.argtypes = [ 16 | HANDLE, 17 | DWORD, 18 | DWORD, 19 | DWORD, 20 | DWORD, 21 | POINTER(OVERLAPPED), 22 | ] 23 | LockFileEx.restype = BOOL 24 | 25 | # Refer: https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-unlockfile 26 | UnlockFileEx = kernel32.UnlockFileEx 27 | UnlockFileEx.argtypes = [ 28 | HANDLE, 29 | DWORD, 30 | DWORD, 31 | DWORD, 32 | POINTER(OVERLAPPED), 33 | ] 34 | UnlockFileEx.restype = BOOL 35 | 36 | # Errors/flags 37 | GetLastError = kernel32.GetLastError 38 | 39 | ERROR_LOCK_VIOLATION = 33 40 | -------------------------------------------------------------------------------- /fasteners/version.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. 4 | # Copyright 2011 OpenStack Foundation. 5 | # 6 | # All Rights Reserved. 7 | # 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 9 | # not use this file except in compliance with the License. You may obtain 10 | # a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | # License for the specific language governing permissions and limitations 18 | # under the License. 19 | 20 | _VERSION = "0.19" 21 | 22 | 23 | def version_string(): 24 | return _VERSION 25 | -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: Fasteners 2 | site_url: https://fasteners.readthedocs.io/ 3 | repo_url: https://github.com/harlowja/fasteners 4 | edit_uri: '' 5 | 6 | nav: 7 | - Home: index.md 8 | - User Guide: 9 | - guide/index.md 10 | - Process locks: guide/inter_process.md 11 | - Thread locks: guide/inter_thread.md 12 | - Glossary: guide/glossary.md 13 | - Reference: 14 | - Process locks: api/inter_process.md 15 | - Thread locks: api/inter_thread.md 16 | - Changelog: CHANGELOG.md 17 | 18 | theme: 19 | name: material 20 | features: 21 | - navigation.indexes 22 | - navigation.instant 23 | - navigation.tracking 24 | logo: img/favicon-192x192.png 25 | favicon: img/favicon-32x32.png 26 | 27 | plugins: 28 | - search 29 | - mkdocstrings: 30 | default_handler: python 31 | handlers: 32 | python: 33 | rendering: 34 | show_category_heading: true 35 | show_source: false 36 | members_order: source 37 | show_if_no_docstring: false 38 | show_root_full_path: false 39 | show_root_heading: true 40 | watch: 41 | - fasteners 42 | 43 | markdown_extensions: 44 | - pymdownx.highlight: 45 | anchor_linenums: true 46 | - pymdownx.inlinehilite 47 | - pymdownx.snippets 48 | - pymdownx.superfences -------------------------------------------------------------------------------- /publish.md: -------------------------------------------------------------------------------- 1 | 1. Update the change log: 2 | 3 | CHANGELOG 4 | 5 | 2. Update the version number: 6 | 7 | setup.cfg 8 | fasteners/version.py 9 | 10 | 4. Make sure that the working directory is clean. 11 | 12 | 13 | 4. Build: 14 | 15 | source venv/bin/activate 16 | python -m build 17 | 18 | 5. Inspect the packages manually. 19 | 20 | 21 | 6. Upload: 22 | 23 | twine upload dist/* 24 | 25 | 7. Tag the git repo. 26 | 27 | 8. Read the docs will be updated automatically. 28 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools", "wheel"] 3 | build-backend = "setuptools.build_meta" 4 | -------------------------------------------------------------------------------- /requirements-docs.txt: -------------------------------------------------------------------------------- 1 | mkdocs 2 | mkdocstrings[python] >= 0.18 3 | mkdocs-material 4 | -------------------------------------------------------------------------------- /requirements-pkg.txt: -------------------------------------------------------------------------------- 1 | build 2 | twine 3 | -------------------------------------------------------------------------------- /requirements-test.txt: -------------------------------------------------------------------------------- 1 | diskcache 2 | eventlet 3 | more_itertools 4 | pytest 5 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = fasteners 3 | version = 0.19 4 | url = https://github.com/harlowja/fasteners 5 | 6 | author = Joshua Harlow 7 | maintainer = Paulius Šarka 8 | 9 | description = A python package that provides useful locks 10 | long_description = file:README.md 11 | long_description_content_type = text/markdown; charset-UTF-8 12 | keywords = lock thread process fasteners 13 | 14 | classifiers = 15 | Development Status :: 4 - Beta 16 | Intended Audience :: Developers 17 | License :: OSI Approved :: Apache Software License 18 | Operating System :: POSIX :: Linux 19 | Operating System :: Microsoft :: Windows 20 | Operating System :: MacOS 21 | Programming Language :: Python :: 3 22 | Programming Language :: Python :: 3.6 23 | Programming Language :: Python :: 3.7 24 | Programming Language :: Python :: 3.8 25 | Programming Language :: Python :: 3.9 26 | Programming Language :: Python :: 3.10 27 | Programming Language :: Python :: Implementation :: PyPy 28 | Topic :: Utilities 29 | 30 | license = Apache-2.0 31 | license_files = LICENSE 32 | 33 | [options] 34 | packages = find: 35 | python_requires = >=3.6 36 | 37 | [options.packages.find] 38 | exclude = 39 | tests 40 | tests_eventlet 41 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/harlowja/fasteners/06c3f06cab4e135b8d921932019a231c180eb9f4/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_decorators.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Copyright (C) 2015 Yahoo! Inc. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. You may obtain 7 | # a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | # License for the specific language governing permissions and limitations 15 | # under the License. 16 | 17 | import threading 18 | 19 | import fasteners 20 | 21 | 22 | class Locked(object): 23 | def __init__(self): 24 | self._lock = threading.Lock() 25 | 26 | @fasteners.locked 27 | def assert_i_am_locked(self): 28 | assert self._lock.locked() 29 | 30 | def assert_i_am_not_locked(self): 31 | assert not self._lock.locked() 32 | 33 | 34 | class ManyLocks(object): 35 | def __init__(self, amount): 36 | self._lock = [] 37 | for _i in range(0, amount): 38 | self._lock.append(threading.Lock()) 39 | 40 | @fasteners.locked 41 | def assert_i_am_locked(self): 42 | assert all(lock.locked() for lock in self._lock) 43 | 44 | def assert_i_am_not_locked(self): 45 | assert not any(lock.locked() for lock in self._lock) 46 | 47 | 48 | class RWLocked(object): 49 | def __init__(self): 50 | self._lock = fasteners.ReaderWriterLock() 51 | 52 | @fasteners.read_locked 53 | def assert_i_am_read_locked(self): 54 | assert self._lock.owner == fasteners.ReaderWriterLock.READER 55 | 56 | @fasteners.write_locked 57 | def assert_i_am_write_locked(self): 58 | assert self._lock.owner == fasteners.ReaderWriterLock.WRITER 59 | 60 | def assert_i_am_not_locked(self): 61 | assert self._lock.owner is None 62 | 63 | 64 | def test_locked(): 65 | obj = Locked() 66 | obj.assert_i_am_locked() 67 | obj.assert_i_am_not_locked() 68 | 69 | 70 | def test_many_locked(): 71 | obj = ManyLocks(10) 72 | obj.assert_i_am_locked() 73 | obj.assert_i_am_not_locked() 74 | 75 | 76 | def test_read_write_locked(): 77 | obj = RWLocked() 78 | obj.assert_i_am_write_locked() 79 | obj.assert_i_am_read_locked() 80 | obj.assert_i_am_not_locked() 81 | -------------------------------------------------------------------------------- /tests/test_eventlet.py: -------------------------------------------------------------------------------- 1 | """ 2 | These tests need to run in child processes, otherwise eventlet monkey_patch 3 | conflicts with multiprocessing and other tests fail. 4 | """ 5 | import concurrent.futures 6 | from multiprocessing import get_context 7 | 8 | 9 | def _test_eventlet_spawn_n_bug(): 10 | """Both threads run at the same time thru the lock""" 11 | import eventlet 12 | eventlet.monkey_patch() 13 | from fasteners import ReaderWriterLock 14 | 15 | STARTED = eventlet.event.Event() 16 | FINISHED = eventlet.event.Event() 17 | lock = ReaderWriterLock() 18 | 19 | def other(): 20 | STARTED.send('started') 21 | with lock.write_lock(): 22 | FINISHED.send('finished') 23 | 24 | with lock.write_lock(): 25 | eventlet.spawn_n(other) 26 | STARTED.wait() 27 | assert FINISHED.wait(1) == 'finished' 28 | 29 | 30 | def _test_eventlet_spawn_n_bugfix(): 31 | """Threads wait for each other as they should""" 32 | import eventlet 33 | eventlet.monkey_patch() 34 | from fasteners import ReaderWriterLock 35 | 36 | STARTED = eventlet.event.Event() 37 | FINISHED = eventlet.event.Event() 38 | lock = ReaderWriterLock(current_thread_functor=eventlet.getcurrent) 39 | 40 | def other(): 41 | STARTED.send('started') 42 | with lock.write_lock(): 43 | FINISHED.send('finished') 44 | 45 | with lock.write_lock(): 46 | eventlet.spawn_n(other) 47 | STARTED.wait() 48 | assert FINISHED.wait(1) is None 49 | 50 | assert FINISHED.wait(1) == 'finished' 51 | 52 | 53 | def test_eventlet_spawn_n_bug(): 54 | with concurrent.futures.ProcessPoolExecutor(mp_context=get_context('spawn')) as executor: 55 | f = executor.submit(_test_eventlet_spawn_n_bug) 56 | f.result() 57 | 58 | 59 | def test_eventlet_spawn_n_bugfix(): 60 | with concurrent.futures.ProcessPoolExecutor(mp_context=get_context('spawn')) as executor: 61 | f = executor.submit(_test_eventlet_spawn_n_bugfix) 62 | f.result() 63 | -------------------------------------------------------------------------------- /tests/test_helpers.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Copyright (C) 2015 Yahoo! Inc. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. You may obtain 7 | # a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | # License for the specific language governing permissions and limitations 15 | # under the License. 16 | 17 | import threading 18 | 19 | import fasteners 20 | 21 | 22 | def test_try_lock(): 23 | lock = threading.Lock() 24 | with fasteners.try_lock(lock) as locked_1: 25 | assert locked_1 26 | with fasteners.try_lock(lock) as locked_2: 27 | assert not locked_2 28 | -------------------------------------------------------------------------------- /tests/test_lock.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. You may obtain 7 | # a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | # License for the specific language governing permissions and limitations 15 | # under the License. 16 | 17 | import collections 18 | from concurrent import futures 19 | import random 20 | import threading 21 | import time 22 | 23 | import pytest 24 | 25 | import fasteners 26 | from fasteners import _utils 27 | 28 | # NOTE(harlowja): Sleep a little so now() can not be the same (which will 29 | # cause false positives when our overlap detection code runs). If there are 30 | # real overlaps then they will still exist. 31 | NAPPY_TIME = 0.05 32 | 33 | # We will spend this amount of time doing some "fake" work. 34 | WORK_TIMES = [(0.01 + x / 100.0) for x in range(0, 5)] 35 | 36 | # If latches/events take longer than this to become empty/set, something is 37 | # usually wrong and should be debugged instead of deadlocking... 38 | WAIT_TIMEOUT = 300 39 | 40 | THREAD_COUNT = 20 41 | 42 | 43 | def _find_overlaps(times, start, end): 44 | overlaps = 0 45 | for (s, e) in times: 46 | if s >= start and e <= end: 47 | overlaps += 1 48 | return overlaps 49 | 50 | 51 | def _spawn_variation(readers, writers, max_workers=None): 52 | start_stops = collections.deque() 53 | lock = fasteners.ReaderWriterLock() 54 | 55 | def read_func(ident): 56 | with lock.read_lock(): 57 | enter_time = time.monotonic() 58 | time.sleep(WORK_TIMES[ident % len(WORK_TIMES)]) 59 | exit_time = time.monotonic() 60 | start_stops.append((lock.READER, enter_time, exit_time)) 61 | time.sleep(NAPPY_TIME) 62 | 63 | def write_func(ident): 64 | with lock.write_lock(): 65 | enter_time = time.monotonic() 66 | time.sleep(WORK_TIMES[ident % len(WORK_TIMES)]) 67 | exit_time = time.monotonic() 68 | start_stops.append((lock.WRITER, enter_time, exit_time)) 69 | time.sleep(NAPPY_TIME) 70 | 71 | if max_workers is None: 72 | max_workers = max(0, readers) + max(0, writers) 73 | if max_workers > 0: 74 | with futures.ThreadPoolExecutor(max_workers=max_workers) as e: 75 | count = 0 76 | for _i in range(0, readers): 77 | e.submit(read_func, count) 78 | count += 1 79 | for _i in range(0, writers): 80 | e.submit(write_func, count) 81 | count += 1 82 | 83 | writer_times = [] 84 | reader_times = [] 85 | for (lock_type, start, stop) in list(start_stops): 86 | if lock_type == lock.WRITER: 87 | writer_times.append((start, stop)) 88 | else: 89 | reader_times.append((start, stop)) 90 | return writer_times, reader_times 91 | 92 | 93 | def _daemon_thread(target): 94 | t = threading.Thread(target=target) 95 | t.daemon = True 96 | return t 97 | 98 | 99 | @pytest.mark.parametrize("contextmanager", [True, False]) 100 | def test_no_double_writers(contextmanager): 101 | lock = fasteners.ReaderWriterLock() 102 | watch = _utils.StopWatch(duration=5) 103 | watch.start() 104 | dups = collections.deque() 105 | active = collections.deque() 106 | 107 | def acquire_check_ctx(me): 108 | with lock.write_lock(): 109 | if len(active) >= 1: 110 | dups.append(me) 111 | dups.extend(active) 112 | active.append(me) 113 | time.sleep(random.random() / 100) 114 | active.remove(me) 115 | 116 | def acquire_check_plain(me): 117 | lock.acquire_write_lock() 118 | if len(active) >= 1: 119 | dups.append(me) 120 | dups.extend(active) 121 | active.append(me) 122 | time.sleep(random.random() / 100) 123 | active.remove(me) 124 | lock.release_write_lock() 125 | 126 | def run(): 127 | me = threading.current_thread() 128 | while not watch.expired(): 129 | if contextmanager: 130 | acquire_check_ctx(me) 131 | else: 132 | acquire_check_plain(me) 133 | 134 | threads = [] 135 | for i in range(0, THREAD_COUNT): 136 | t = _daemon_thread(run) 137 | threads.append(t) 138 | t.start() 139 | while threads: 140 | t = threads.pop() 141 | t.join() 142 | 143 | assert not dups 144 | assert not active 145 | 146 | 147 | @pytest.mark.parametrize("contextmanager", [True, False]) 148 | def test_no_concurrent_readers_writers(contextmanager): 149 | lock = fasteners.ReaderWriterLock() 150 | watch = _utils.StopWatch(duration=5) 151 | watch.start() 152 | dups = collections.deque() 153 | active = collections.deque() 154 | 155 | def acquire_check_ctx(me, reader): 156 | if reader: 157 | lock_func = lock.read_lock 158 | else: 159 | lock_func = lock.write_lock 160 | with lock_func(): 161 | if not reader: 162 | # There should be no-one else currently active, if there 163 | # is ensure we capture them so that we can later blow-up 164 | # the test. 165 | if len(active) >= 1: 166 | dups.append(me) 167 | dups.extend(active) 168 | active.append(me) 169 | time.sleep(random.random() / 100) 170 | active.remove(me) 171 | 172 | def acquire_check_plain(me, reader): 173 | if reader: 174 | lock_func, unlock_func = lock.acquire_read_lock, lock.release_read_lock 175 | else: 176 | lock_func, unlock_func = lock.acquire_write_lock, lock.release_write_lock 177 | 178 | lock_func() 179 | if not reader: 180 | # There should be no-one else currently active, if there 181 | # is ensure we capture them so that we can later blow-up 182 | # the test. 183 | if len(active) >= 1: 184 | dups.append(me) 185 | dups.extend(active) 186 | active.append(me) 187 | time.sleep(random.random() / 100) 188 | active.remove(me) 189 | unlock_func() 190 | 191 | def run(): 192 | me = threading.current_thread() 193 | while not watch.expired(): 194 | if contextmanager: 195 | acquire_check_ctx(me, random.choice([True, False])) 196 | else: 197 | acquire_check_plain(me, random.choice([True, False])) 198 | 199 | threads = [] 200 | for i in range(0, THREAD_COUNT): 201 | t = _daemon_thread(run) 202 | threads.append(t) 203 | t.start() 204 | while threads: 205 | t = threads.pop() 206 | t.join() 207 | 208 | assert not dups 209 | assert not active 210 | 211 | 212 | def test_writer_abort(): 213 | lock = fasteners.ReaderWriterLock() 214 | assert lock.owner is None 215 | 216 | with pytest.raises(RuntimeError): 217 | with lock.write_lock(): 218 | assert lock.owner == lock.WRITER 219 | raise RuntimeError("Broken") 220 | 221 | assert lock.owner is None 222 | 223 | 224 | def test_reader_abort(): 225 | lock = fasteners.ReaderWriterLock() 226 | assert lock.owner is None 227 | 228 | with pytest.raises(RuntimeError): 229 | with lock.read_lock(): 230 | assert lock.owner == lock.READER 231 | raise RuntimeError("Broken") 232 | 233 | assert lock.owner is None 234 | 235 | 236 | def test_double_reader_abort(): 237 | lock = fasteners.ReaderWriterLock() 238 | activated = collections.deque() 239 | 240 | def double_bad_reader(): 241 | with lock.read_lock(): 242 | with lock.read_lock(): 243 | raise RuntimeError("Broken") 244 | 245 | def happy_writer(): 246 | with lock.write_lock(): 247 | activated.append(lock.owner) 248 | 249 | with futures.ThreadPoolExecutor(max_workers=20) as e: 250 | for i in range(0, 20): 251 | if i % 2 == 0: 252 | e.submit(double_bad_reader) 253 | else: 254 | e.submit(happy_writer) 255 | 256 | assert sum(a == 'w' for a in activated) == 10 257 | 258 | 259 | def test_double_reader_writer(): 260 | lock = fasteners.ReaderWriterLock() 261 | activated = collections.deque() 262 | active = threading.Event() 263 | 264 | def double_reader(): 265 | with lock.read_lock(): 266 | active.set() 267 | while not lock.has_pending_writers: 268 | time.sleep(0.001) 269 | with lock.read_lock(): 270 | activated.append(lock.owner) 271 | 272 | def happy_writer(): 273 | with lock.write_lock(): 274 | activated.append(lock.owner) 275 | 276 | reader = _daemon_thread(double_reader) 277 | reader.start() 278 | active.wait(WAIT_TIMEOUT) 279 | assert active.is_set() 280 | 281 | writer = _daemon_thread(happy_writer) 282 | writer.start() 283 | 284 | reader.join() 285 | writer.join() 286 | assert list(activated) == ['r', 'w'] 287 | 288 | 289 | def test_reader_chaotic(): 290 | lock = fasteners.ReaderWriterLock() 291 | activated = collections.deque() 292 | 293 | def chaotic_reader(blow_up): 294 | with lock.read_lock(): 295 | if blow_up: 296 | raise RuntimeError("Broken") 297 | else: 298 | activated.append(lock.owner) 299 | 300 | def happy_writer(): 301 | with lock.write_lock(): 302 | activated.append(lock.owner) 303 | 304 | with futures.ThreadPoolExecutor(max_workers=20) as e: 305 | for i in range(0, 20): 306 | if i % 2 == 0: 307 | e.submit(chaotic_reader, blow_up=bool(i % 4 == 0)) 308 | else: 309 | e.submit(happy_writer) 310 | 311 | assert sum(a == 'w' for a in activated) == 10 312 | assert sum(a == 'r' for a in activated) == 5 313 | 314 | 315 | def test_writer_chaotic(): 316 | lock = fasteners.ReaderWriterLock() 317 | activated = collections.deque() 318 | 319 | def chaotic_writer(blow_up): 320 | with lock.write_lock(): 321 | if blow_up: 322 | raise RuntimeError("Broken") 323 | else: 324 | activated.append(lock.owner) 325 | 326 | def happy_reader(): 327 | with lock.read_lock(): 328 | activated.append(lock.owner) 329 | 330 | with futures.ThreadPoolExecutor(max_workers=20) as e: 331 | for i in range(0, 20): 332 | if i % 2 == 0: 333 | e.submit(chaotic_writer, blow_up=bool(i % 4 == 0)) 334 | else: 335 | e.submit(happy_reader) 336 | 337 | assert sum(a == 'w' for a in activated) == 5 338 | assert sum(a == 'r' for a in activated) == 10 339 | 340 | 341 | def test_writer_reader_writer_ctx(): 342 | lock = fasteners.ReaderWriterLock() 343 | with lock.write_lock(): 344 | assert lock.is_writer() 345 | with lock.read_lock(): 346 | assert lock.is_reader() 347 | with lock.write_lock(): 348 | assert lock.is_writer() 349 | 350 | 351 | def test_writer_reader_writer_plain(): 352 | lock = fasteners.ReaderWriterLock() 353 | lock.acquire_write_lock() 354 | assert lock.is_writer() 355 | lock.acquire_read_lock() 356 | assert lock.is_reader() 357 | lock.acquire_write_lock() 358 | assert lock.is_writer() 359 | 360 | 361 | def test_single_reader_writer_ctx(): 362 | lock = fasteners.ReaderWriterLock() 363 | with lock.read_lock(): 364 | assert lock.is_reader() 365 | with lock.write_lock(): 366 | assert lock.is_writer() 367 | with lock.read_lock(): 368 | assert lock.is_reader() 369 | assert not lock.is_reader() 370 | assert not lock.is_writer() 371 | 372 | 373 | def test_single_reader_writer_plain(): 374 | lock = fasteners.ReaderWriterLock() 375 | 376 | lock.acquire_read_lock() 377 | assert lock.is_reader() 378 | lock.release_read_lock() 379 | 380 | lock.acquire_write_lock() 381 | assert lock.is_writer() 382 | lock.release_write_lock() 383 | 384 | lock.acquire_read_lock() 385 | assert lock.is_reader() 386 | lock.release_read_lock() 387 | 388 | assert not lock.is_reader() 389 | assert not lock.is_writer() 390 | 391 | 392 | def test_reader_to_writer_ctx(): 393 | lock = fasteners.ReaderWriterLock() 394 | 395 | with lock.read_lock(): 396 | with pytest.raises(RuntimeError): 397 | with lock.write_lock(): 398 | pass 399 | assert lock.is_reader() 400 | assert not lock.is_writer() 401 | 402 | assert not lock.is_reader() 403 | assert not lock.is_writer() 404 | 405 | 406 | def test_reader_to_writer_plain(): 407 | lock = fasteners.ReaderWriterLock() 408 | 409 | lock.acquire_read_lock() 410 | with pytest.raises(RuntimeError): 411 | lock.acquire_write_lock() 412 | assert lock.is_reader() 413 | assert not lock.is_writer() 414 | lock.release_read_lock() 415 | 416 | assert not lock.is_reader() 417 | assert not lock.is_writer() 418 | 419 | 420 | def test_writer_to_reader_ctx(): 421 | lock = fasteners.ReaderWriterLock() 422 | 423 | with lock.write_lock(): 424 | with lock.read_lock(): 425 | assert lock.is_writer() 426 | assert lock.is_reader() 427 | 428 | assert lock.is_writer() 429 | assert not lock.is_reader() 430 | 431 | assert not lock.is_writer() 432 | assert not lock.is_reader() 433 | 434 | 435 | def test_writer_to_reader_plain(): 436 | lock = fasteners.ReaderWriterLock() 437 | 438 | lock.acquire_write_lock() 439 | lock.acquire_read_lock() 440 | assert lock.is_writer() 441 | assert lock.is_reader() 442 | 443 | lock.release_read_lock() 444 | assert lock.is_writer() 445 | assert not lock.is_reader() 446 | 447 | lock.release_write_lock() 448 | assert not lock.is_writer() 449 | assert not lock.is_reader() 450 | 451 | 452 | def test_double_writer_ctx(): 453 | lock = fasteners.ReaderWriterLock() 454 | 455 | with lock.write_lock(): 456 | assert not lock.is_reader() 457 | assert lock.is_writer() 458 | 459 | with lock.write_lock(): 460 | assert lock.is_writer() 461 | 462 | assert lock.is_writer() 463 | 464 | assert not lock.is_reader() 465 | assert not lock.is_writer() 466 | 467 | 468 | def test_double_writer_plain(): 469 | lock = fasteners.ReaderWriterLock() 470 | 471 | lock.acquire_write_lock() 472 | assert not lock.is_reader() 473 | assert lock.is_writer() 474 | 475 | lock.acquire_write_lock() 476 | assert lock.is_writer() 477 | 478 | lock.release_write_lock() 479 | assert lock.is_writer() 480 | 481 | lock.release_write_lock() 482 | assert not lock.is_reader() 483 | assert not lock.is_writer() 484 | 485 | 486 | def test_double_reader_ctx(): 487 | lock = fasteners.ReaderWriterLock() 488 | 489 | with lock.read_lock(): 490 | assert lock.is_reader() 491 | assert not lock.is_writer() 492 | 493 | with lock.read_lock(): 494 | assert lock.is_reader() 495 | 496 | assert lock.is_reader() 497 | 498 | assert not lock.is_reader() 499 | assert not lock.is_writer() 500 | 501 | 502 | def test_double_reader_plain(): 503 | lock = fasteners.ReaderWriterLock() 504 | 505 | lock.acquire_read_lock() 506 | assert lock.is_reader() 507 | assert not lock.is_writer() 508 | 509 | lock.acquire_read_lock() 510 | assert lock.is_reader() 511 | 512 | lock.release_read_lock() 513 | assert lock.is_reader() 514 | 515 | lock.release_read_lock() 516 | assert not lock.is_reader() 517 | assert not lock.is_writer() 518 | 519 | 520 | def test_multi_reader_multi_writer(): 521 | writer_times, reader_times = _spawn_variation(10, 10) 522 | assert len(writer_times) == 10 523 | assert len(reader_times) == 10 524 | 525 | for (start, stop) in writer_times: 526 | assert _find_overlaps(reader_times, start, stop) == 0 527 | assert _find_overlaps(writer_times, start, stop) == 1 528 | for (start, stop) in reader_times: 529 | assert _find_overlaps(writer_times, start, stop) == 0 530 | 531 | 532 | def test_multi_reader_single_writer(): 533 | writer_times, reader_times = _spawn_variation(9, 1) 534 | assert len(writer_times) == 1 535 | assert len(reader_times) == 9 536 | 537 | start, stop = writer_times[0] 538 | assert _find_overlaps(reader_times, start, stop) == 0 539 | 540 | 541 | def test_multi_writer(): 542 | writer_times, reader_times = _spawn_variation(0, 10) 543 | assert len(writer_times) == 10 544 | assert len(reader_times) == 0 545 | 546 | for (start, stop) in writer_times: 547 | assert _find_overlaps(writer_times, start, stop) == 1 548 | 549 | 550 | def test_deadlock_reproducer(): 551 | lock = fasteners.ReaderWriterLock() 552 | 553 | def thread1(): 554 | for _ in range(0, 1000): 555 | print("thread1 iteration: {}".format(_)) 556 | print("thread1 waiting for write lock") 557 | with lock.write_lock(): 558 | print("thread1 got write lock") 559 | print("thread1 waiting for read lock") 560 | with lock.read_lock(): 561 | print("thread1 got read lock") 562 | print("thread1 released read lock") 563 | print("thread1 released write lock") 564 | 565 | def thread2(): 566 | for _ in range(0, 1000): 567 | print("thread2 iteration: {}".format(_)) 568 | print("thread2 waiting for write lock") 569 | with lock.write_lock(): 570 | print("thread2 got write lock") 571 | print("thread2 released write lock") 572 | 573 | funcs = (thread1, thread2) 574 | 575 | threads = [] 576 | for func in funcs: 577 | t = _daemon_thread(func) 578 | threads.append(t) 579 | t.start() 580 | while threads: 581 | t = threads.pop() 582 | t.join() 583 | 584 | 585 | def test_error_when_releasing_write_lock_without_write_lock(): 586 | lock = fasteners.ReaderWriterLock() 587 | with pytest.raises(RuntimeError): 588 | lock.release_write_lock() 589 | 590 | 591 | def test_error_when_releasing_read_lock_without_read_lock(): 592 | lock = fasteners.ReaderWriterLock() 593 | with pytest.raises(RuntimeError): 594 | lock.release_read_lock() 595 | -------------------------------------------------------------------------------- /tests/test_process_lock.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Copyright 2011 OpenStack Foundation. 4 | # Copyright 2011 Justin Santa Barbara 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 7 | # not use this file except in compliance with the License. You may obtain 8 | # a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 15 | # License for the specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import contextlib 19 | import errno 20 | import multiprocessing 21 | import os 22 | import shutil 23 | import sys 24 | import tempfile 25 | import threading 26 | import time 27 | 28 | import pytest 29 | 30 | from fasteners import process_lock as pl 31 | from fasteners.process_mechanism import _interprocess_mechanism 32 | 33 | WIN32 = os.name == 'nt' 34 | 35 | 36 | @contextlib.contextmanager 37 | def scoped_child_processes(children, timeout=0.1, exitcode=0): 38 | for child in children: 39 | child.daemon = True 40 | child.start() 41 | yield 42 | start = time.time() 43 | timed_out = 0 44 | 45 | for child in children: 46 | child.join(max(timeout - (time.time() - start), 0)) 47 | if child.is_alive(): 48 | timed_out += 1 49 | child.terminate() 50 | 51 | if timed_out: 52 | msg = "{} child processes killed due to timeout\n".format(timed_out) 53 | sys.stderr.write(msg) 54 | 55 | if exitcode is not None: 56 | for child in children: 57 | c_code = child.exitcode 58 | msg = "Child exitcode {} != {}" 59 | assert c_code == exitcode, msg.format(c_code, exitcode) 60 | 61 | 62 | def try_lock(lock_file): 63 | try: 64 | my_lock = pl.InterProcessLock(lock_file) 65 | my_lock.lockfile = open(lock_file, 'w') 66 | my_lock.trylock() 67 | my_lock.unlock() 68 | sys.exit(1) 69 | except IOError: 70 | sys.exit(0) 71 | 72 | 73 | def inter_processlock_helper(lockname, lock_filename, pipe): 74 | lock2 = pl.InterProcessLock(lockname) 75 | lock2.lockfile = open(lock_filename, 'w') 76 | have_lock = False 77 | while not have_lock: 78 | try: 79 | lock2.trylock() 80 | have_lock = True 81 | except IOError: 82 | pass 83 | # Hold the lock and wait for the parent 84 | pipe.send(None) 85 | pipe.recv() 86 | 87 | 88 | @pytest.fixture() 89 | def lock_dir(): 90 | tmp_dir = tempfile.mkdtemp() 91 | yield tmp_dir 92 | shutil.rmtree(tmp_dir, ignore_errors=True) 93 | 94 | 95 | @pytest.fixture() 96 | def handles_dir(): 97 | tmp_dir = tempfile.mkdtemp() 98 | yield tmp_dir 99 | shutil.rmtree(tmp_dir, ignore_errors=True) 100 | 101 | 102 | def test_lock_acquire_release_file_lock(lock_dir): 103 | lock_file = os.path.join(lock_dir, 'lock') 104 | lock = pl.InterProcessLock(lock_file) 105 | 106 | def attempt_acquire(count): 107 | children = [ 108 | multiprocessing.Process(target=try_lock, args=(lock_file,)) 109 | for i in range(count)] 110 | with scoped_child_processes(children, timeout=10, exitcode=None): 111 | pass 112 | return sum(c.exitcode for c in children) 113 | 114 | assert lock.acquire() 115 | try: 116 | acquired_children = attempt_acquire(10) 117 | assert acquired_children == 0 118 | finally: 119 | lock.release() 120 | 121 | acquired_children = attempt_acquire(5) 122 | assert acquired_children != 0 123 | 124 | 125 | def test_nested_synchronized_external_works(lock_dir): 126 | sentinel = object() 127 | 128 | @pl.interprocess_locked(os.path.join(lock_dir, 'test-lock-1')) 129 | def outer_lock(): 130 | @pl.interprocess_locked(os.path.join(lock_dir, 'test-lock-2')) 131 | def inner_lock(): 132 | return sentinel 133 | 134 | return inner_lock() 135 | 136 | assert outer_lock() == sentinel 137 | 138 | 139 | def _lock_files(lock_path, handles_dir, num_handles=50): 140 | with pl.InterProcessLock(lock_path): 141 | 142 | # Open some files we can use for locking 143 | handles = [] 144 | for n in range(num_handles): 145 | path = os.path.join(handles_dir, ('file-%s' % n)) 146 | handles.append(open(path, 'w')) 147 | 148 | # Loop over all the handles and try locking the file 149 | # without blocking, keep a count of how many files we 150 | # were able to lock and then unlock. If the lock fails 151 | # we get an IOError and bail out with bad exit code 152 | count = 0 153 | for handle in handles: 154 | try: 155 | _interprocess_mechanism.trylock(handle) 156 | count += 1 157 | _interprocess_mechanism.unlock(handle) 158 | except IOError: 159 | sys.exit(2) 160 | finally: 161 | handle.close() 162 | 163 | # Check if we were able to open all files 164 | if count != num_handles: 165 | raise AssertionError("Unable to open all handles") 166 | 167 | 168 | def _do_test_lock_externally(lock_dir_, handles_dir_): 169 | lock_path = os.path.join(lock_dir_, "lock") 170 | 171 | num_handles = 50 172 | num_processes = 50 173 | args = (lock_path, handles_dir_, num_handles) 174 | children = [multiprocessing.Process(target=_lock_files, args=args) 175 | for _ in range(num_processes)] 176 | 177 | with scoped_child_processes(children, timeout=30, exitcode=0): 178 | pass 179 | 180 | 181 | def test_lock_externally(lock_dir, handles_dir): 182 | _do_test_lock_externally(lock_dir, handles_dir) 183 | 184 | 185 | def test_lock_externally_lock_dir_not_exist(lock_dir, handles_dir): 186 | os.rmdir(lock_dir) 187 | _do_test_lock_externally(lock_dir, handles_dir) 188 | 189 | 190 | def test_lock_file_exists(lock_dir): 191 | lock_file = os.path.join(lock_dir, 'lock') 192 | 193 | @pl.interprocess_locked(lock_file) 194 | def foo(): 195 | assert os.path.exists(lock_file) 196 | 197 | foo() 198 | 199 | 200 | def test_bad_release(lock_dir): 201 | lock_file = os.path.join(lock_dir, 'lock') 202 | lock = pl.InterProcessLock(lock_file) 203 | with pytest.raises(threading.ThreadError): 204 | lock.release() 205 | 206 | 207 | def test_interprocess_lock(lock_dir): 208 | lock_file = os.path.join(lock_dir, 'lock') 209 | lock_name = 'foo' 210 | 211 | child_pipe, them = multiprocessing.Pipe() 212 | child = multiprocessing.Process( 213 | target=inter_processlock_helper, args=(lock_name, lock_file, them)) 214 | 215 | with scoped_child_processes((child,)): 216 | 217 | # Make sure the child grabs the lock first 218 | if not child_pipe.poll(5): 219 | pytest.fail('Timed out waiting for child to grab lock') 220 | 221 | start = time.time() 222 | lock1 = pl.InterProcessLock(lock_name) 223 | lock1.lockfile = open(lock_file, 'w') 224 | # NOTE(bnemec): There is a brief window between when the lock file 225 | # is created and when it actually becomes locked. If we happen to 226 | # context switch in that window we may succeed in locking the 227 | # file. Keep retrying until we either get the expected exception 228 | # or timeout waiting. 229 | while time.time() - start < 5: 230 | try: 231 | lock1.trylock() 232 | lock1.unlock() 233 | time.sleep(0) 234 | except IOError: 235 | # This is what we expect to happen 236 | break 237 | else: 238 | pytest.fail('Never caught expected lock exception') 239 | 240 | child_pipe.send(None) 241 | 242 | 243 | @pytest.mark.skipif(WIN32, reason='Windows cannot open file handles twice') 244 | def test_non_destructive(lock_dir): 245 | lock_file = os.path.join(lock_dir, 'not-destroyed') 246 | with open(lock_file, 'w') as f: 247 | f.write('test') 248 | with pl.InterProcessLock(lock_file): 249 | with open(lock_file) as f: 250 | assert f.read() == 'test' 251 | 252 | 253 | class BrokenLock(pl.InterProcessLock): 254 | def __init__(self, name, errno_code): 255 | super(BrokenLock, self).__init__(name) 256 | self.errno_code = errno_code 257 | 258 | def unlock(self): 259 | pass 260 | 261 | def trylock(self): 262 | err = IOError() 263 | err.errno = self.errno_code 264 | raise err 265 | 266 | 267 | def test_bad_acquire(lock_dir): 268 | lock_file = os.path.join(lock_dir, 'lock') 269 | lock = BrokenLock(lock_file, errno.EBUSY) 270 | with pytest.raises(threading.ThreadError): 271 | lock.acquire() 272 | 273 | 274 | def test_lock_twice(lock_dir): 275 | lock_file = os.path.join(lock_dir, 'lock') 276 | lock = pl.InterProcessLock(lock_file) 277 | 278 | ok = lock.acquire(blocking=False) 279 | assert ok 280 | 281 | # ok on Unix, not ok on Windows 282 | ok = lock.acquire(blocking=False) 283 | assert ok or not ok 284 | 285 | # should release without crashing 286 | lock.release() 287 | -------------------------------------------------------------------------------- /tests/test_reader_writer_lock.py: -------------------------------------------------------------------------------- 1 | from multiprocessing import Pool 2 | from multiprocessing import Process 3 | import os 4 | from pathlib import Path 5 | import random 6 | import shutil 7 | import tempfile 8 | import time 9 | 10 | from diskcache import Cache 11 | from diskcache import Deque 12 | import more_itertools as mo 13 | import pytest 14 | 15 | from fasteners.process_lock import InterProcessReaderWriterLock as ReaderWriterLock 16 | 17 | PROCESS_COUNT = 20 18 | 19 | 20 | @pytest.fixture() 21 | def lock_file(): 22 | lock_file_ = tempfile.NamedTemporaryFile() 23 | lock_file_.close() 24 | yield lock_file_.name 25 | os.remove(lock_file_.name) 26 | 27 | 28 | @pytest.fixture() 29 | def dc(): 30 | disk_cache_dir_ = tempfile.mkdtemp() 31 | with Cache(directory=disk_cache_dir_) as dc: 32 | yield dc 33 | shutil.rmtree(disk_cache_dir_, ignore_errors=True) 34 | 35 | 36 | @pytest.fixture() 37 | def deque(): 38 | disk_cache_dir_ = tempfile.mkdtemp() 39 | with Cache(directory=disk_cache_dir_) as dc: 40 | yield Deque.fromcache(dc) 41 | shutil.rmtree(disk_cache_dir_, ignore_errors=True) 42 | 43 | 44 | def test_lock(lock_file): 45 | with ReaderWriterLock(lock_file).write_lock(): 46 | pass 47 | 48 | with ReaderWriterLock(lock_file).read_lock(): 49 | pass 50 | 51 | 52 | def no_concurrent_writers(lock_file, dc): 53 | for _ in range(10): 54 | with ReaderWriterLock(lock_file).write_lock(): 55 | if dc.get('active_count', 0) >= 1: 56 | dc.incr('dups_count') 57 | dc.incr('active_count') 58 | time.sleep(random.random() / 1000) 59 | dc.decr('active_count') 60 | dc.incr('visited_count') 61 | 62 | 63 | def test_no_concurrent_writers(lock_file, dc): 64 | pool = Pool(PROCESS_COUNT) 65 | pool.starmap(no_concurrent_writers, [(lock_file, dc)] * PROCESS_COUNT, chunksize=1) 66 | 67 | assert dc.get('active_count') == 0 68 | assert dc.get('dups_count') is None 69 | assert dc.get('visited_count') == 10 * PROCESS_COUNT 70 | 71 | 72 | def no_concurrent_readers_writers(lock_file, dc): 73 | for _ in range(10): 74 | reader = random.choice([True, False]) 75 | if reader: 76 | lock_func = ReaderWriterLock(lock_file).read_lock 77 | else: 78 | lock_func = ReaderWriterLock(lock_file).write_lock 79 | with lock_func(): 80 | if not reader: 81 | if dc.get('active_count', 0) >= 1: 82 | dc.incr('dups_count') 83 | dc.incr('active_count') 84 | time.sleep(random.random() / 1000) 85 | dc.decr('active_count') 86 | dc.incr('visited_count') 87 | 88 | 89 | def test_no_concurrent_readers_writers(lock_file, dc): 90 | pool = Pool(PROCESS_COUNT) 91 | pool.starmap(no_concurrent_readers_writers, [(lock_file, dc)] * PROCESS_COUNT, 92 | chunksize=1) 93 | 94 | assert dc.get('active_count') == 0 95 | assert dc.get('dups_count') is None 96 | assert dc.get('visited_count') == 10 * PROCESS_COUNT 97 | 98 | 99 | def reader_releases_lock_upon_crash_reader_lock(lock_file, dc, i): 100 | with ReaderWriterLock(lock_file).read_lock(): 101 | dc.set('pid{}'.format(i), os.getpid()) 102 | raise RuntimeError('') 103 | 104 | 105 | def reader_releases_lock_upon_crash_writer_lock(lock_file, dc, i): 106 | ReaderWriterLock(lock_file).acquire_write_lock(timeout=5) 107 | dc.set('pid{}'.format(i), os.getpid()) 108 | 109 | 110 | def test_reader_releases_lock_upon_crash(lock_file, dc): 111 | p1 = Process(target=reader_releases_lock_upon_crash_reader_lock, 112 | args=(lock_file, dc, 1)) 113 | p2 = Process(target=reader_releases_lock_upon_crash_writer_lock, 114 | args=(lock_file, dc, 2)) 115 | 116 | p1.start() 117 | p1.join() 118 | 119 | p2.start() 120 | p2.join() 121 | 122 | assert dc.get('pid1') != dc.get('pid2') 123 | assert p1.exitcode != 0 124 | assert p2.exitcode == 0 125 | 126 | 127 | def writer_releases_lock_upon_crash(lock_file, dc, i, crash): 128 | ReaderWriterLock(lock_file).acquire_write_lock(timeout=5) 129 | dc.set('pid{}'.format(i), os.getpid()) 130 | if crash: 131 | raise RuntimeError('') 132 | 133 | 134 | def test_writer_releases_lock_upon_crash(lock_file, dc): 135 | p1 = Process(target=writer_releases_lock_upon_crash, 136 | args=(lock_file, dc, 1, True)) 137 | p2 = Process(target=writer_releases_lock_upon_crash, 138 | args=(lock_file, dc, 2, False)) 139 | 140 | p1.start() 141 | p1.join() 142 | 143 | p2.start() 144 | p2.join() 145 | 146 | assert dc.get('pid1') != dc.get('pid2') 147 | assert p1.exitcode != 0 148 | assert p2.exitcode == 0 149 | 150 | 151 | def _spawn_variation(lock_file, deque, readers, writers): 152 | pool = Pool(readers + writers) 153 | pool.starmap(_spawling, [(lock_file, deque, type_) for type_ in ['w'] * writers + ['r'] * readers]) 154 | return deque 155 | 156 | 157 | def _spawling(lock_file, visits, type_): 158 | lock = ReaderWriterLock(lock_file) 159 | 160 | if type_ == 'w': 161 | lock.acquire_write_lock(timeout=5) 162 | else: 163 | lock.acquire_read_lock(timeout=5) 164 | 165 | visits.append((os.getpid(), type_)) 166 | time.sleep(random.random() / 100 + 0.01) 167 | visits.append((os.getpid(), type_)) 168 | 169 | if type_ == 'w': 170 | lock.release_write_lock() 171 | else: 172 | lock.release_read_lock() 173 | 174 | 175 | def _assert_valid(visits): 176 | """Check if writes dont overlap other writes and reads""" 177 | 178 | # check that writes open and close consequently 179 | write_blocks = mo.split_at(visits, lambda x: x[1] == 'r') 180 | for write_block in write_blocks: 181 | for v1, v2 in mo.chunked(write_block, 2): 182 | assert v1[0] == v2[0] 183 | 184 | # check that reads open and close in groups between writes 185 | read_blocks = mo.split_at(visits, lambda x: x[1] == 'w') 186 | for read_block in read_blocks: 187 | for v1, v2 in mo.chunked(sorted(read_block), 2): 188 | assert v1[0] == v2[0] 189 | 190 | 191 | def test_multi_reader_multi_writer(lock_file, deque): 192 | visits = _spawn_variation(Path(lock_file), deque, 10, 10) 193 | assert len(visits) == 20 * 2 194 | _assert_valid(visits) 195 | 196 | 197 | def test_multi_reader_single_writer(lock_file, deque): 198 | visits = _spawn_variation(Path(lock_file), deque, 9, 1) 199 | assert len(visits) == 10 * 2 200 | _assert_valid(visits) 201 | 202 | 203 | def test_multi_writer(lock_file, deque): 204 | visits = _spawn_variation(Path(lock_file), deque, 0, 10) 205 | assert len(visits) == 10 * 2 206 | _assert_valid(visits) 207 | 208 | 209 | def test_lock_writer_twice(lock_file): 210 | lock = ReaderWriterLock(lock_file) 211 | 212 | ok = lock.acquire_write_lock(blocking=False) 213 | assert ok 214 | 215 | # ok on Unix, not ok on Windows 216 | ok = lock.acquire_write_lock(blocking=False) 217 | assert ok or not ok 218 | 219 | # should release without crashing 220 | lock.release_write_lock() 221 | 222 | 223 | @pytest.mark.skipif(os.name != 'nt', reason='Only Windows is affected') 224 | def test_lock_file_ex_global_modification(lock_file): 225 | """Some libraries modify the global LockFileEx pointer, and we have to be 226 | resistant to that (as well as not modify the global pointer ourselves!)""" 227 | 228 | from ctypes import windll 229 | from ctypes.wintypes import DWORD, HANDLE 230 | 231 | windll.kernel32.LockFileEx.argtypes = [HANDLE, DWORD] # nonsensical signature 232 | 233 | lock = ReaderWriterLock(lock_file) 234 | lock.acquire_write_lock(blocking=False) 235 | lock.release_write_lock() 236 | --------------------------------------------------------------------------------