├── .circleci └── config.yml ├── .coveragerc ├── .gitignore ├── LICENSE ├── MANIFEST.in ├── README.rst ├── RELEASENOTE.md ├── appveyor.yml ├── benchmark └── run_benchmark.py ├── extra-requirements.txt ├── persistqueue ├── __init__.py ├── exceptions.py ├── mysqlqueue.py ├── pdict.py ├── py.typed ├── queue.py ├── serializers │ ├── __init__.py │ ├── cbor2.py │ ├── json.py │ ├── msgpack.py │ └── pickle.py ├── sqlackqueue.py ├── sqlbase.py ├── sqlqueue.py └── tests │ ├── __init__.py │ ├── test_mysqlqueue.py │ ├── test_pdict.py │ ├── test_queue.py │ ├── test_sqlackqueue.py │ └── test_sqlqueue.py ├── requirements.txt ├── scripts └── publish.sh ├── setup.cfg ├── setup.py ├── test-requirements.txt └── tox.ini /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | jobs: 3 | py27: 4 | docker: 5 | # Primary container image where all steps run. 6 | - image: circleci/python:2.7.17 7 | environment: 8 | TOXENV: py27 9 | # MySQL env for mysql queue tests 10 | - image: circleci/mysql:8.0 11 | environment: 12 | MYSQL_ROOT_PASSWORD: rootpw 13 | MYSQL_DATABASE: testqueue 14 | MYSQL_USER: user 15 | MYSQL_PASSWORD: passw0rd 16 | MYSQL_HOST: '%' 17 | 18 | steps: &common_steps 19 | - checkout 20 | - run: 21 | # Our primary container isn't MYSQL so run a sleep command until it's ready. 22 | name: Waiting for MySQL to be ready 23 | command: | 24 | for i in `seq 1 10`; 25 | do 26 | nc -z 127.0.0.1 3306 && echo Success && exit 0 27 | echo -n . 28 | sleep 5 29 | done 30 | echo Failed waiting for MySQL && exit 1 31 | - run: 32 | command: | 33 | pip install tox 34 | - run: 35 | command: | # tell the operating system to remove the file size limit on core dump files 36 | tox 37 | - run: bash <(curl -s https://codecov.io/bash) -cF python 38 | - run: 39 | command: | 40 | mkdir -p /tmp/core_dumps 41 | ls core.* && cp core.* /tmp/core_dumps 42 | when: on_fail 43 | - store_artifacts: 44 | # collect core dumps 45 | path: /tmp/core_dumps 46 | - store_artifacts: 47 | path: .coverage 48 | - store_artifacts: 49 | path: coverage.xml 50 | - store_artifacts: 51 | path: htmlcov 52 | 53 | py38: 54 | docker: 55 | # Primary container image where all steps run. 56 | - image: cimg/python:3.8 57 | environment: 58 | TOXENV: py38 59 | # MySQL env for mysql queue tests 60 | - image: circleci/mysql:8.0 61 | environment: 62 | MYSQL_ROOT_PASSWORD: 123456 63 | MYSQL_DATABASE: testqueue 64 | MYSQL_USER: user 65 | MYSQL_PASSWORD: passw0rd 66 | steps: *common_steps 67 | py39: 68 | docker: 69 | # Primary container image where all steps run. 70 | - image: cimg/python:3.9 71 | environment: 72 | TOXENV: py39 73 | # MySQL env for mysql queue tests 74 | - image: circleci/mysql:8.0 75 | environment: 76 | MYSQL_ROOT_PASSWORD: 123456 77 | MYSQL_DATABASE: testqueue 78 | MYSQL_USER: user 79 | MYSQL_PASSWORD: passw0rd 80 | steps: *common_steps 81 | py310: 82 | docker: 83 | # Primary container image where all steps run. 84 | - image: cimg/python:3.10 85 | environment: 86 | TOXENV: py310 87 | # MySQL env for mysql queue tests 88 | - image: circleci/mysql:8.0 89 | environment: 90 | MYSQL_ROOT_PASSWORD: 123456 91 | MYSQL_DATABASE: testqueue 92 | MYSQL_USER: user 93 | MYSQL_PASSWORD: passw0rd 94 | steps: *common_steps 95 | py311: 96 | docker: 97 | # Primary container image where all steps run. 98 | - image: cimg/python:3.11 99 | environment: 100 | TOXENV: py311 101 | # MySQL env for mysql queue tests 102 | - image: circleci/mysql:8.0 103 | environment: 104 | MYSQL_ROOT_PASSWORD: 123456 105 | MYSQL_DATABASE: testqueue 106 | MYSQL_USER: user 107 | MYSQL_PASSWORD: passw0rd 108 | steps: *common_steps 109 | py312: 110 | docker: 111 | # Primary container image where all steps run. 112 | - image: cimg/python:3.12 113 | environment: 114 | TOXENV: py312 115 | # MySQL env for mysql queue tests 116 | - image: circleci/mysql:8.0 117 | environment: 118 | MYSQL_ROOT_PASSWORD: 123456 119 | MYSQL_DATABASE: testqueue 120 | MYSQL_USER: user 121 | MYSQL_PASSWORD: passw0rd 122 | steps: *common_steps 123 | 124 | pep8: 125 | docker: 126 | # Primary container image where all steps run. 127 | - image: cimg/python:3.8 128 | environment: 129 | TOXENV: pep8 130 | # MySQL env for mysql queue tests 131 | - image: circleci/mysql:8.0 132 | environment: 133 | MYSQL_ROOT_PASSWORD: rootpw 134 | MYSQL_DATABASE: testqueue 135 | MYSQL_USER: user 136 | MYSQL_PASSWORD: passw0rd 137 | steps: *common_steps 138 | 139 | 140 | cover: 141 | docker: 142 | # Primary container image where all steps run. 143 | - image: cimg/python:3.8 144 | environment: 145 | TOXENV: cover 146 | # MySQL env for mysql queue tests 147 | - image: circleci/mysql:8.0 148 | environment: 149 | MYSQL_ROOT_PASSWORD: 123456 150 | MYSQL_DATABASE: testqueue 151 | MYSQL_USER: user 152 | MYSQL_PASSWORD: passw0rd 153 | steps: *common_steps 154 | 155 | workflows: 156 | version: 2 157 | build: 158 | jobs: 159 | - pep8 160 | - py38 161 | - py39 162 | - py310 163 | - py311 164 | - py312 165 | - cover 166 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | source = persistqueue/* 4 | omit = 5 | ./tests/* 6 | ./.tox/* 7 | ./setup.py 8 | [xml] 9 | output = coverage.xml 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | .testrepository/ 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | local_settings.py 55 | 56 | # Flask stuff: 57 | instance/ 58 | .webassets-cache 59 | 60 | # Scrapy stuff: 61 | .scrapy 62 | 63 | # Sphinx documentation 64 | docs/_build/ 65 | 66 | # PyBuilder 67 | target/ 68 | 69 | # IPython Notebook 70 | .ipynb_checkpoints 71 | 72 | # pyenv 73 | .python-version 74 | 75 | # celery beat schedule file 76 | celerybeat-schedule 77 | 78 | # dotenv 79 | .env 80 | 81 | # virtualenv 82 | venv/ 83 | ENV/ 84 | 85 | # Spyder project settings 86 | .spyderproject 87 | 88 | # Rope project settings 89 | .ropeproject 90 | 91 | # IDE specific folders 92 | .idea/ 93 | .vscode/ 94 | 95 | # MacOS 96 | .DS_Store 97 | 98 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) G. B. Versiani. 2 | Copyright (c) Peter Wang. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, 6 | are permitted provided that the following conditions are met: 7 | 8 | 1. Redistributions of source code must retain the above copyright notice, 9 | this list of conditions and the following disclaimer. 10 | 11 | 2. Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the distribution. 14 | 15 | 3. Neither the name of python-pqueue nor the names of its contributors may be used 16 | to endorse or promote products derived from this software without 17 | specific prior written permission. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 20 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 21 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 22 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 23 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 24 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 25 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 26 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 28 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include README.rst 3 | include *.txt 4 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | persist-queue - A thread-safe, disk-based queue for Python 2 | ========================================================== 3 | 4 | .. image:: https://img.shields.io/circleci/project/github/peter-wangxu/persist-queue/master.svg?label=Linux%20%26%20Mac 5 | :target: https://circleci.com/gh/peter-wangxu/persist-queue 6 | 7 | .. image:: https://img.shields.io/appveyor/ci/peter-wangxu/persist-queue/master.svg?label=Windows 8 | :target: https://ci.appveyor.com/project/peter-wangxu/persist-queue 9 | 10 | .. image:: https://img.shields.io/codecov/c/github/peter-wangxu/persist-queue/master.svg 11 | :target: https://codecov.io/gh/peter-wangxu/persist-queue 12 | 13 | .. image:: https://img.shields.io/pypi/v/persist-queue.svg 14 | :target: https://pypi.python.org/pypi/persist-queue 15 | 16 | .. image:: https://img.shields.io/pypi/pyversions/persist-queue 17 | :alt: PyPI - Python Version 18 | 19 | ``persist-queue`` implements a file-based queue and a serial of sqlite3-based queues. The goals is to achieve following requirements: 20 | 21 | * Disk-based: each queued item should be stored in disk in case of any crash. 22 | * Thread-safe: can be used by multi-threaded producers and multi-threaded consumers. 23 | * Recoverable: Items can be read after process restart. 24 | * Green-compatible: can be used in ``greenlet`` or ``eventlet`` environment. 25 | 26 | While *queuelib* and *python-pqueue* cannot fulfil all of above. After some try, I found it's hard to achieve based on their current 27 | implementation without huge code change. this is the motivation to start this project. 28 | 29 | By default, *persist-queue* use *pickle* object serialization module to support object instances. 30 | Most built-in type, like `int`, `dict`, `list` are able to be persisted by `persist-queue` directly, to support customized objects, 31 | please refer to `Pickling and unpickling extension types(Python2) `_ 32 | and `Pickling Class Instances(Python3) `_ 33 | 34 | This project is based on the achievements of `python-pqueue `_ 35 | and `queuelib `_ 36 | 37 | Slack channels 38 | ^^^^^^^^^^^^^^ 39 | 40 | Join `persist-queue `_ channel 43 | 44 | 45 | Requirements 46 | ------------ 47 | * Python 3.5 or newer versions (refer to `Deprecation`_ for older Python versions) 48 | * Full support for Linux and MacOS. 49 | * Windows support (with `Caution`_ if ``persistqueue.Queue`` is used). 50 | 51 | Features 52 | -------- 53 | 54 | - Multiple platforms support: Linux, macOS, Windows 55 | - Pure python 56 | - Both filed based queues and sqlite3 based queues are supported 57 | - Filed based queue: multiple serialization protocol support: pickle(default), msgpack, cbor, json 58 | 59 | Deprecation 60 | ----------- 61 | - `persist-queue` drops Python 2 support since version `1.0.0`, no new feature will be developed under Python 2 as `Python 2 was sunset on January 1, 2020 `_. 62 | - `Python 3.4 release has reached end of life `_ and 63 | `DBUtils `_ ceased support for `Python 3.4`, `persist queue` drops MySQL based queue for python 3.4 since version 0.8.0. 64 | other queue implementations such as file based queue and sqlite3 based queue are still workable. 65 | 66 | Installation 67 | ------------ 68 | 69 | from pypi 70 | ^^^^^^^^^ 71 | 72 | .. code-block:: console 73 | 74 | pip install persist-queue 75 | # for msgpack, cbor and mysql support, use following command 76 | pip install "persist-queue[extra]" 77 | 78 | 79 | from source code 80 | ^^^^^^^^^^^^^^^^ 81 | 82 | .. code-block:: console 83 | 84 | git clone https://github.com/peter-wangxu/persist-queue 85 | cd persist-queue 86 | # for msgpack and cbor support, run 'pip install -r extra-requirements.txt' first 87 | python setup.py install 88 | 89 | 90 | Benchmark 91 | --------- 92 | 93 | Here are the time spent(in seconds) for writing/reading **1000** items to the 94 | disk comparing the sqlite3 and file queue. 95 | 96 | - Windows 97 | - OS: Windows 10 98 | - Disk: SATA3 SSD 99 | - RAM: 16 GiB 100 | 101 | +---------------+---------+-------------------------+----------------------------+ 102 | | | Write | Write/Read(1 task_done) | Write/Read(many task_done) | 103 | +---------------+---------+-------------------------+----------------------------+ 104 | | SQLite3 Queue | 1.8880 | 2.0290 | 3.5940 | 105 | +---------------+---------+-------------------------+----------------------------+ 106 | | File Queue | 4.9520 | 5.0560 | 8.4900 | 107 | +---------------+---------+-------------------------+----------------------------+ 108 | 109 | **windows note** 110 | Performance of Windows File Queue has dramatic improvement since `v0.4.1` due to the 111 | atomic renaming support(3-4X faster) 112 | 113 | - Linux 114 | - OS: Ubuntu 16.04 (VM) 115 | - Disk: SATA3 SSD 116 | - RAM: 4 GiB 117 | 118 | +---------------+--------+-------------------------+----------------------------+ 119 | | | Write | Write/Read(1 task_done) | Write/Read(many task_done) | 120 | +---------------+--------+-------------------------+----------------------------+ 121 | | SQLite3 Queue | 1.8282 | 1.8075 | 2.8639 | 122 | +---------------+--------+-------------------------+----------------------------+ 123 | | File Queue | 0.9123 | 1.0411 | 2.5104 | 124 | +---------------+--------+-------------------------+----------------------------+ 125 | 126 | - Mac OS 127 | - OS: 10.14 (macOS Mojave) 128 | - Disk: PCIe SSD 129 | - RAM: 16 GiB 130 | 131 | +---------------+--------+-------------------------+----------------------------+ 132 | | | Write | Write/Read(1 task_done) | Write/Read(many task_done) | 133 | +---------------+--------+-------------------------+----------------------------+ 134 | | SQLite3 Queue | 0.1879 | 0.2115 | 0.3147 | 135 | +---------------+--------+-------------------------+----------------------------+ 136 | | File Queue | 0.5158 | 0.5357 | 1.0446 | 137 | +---------------+--------+-------------------------+----------------------------+ 138 | 139 | **note** 140 | 141 | - The value above is in seconds for reading/writing *1000* items, the less 142 | the better 143 | - Above result was got from: 144 | 145 | .. code-block:: console 146 | 147 | python benchmark/run_benchmark.py 1000 148 | 149 | 150 | To see the real performance on your host, run the script under ``benchmark/run_benchmark.py``: 151 | 152 | .. code-block:: console 153 | 154 | python benchmark/run_benchmark.py 155 | 156 | 157 | Examples 158 | -------- 159 | 160 | 161 | Example usage with a SQLite3 based queue 162 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 163 | 164 | .. code-block:: python 165 | 166 | >>> import persistqueue 167 | >>> q = persistqueue.SQLiteQueue('mypath', auto_commit=True) 168 | >>> q.put('str1') 169 | >>> q.put('str2') 170 | >>> q.put('str3') 171 | >>> q.get() 172 | 'str1' 173 | >>> del q 174 | 175 | 176 | Close the console, and then recreate the queue: 177 | 178 | .. code-block:: python 179 | 180 | >>> import persistqueue 181 | >>> q = persistqueue.SQLiteQueue('mypath', auto_commit=True) 182 | >>> q.get() 183 | 'str2' 184 | >>> 185 | 186 | New functions: 187 | *Available since v0.8.0* 188 | 189 | - ``shrink_disk_usage`` perform a ``VACUUM`` against the sqlite, and rebuild the database file, this usually takes long time and frees a lot of disk space after ``get()`` 190 | 191 | 192 | Example usage of SQLite3 based ``UniqueQ`` 193 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 194 | This queue does not allow duplicate items. 195 | 196 | .. code-block:: python 197 | 198 | >>> import persistqueue 199 | >>> q = persistqueue.UniqueQ('mypath') 200 | >>> q.put('str1') 201 | >>> q.put('str1') 202 | >>> q.size 203 | 1 204 | >>> q.put('str2') 205 | >>> q.size 206 | 2 207 | >>> 208 | 209 | Example usage of SQLite3 based ``SQLiteAckQueue``/``UniqueAckQ`` 210 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 211 | The core functions: 212 | 213 | - ``put``: add item to the queue. Returns ``id`` 214 | - ``get``: get item from queue and mark as unack. Returns ``item``, Optional paramaters (``block``, ``timeout``, ``id``, ``next_in_order``, ``raw``) 215 | - ``update``: update an item. Returns ``id``, Paramaters (``item``), Optional parameter if item not in raw format (``id``) 216 | - ``ack``: mark item as acked. Returns ``id``, Parameters (``item`` or ``id``) 217 | - ``nack``: there might be something wrong with current consumer, so mark item as ready and new consumer will get it. Returns ``id``, Parameters (``item`` or ``id``) 218 | - ``ack_failed``: there might be something wrong during process, so just mark item as failed. Returns ``id``, Parameters (``item`` or ``id``) 219 | - ``clear_acked_data``: perform a sql delete agaist sqlite. It removes 1000 items, while keeping 1000 of the most recent, whose status is ``AckStatus.acked`` (note: this does not shrink the file size on disk) Optional paramters (``max_delete``, ``keep_latest``, ``clear_ack_failed``) 220 | - ``shrink_disk_usage`` perform a ``VACUUM`` against the sqlite, and rebuild the database file, this usually takes long time and frees a lot of disk space after ``clear_acked_data`` 221 | - ``queue``: returns the database contents as a Python List[Dict] 222 | - ``active_size``: The active size changes when an item is added (put) and completed (ack/ack_failed) unlike ``qsize`` which changes when an item is pulled (get) or returned (nack). 223 | 224 | .. code-block:: python 225 | 226 | >>> import persistqueue 227 | >>> ackq = persistqueue.SQLiteAckQueue('path') 228 | >>> ackq.put('str1') 229 | >>> item = ackq.get() 230 | >>> # Do something with the item 231 | >>> ackq.ack(item) # If done with the item 232 | >>> ackq.nack(item) # Else mark item as `nack` so that it can be proceeded again by any worker 233 | >>> ackq.ack_failed(item) # Or else mark item as `ack_failed` to discard this item 234 | 235 | Parameters: 236 | 237 | - ``clear_acked_data`` 238 | - ``max_delete`` (defaults to 1000): This is the LIMIT. How many items to delete. 239 | - ``keep_latest`` (defaults to 1000): This is the OFFSET. How many recent items to keep. 240 | - ``clear_ack_failed`` (defaults to False): Clears the ``AckStatus.ack_failed`` in addition to the ``AckStatus.ack``. 241 | 242 | - ``get`` 243 | - ``raw`` (defaults to False): Returns the metadata along with the record, which includes the id (``pqid``) and timestamp. On the SQLiteAckQueue, the raw results can be ack, nack, ack_failed similar to the normal return. 244 | - ``id`` (defaults to None): Accepts an `id` or a raw item containing ``pqid``. Will select the item based on the row id. 245 | - ``next_in_order`` (defaults to False): Requires the ``id`` attribute. This option tells the SQLiteAckQueue/UniqueAckQ to get the next item based on ``id``, not the first available. This allows the user to get, nack, get, nack and progress down the queue, instead of continuing to get the same nack'd item over again. 246 | 247 | ``raw`` example: 248 | 249 | .. code-block:: python 250 | 251 | >>> q.put('val1') 252 | >>> d = q.get(raw=True) 253 | >>> print(d) 254 | >>> {'pqid': 1, 'data': 'val1', 'timestamp': 1616719225.012912} 255 | >>> q.ack(d) 256 | 257 | ``next_in_order`` example: 258 | 259 | .. code-block:: python 260 | 261 | >>> q.put("val1") 262 | >>> q.put("val2") 263 | >>> q.put("val3") 264 | >>> item = q.get() 265 | >>> id = q.nack(item) 266 | >>> item = q.get(id=id, next_in_order=True) 267 | >>> print(item) 268 | >>> val2 269 | 270 | 271 | Note: 272 | 273 | 1. The SQLiteAckQueue always uses "auto_commit=True". 274 | 2. The Queue could be set in non-block style, e.g. "SQLiteAckQueue.get(block=False, timeout=5)". 275 | 3. ``UniqueAckQ`` only allows for unique items 276 | 277 | Example usage with a file based queue 278 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 279 | 280 | Parameters: 281 | 282 | - ``path``: specifies the directory wher enqueued data persisted. 283 | - ``maxsize``: indicates the maximum size stored in the queue, if maxsize<=0 the queue is unlimited. 284 | - ``chunksize``: indicates how many entries should exist in each chunk file on disk. When a all entries in a chunk file was dequeued by get(), the file would be removed from filesystem. 285 | - ``tempdir``: indicates where temporary files should be stored. The tempdir has to be located on the same disk as the enqueued data in order to obtain atomic operations. 286 | - ``serializer``: controls how enqueued data is serialized. 287 | - ``auto_save``: `True` or `False`. By default, the change is only persisted when task_done() is called. If autosave is enabled, info data is persisted immediately when get() is called. Adding data to the queue with put() will always persist immediately regardless of this setting. 288 | 289 | .. code-block:: python 290 | 291 | >>> from persistqueue import Queue 292 | >>> q = Queue("mypath") 293 | >>> q.put('a') 294 | >>> q.put('b') 295 | >>> q.put('c') 296 | >>> q.get() 297 | 'a' 298 | >>> q.task_done() 299 | 300 | 301 | Close the python console, and then we restart the queue from the same path, 302 | 303 | .. code-block:: python 304 | 305 | >>> from persistqueue import Queue 306 | >>> q = Queue('mypath') 307 | >>> q.get() 308 | 'b' 309 | >>> q.task_done() 310 | 311 | Example usage with an auto-saving file based queue 312 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 313 | 314 | *Available since: v0.5.0* 315 | 316 | By default, items added to the queue are persisted during the ``put()`` call, 317 | and items removed from a queue are only persisted when ``task_done()`` is 318 | called. 319 | 320 | .. code-block:: python 321 | 322 | >>> from persistqueue import Queue 323 | >>> q = Queue("mypath") 324 | >>> q.put('a') 325 | >>> q.put('b') 326 | >>> q.get() 327 | 'a' 328 | >>> q.get() 329 | 'b' 330 | 331 | After exiting and restarting the queue from the same path, we see the items 332 | remain in the queue, because ``task_done()`` wasn't called before. 333 | 334 | .. code-block:: python 335 | 336 | >>> from persistqueue import Queue 337 | >>> q = Queue('mypath') 338 | >>> q.get() 339 | 'a' 340 | >>> q.get() 341 | 'b' 342 | 343 | This can be advantageous. For example, if your program crashes before finishing 344 | processing an item, it will remain in the queue after restarting. You can also 345 | spread out the ``task_done()`` calls for performance reasons to avoid lots of 346 | individual writes. 347 | 348 | Using ``autosave=True`` on a file based queue will automatically save on every 349 | call to ``get()``. Calling ``task_done()`` is not necessary, but may still be 350 | used to ``join()`` against the queue. 351 | 352 | .. code-block:: python 353 | 354 | >>> from persistqueue import Queue 355 | >>> q = Queue("mypath", autosave=True) 356 | >>> q.put('a') 357 | >>> q.put('b') 358 | >>> q.get() 359 | 'a' 360 | 361 | After exiting and restarting the queue from the same path, only the second item 362 | remains: 363 | 364 | .. code-block:: python 365 | 366 | >>> from persistqueue import Queue 367 | >>> q = Queue('mypath', autosave=True) 368 | >>> q.get() 369 | 'b' 370 | 371 | 372 | Example usage with a SQLite3 based dict 373 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 374 | 375 | .. code-block:: python 376 | 377 | >>> from persisitqueue import PDict 378 | >>> q = PDict("testpath", "testname") 379 | >>> q['key1'] = 123 380 | >>> q['key2'] = 321 381 | >>> q['key1'] 382 | 123 383 | >>> len(q) 384 | 2 385 | >>> del q['key1'] 386 | >>> q['key1'] 387 | Traceback (most recent call last): 388 | File "", line 1, in 389 | File "persistqueue\pdict.py", line 58, in __getitem__ 390 | raise KeyError('Key: {} not exists.'.format(item)) 391 | KeyError: 'Key: key1 not exists.' 392 | 393 | Close the console and restart the PDict 394 | 395 | 396 | .. code-block:: python 397 | 398 | >>> from persisitqueue import PDict 399 | >>> q = PDict("testpath", "testname") 400 | >>> q['key2'] 401 | 321 402 | 403 | 404 | Multi-thread usage for **SQLite3** based queue 405 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 406 | 407 | .. code-block:: python 408 | 409 | from persistqueue import FIFOSQLiteQueue 410 | 411 | q = FIFOSQLiteQueue(path="./test", multithreading=True) 412 | 413 | def worker(): 414 | while True: 415 | item = q.get() 416 | do_work(item) 417 | 418 | for i in range(num_worker_threads): 419 | t = Thread(target=worker) 420 | t.daemon = True 421 | t.start() 422 | 423 | for item in source(): 424 | q.put(item) 425 | 426 | 427 | multi-thread usage for **Queue** 428 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 429 | 430 | .. code-block:: python 431 | 432 | from persistqueue import Queue 433 | 434 | q = Queue() 435 | 436 | def worker(): 437 | while True: 438 | item = q.get() 439 | do_work(item) 440 | q.task_done() 441 | 442 | for i in range(num_worker_threads): 443 | t = Thread(target=worker) 444 | t.daemon = True 445 | t.start() 446 | 447 | for item in source(): 448 | q.put(item) 449 | 450 | q.join() # block until all tasks are done 451 | 452 | Example usage with a MySQL based queue 453 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 454 | 455 | *Available since: v0.8.0* 456 | 457 | .. code-block:: python 458 | 459 | >>> import persistqueue 460 | >>> db_conf = { 461 | >>> "host": "127.0.0.1", 462 | >>> "user": "user", 463 | >>> "passwd": "passw0rd", 464 | >>> "db_name": "testqueue", 465 | >>> # "name": "", 466 | >>> "port": 3306 467 | >>> } 468 | >>> q = persistqueue.MySQLQueue(name="testtable", **db_conf) 469 | >>> q.put('str1') 470 | >>> q.put('str2') 471 | >>> q.put('str3') 472 | >>> q.get() 473 | 'str1' 474 | >>> del q 475 | 476 | 477 | Close the console, and then recreate the queue: 478 | 479 | .. code-block:: python 480 | 481 | >>> import persistqueue 482 | >>> q = persistqueue.MySQLQueue(name="testtable", **db_conf) 483 | >>> q.get() 484 | 'str2' 485 | >>> 486 | 487 | 488 | 489 | **note** 490 | 491 | Due to the limitation of file queue described in issue `#89 `_, 492 | `task_done` in one thread may acknowledge items in other threads which should not be. Considering the `SQLiteAckQueue` if you have such requirement. 493 | 494 | 495 | Serialization via msgpack/cbor/json 496 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 497 | - v0.4.1: Currently only available for file based Queue 498 | - v0.4.2: Also available for SQLite3 based Queues 499 | 500 | .. code-block:: python 501 | 502 | >>> from persistqueue 503 | >>> q = persistqueue.Queue('mypath', serializer=persistqueue.serializers.msgpack) 504 | >>> # via cbor2 505 | >>> # q = persistqueue.Queue('mypath', serializer=persistqueue.serializers.cbor2) 506 | >>> # via json 507 | >>> # q = Queue('mypath', serializer=persistqueue.serializers.json) 508 | >>> q.get() 509 | 'b' 510 | >>> q.task_done() 511 | 512 | Explicit resource reclaim 513 | ^^^^^^^^^^^^^^^^^^^^^^^^^ 514 | 515 | For some reasons, an application may require explicit reclamation for file 516 | handles or sql connections before end of execution. In these cases, user can 517 | simply call: 518 | .. code-block:: python 519 | 520 | q = Queue() # or q = persistqueue.SQLiteQueue('mypath', auto_commit=True) 521 | del q 522 | 523 | 524 | to reclaim related file handles or sql connections. 525 | 526 | Tips 527 | ---- 528 | 529 | ``task_done`` is required both for file based queue and SQLite3 based queue (when ``auto_commit=False``) 530 | to persist the cursor of next ``get`` to the disk. 531 | 532 | 533 | Performance impact 534 | ------------------ 535 | 536 | - **WAL** 537 | 538 | Starting on v0.3.2, the ``persistqueue`` is leveraging the sqlite3 builtin feature 539 | `WAL `_ which can improve the performance 540 | significantly, a general testing indicates that ``persistqueue`` is 2-4 times 541 | faster than previous version. 542 | 543 | - **auto_commit=False** 544 | 545 | Since persistqueue v0.3.0, a new parameter ``auto_commit`` is introduced to tweak 546 | the performance for sqlite3 based queues as needed. When specify ``auto_commit=False``, user 547 | needs to perform ``queue.task_done()`` to persist the changes made to the disk since 548 | last ``task_done`` invocation. 549 | 550 | - **pickle protocol selection** 551 | 552 | From v0.3.6, the ``persistqueue`` will select ``Protocol version 2`` for python2 and ``Protocol version 4`` for python3 553 | respectively. This selection only happens when the directory is not present when initializing the queue. 554 | 555 | Tests 556 | ----- 557 | 558 | *persist-queue* use ``tox`` to trigger tests. 559 | 560 | - Unit test 561 | 562 | .. code-block:: console 563 | 564 | tox -e 565 | 566 | Available ````: ``py27``, ``py34``, ``py35``, ``py36``, ``py37`` 567 | 568 | 569 | - PEP8 check 570 | 571 | .. code-block:: console 572 | 573 | tox -e pep8 574 | 575 | 576 | `pyenv `_ is usually a helpful tool to manage multiple versions of Python. 577 | 578 | Caution 579 | ------- 580 | 581 | Currently, the atomic operation is supported on Windows while still in experimental, 582 | That's saying, the data in ``persistqueue.Queue`` could be in unreadable state when an incidental failure occurs during ``Queue.task_done``. 583 | 584 | **DO NOT put any critical data on persistqueue.queue on Windows**. 585 | 586 | 587 | Contribution 588 | ------------ 589 | 590 | Simply fork this repo and send PR for your code change(also tests to cover your change), remember to give a title and description of your PR. I am willing to 591 | enhance this project with you :). 592 | 593 | 594 | License 595 | ------- 596 | 597 | `BSD `_ 598 | 599 | Contributors 600 | ------------ 601 | 602 | `Contributors `_ 603 | 604 | FAQ 605 | --- 606 | 607 | * ``sqlite3.OperationalError: database is locked`` is raised. 608 | 609 | persistqueue open 2 connections for the db if ``multithreading=True``, the 610 | SQLite database is locked until that transaction is committed. The ``timeout`` 611 | parameter specifies how long the connection should wait for the lock to go away 612 | until raising an exception. Default time is **10**, increase ``timeout`` 613 | when creating the queue if above error occurs. 614 | 615 | * sqlite3 based queues are not thread-safe. 616 | 617 | The sqlite3 queues are heavily tested under multi-threading environment, if you find it's not thread-safe, please 618 | make sure you set the ``multithreading=True`` when initializing the queue before submitting new issue:). 619 | -------------------------------------------------------------------------------- /RELEASENOTE.md: -------------------------------------------------------------------------------- 1 | # 1.0.0-alpha 2 | 3 | 1. Only Python3.x series are offically supported by persistqueue, since Python 2 was no longer under maintenance since 2020 4 | 5 | 2. `persistqueue.Queue` using `serializer=persistqueue.serializers.pickle` created under python2 was no longer able to be read by persist queue after 1.0.0 6 | as the default pickle version changed from `2` to `4` 7 | 8 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | # image: 2 | # - Visual Studio 2017 # contains python2.7 ~ python3.8 and mysql 5.7 3 | # - Visual Studio 2022 # contains python3.7 ~ python3.12 and mysql 8.0 4 | 5 | # services: # mysql is now MySQL5.7 now for MySQL8.0 6 | # - mysql 7 | init: 8 | - ps: | 9 | if ($env:APPVEYOR_BUILD_WORKER_IMAGE -eq "Visual Studio 2022") 10 | { 11 | Start-Service MySQL80 12 | } 13 | else 14 | { 15 | Start-Service MySQL57 16 | } 17 | 18 | 19 | environment: 20 | 21 | matrix: 22 | 23 | # For Python versions available on Appveyor, see 24 | # http://www.appveyor.com/docs/installed-software#python 25 | # The list here is complete (excluding Python 2.6, which 26 | # isn't covered by this document) at the time of writing. 27 | - TOXENV: "pep8" 28 | PYTHON: "C:\\Python38-x64" 29 | DISTUTILS_USE_SDK: "1" 30 | APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 31 | 32 | # - TOXENV: "py27" 33 | # PYTHON: "C:\\Python27-x64" 34 | # DISTUTILS_USE_SDK: "1" 35 | # - TOXENV: "py35" 36 | # PYTHON: "C:\\Python35-x64" 37 | # DISTUTILS_USE_SDK: "1" 38 | # APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 39 | 40 | # - TOXENV: "py36" 41 | # PYTHON: "C:\\Python36-x64" 42 | # DISTUTILS_USE_SDK: "1" 43 | # APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 44 | # - TOXENV: "py37" 45 | # PYTHON: "C:\\Python37-x64" 46 | # DISTUTILS_USE_SDK: "1" 47 | # APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 48 | - TOXENV: "py38" 49 | PYTHON: "C:\\Python38-x64" 50 | DISTUTILS_USE_SDK: "1" 51 | APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 52 | - TOXENV: "py39" 53 | PYTHON: "C:\\Python39-x64" 54 | DISTUTILS_USE_SDK: "1" 55 | APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022 56 | - TOXENV: "py310" 57 | PYTHON: "C:\\Python310-x64" 58 | DISTUTILS_USE_SDK: "1" 59 | APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022 60 | - TOXENV: "py311" 61 | PYTHON: "C:\\Python311-x64" 62 | DISTUTILS_USE_SDK: "1" 63 | APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022 64 | - TOXENV: "py312" 65 | PYTHON: "C:\\Python312-x64" 66 | APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022 67 | DISTUTILS_USE_SDK: "1" 68 | - TOXENV: "cover" 69 | PYTHON: "C:\\Python38-x64" 70 | DISTUTILS_USE_SDK: "1" 71 | APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 72 | 73 | install: 74 | # We need wheel installed to build wheels 75 | - "%PYTHON%\\python.exe -m pip install tox" 76 | 77 | build: false 78 | 79 | test_script: 80 | # Put your test command here. 81 | # If you don't need to build C extensions on 64-bit Python 3.3 or 3.4, 82 | # you can remove "build.cmd" from the front of the command, as it's 83 | # only needed to support those cases. 84 | # Note that you must use the environment variable %PYTHON% to refer to 85 | # the interpreter you're using - Appveyor does not do anything special 86 | # to put the Python evrsion you want to use on PATH. 87 | - ps: | 88 | if ($env:APPVEYOR_BUILD_WORKER_IMAGE -eq "Visual Studio 2017") 89 | { 90 | $env:MYSQL_PWD="Password12!" 91 | $cmd = '"C:\Program Files\MySQL\MySQL Server 5.7\bin\mysql" -e "create database testqueue;" --user=root' 92 | iex "& $cmd" 93 | } 94 | else 95 | { 96 | $env:MYSQL_PWD="Password12!" 97 | $cmd = '"C:\Program Files\MySQL\MySQL Server 8.0\bin\mysql" -e "create database testqueue;" --user=root' 98 | iex "& $cmd" 99 | } 100 | 101 | - | 102 | echo image: %APPVEYOR_BUILD_WORKER_IMAGE%, tox:%TOXENV% 103 | "%PYTHON%\\Scripts\\tox.exe" 104 | 105 | #on_success: 106 | # You can use this step to upload your artifacts to a public website. 107 | # See Appveyor's documentation for more details. Or you can simply 108 | # access your wheels from the Appveyor "artifacts" tab for your build. 109 | -------------------------------------------------------------------------------- /benchmark/run_benchmark.py: -------------------------------------------------------------------------------- 1 | """This file provides tests to benchmark performance sqlite/file queue 2 | on specific hardware. User can easily evaluate the performance by running this 3 | file directly via `python run_benchmark.py` 4 | """ 5 | from persistqueue import SQLiteQueue 6 | from persistqueue import Queue 7 | import tempfile 8 | import time 9 | 10 | BENCHMARK_COUNT = 100 11 | 12 | 13 | def time_it(func): 14 | def _exec(*args, **kwargs): 15 | start = time.time() 16 | func(*args, **kwargs) 17 | end = time.time() 18 | print( 19 | "\t{} => time used: {:.4f} seconds.".format( 20 | func.__doc__, 21 | (end - start))) 22 | 23 | return _exec 24 | 25 | 26 | class FileQueueBench(object): 27 | """Benchmark File queue performance.""" 28 | 29 | def __init__(self, prefix=None): 30 | self.path = prefix 31 | 32 | @time_it 33 | def benchmark_file_write(self): 34 | """Writing items.""" 35 | 36 | self.path = tempfile.mkdtemp('b_file_10000') 37 | q = Queue(self.path) 38 | for i in range(BENCHMARK_COUNT): 39 | q.put('bench%d' % i) 40 | assert q.qsize() == BENCHMARK_COUNT 41 | 42 | @time_it 43 | def benchmark_file_read_write_false(self): 44 | """Writing and reading items(1 task_done).""" 45 | 46 | self.path = tempfile.mkdtemp('b_file_10000') 47 | q = Queue(self.path) 48 | for i in range(BENCHMARK_COUNT): 49 | q.put('bench%d' % i) 50 | 51 | for i in range(BENCHMARK_COUNT): 52 | q.get() 53 | q.task_done() 54 | assert q.qsize() == 0 55 | 56 | @time_it 57 | def benchmark_file_read_write_autosave(self): 58 | """Writing and reading items(autosave).""" 59 | 60 | self.path = tempfile.mkdtemp('b_file_10000') 61 | q = Queue(self.path, autosave=True) 62 | for i in range(BENCHMARK_COUNT): 63 | q.put('bench%d' % i) 64 | 65 | for i in range(BENCHMARK_COUNT): 66 | q.get() 67 | assert q.qsize() == 0 68 | 69 | @time_it 70 | def benchmark_file_read_write_true(self): 71 | """Writing and reading items(many task_done).""" 72 | 73 | self.path = tempfile.mkdtemp('b_file_10000') 74 | q = Queue(self.path) 75 | for i in range(BENCHMARK_COUNT): 76 | q.put('bench%d' % i) 77 | 78 | for i in range(BENCHMARK_COUNT): 79 | q.get() 80 | q.task_done() 81 | assert q.qsize() == 0 82 | 83 | @classmethod 84 | def run(cls): 85 | print(cls.__doc__) 86 | ins = cls() 87 | for name in sorted(cls.__dict__): 88 | if name.startswith('benchmark'): 89 | func = getattr(ins, name) 90 | func() 91 | 92 | 93 | class Sqlite3QueueBench(object): 94 | """Benchmark Sqlite3 queue performance.""" 95 | 96 | @time_it 97 | def benchmark_sqlite_write(self): 98 | """Writing items.""" 99 | 100 | self.path = tempfile.mkdtemp('b_sql_10000') 101 | q = SQLiteQueue(self.path, auto_commit=False) 102 | for i in range(BENCHMARK_COUNT): 103 | q.put('bench%d' % i) 104 | 105 | assert q.qsize() == BENCHMARK_COUNT 106 | 107 | @time_it 108 | def benchmark_sqlite_read_write_false(self): 109 | """Writing and reading items(1 task_done).""" 110 | self.path = tempfile.mkdtemp('b_sql_10000') 111 | q = SQLiteQueue(self.path, auto_commit=False) 112 | for i in range(BENCHMARK_COUNT): 113 | q.put('bench%d' % i) 114 | for i in range(BENCHMARK_COUNT): 115 | q.get() 116 | q.task_done() 117 | assert q.qsize() == 0 118 | 119 | @time_it 120 | def benchmark_sqlite_read_write_true(self): 121 | """Writing and reading items(many task_done).""" 122 | self.path = tempfile.mkdtemp('b_sql_10000') 123 | q = SQLiteQueue(self.path, auto_commit=True) 124 | for i in range(BENCHMARK_COUNT): 125 | q.put('bench%d' % i) 126 | 127 | for i in range(BENCHMARK_COUNT): 128 | q.get() 129 | q.task_done() 130 | assert q.qsize() == 0 131 | 132 | @classmethod 133 | def run(cls): 134 | print(cls.__doc__) 135 | ins = cls() 136 | for name in sorted(cls.__dict__): 137 | 138 | if name.startswith('benchmark'): 139 | func = getattr(ins, name) 140 | func() 141 | 142 | 143 | if __name__ == '__main__': 144 | import sys 145 | 146 | if len(sys.argv) > 1: 147 | BENCHMARK_COUNT = int(sys.argv[1]) 148 | print(" = {}".format(BENCHMARK_COUNT)) 149 | file_bench = FileQueueBench() 150 | file_bench.run() 151 | sql_bench = Sqlite3QueueBench() 152 | sql_bench.run() 153 | -------------------------------------------------------------------------------- /extra-requirements.txt: -------------------------------------------------------------------------------- 1 | msgpack>=0.5.6 2 | cbor2>=5.2.0 3 | PyMySQL 4 | DBUtils<3.0.0 # since 3.0.0 no longer supports Python2.x 5 | -------------------------------------------------------------------------------- /persistqueue/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'Peter Wang' 2 | __license__ = 'BSD' 3 | __version__ = '1.0.0' 4 | 5 | # Relative imports assuming the current package structure 6 | from .exceptions import Empty, Full # noqa: F401 7 | from .queue import Queue # noqa: F401 8 | import logging 9 | log = logging.getLogger(__name__) 10 | 11 | # Attempt to import optional components, logging if not found. 12 | try: 13 | from .pdict import PDict # noqa: F401 14 | from .sqlqueue import ( # noqa: F401 15 | SQLiteQueue, 16 | FIFOSQLiteQueue, 17 | FILOSQLiteQueue, 18 | UniqueQ 19 | ) 20 | from .sqlackqueue import ( # noqa: F401 21 | SQLiteAckQueue, 22 | FIFOSQLiteAckQueue, 23 | FILOSQLiteAckQueue, 24 | UniqueAckQ, 25 | AckStatus 26 | ) 27 | except ImportError: 28 | # If sqlite3 is not available, log a message. 29 | log.info("No sqlite3 module found, sqlite3 based queues are not available") 30 | 31 | try: 32 | from .mysqlqueue import MySQLQueue # noqa: F401 33 | except ImportError: 34 | # failed due to DBUtils not installed via extra-requirements.txt 35 | log.info("DBUtils may not be installed, install " 36 | "via 'pip install persist-queue[extra]'") 37 | 38 | # Define what symbols are exported by the module. 39 | __all__ = [ 40 | "Queue", 41 | "SQLiteQueue", 42 | "FIFOSQLiteQueue", 43 | "FILOSQLiteQueue", 44 | "UniqueQ", 45 | "PDict", 46 | "SQLiteAckQueue", 47 | "FIFOSQLiteAckQueue", 48 | "FILOSQLiteAckQueue", 49 | "UniqueAckQ", 50 | "AckStatus", 51 | "MySQLQueue", 52 | "Empty", 53 | "Full", 54 | "__author__", 55 | "__license__", 56 | "__version__" 57 | ] 58 | -------------------------------------------------------------------------------- /persistqueue/exceptions.py: -------------------------------------------------------------------------------- 1 | class Empty(Exception): 2 | """Exception raised when an operation is attempted on an empty queue.""" 3 | pass 4 | 5 | 6 | class Full(Exception): 7 | """Exception raised when an attempt is made to add an item to a full 8 | container.""" 9 | pass 10 | -------------------------------------------------------------------------------- /persistqueue/mysqlqueue.py: -------------------------------------------------------------------------------- 1 | from dbutils.pooled_db import PooledDB 2 | import threading 3 | import time as _time 4 | import persistqueue 5 | from .sqlbase import SQLBase 6 | from typing import Any, Optional 7 | 8 | 9 | class MySQLQueue(SQLBase): 10 | """Mysql(or future standard dbms) based FIFO queue.""" 11 | _TABLE_NAME = 'queue' 12 | _KEY_COLUMN = '_id' # the name of the key column, used in DB CRUD 13 | # SQL to create a table 14 | _SQL_CREATE = ( 15 | 'CREATE TABLE IF NOT EXISTS {table_name} (' 16 | '{key_column} INTEGER PRIMARY KEY AUTO_INCREMENT, ' 17 | 'data BLOB, timestamp FLOAT)') 18 | # SQL to insert a record 19 | _SQL_INSERT = 'INSERT INTO {table_name} (data, timestamp) VALUES (%s, %s)' 20 | # SQL to select a record 21 | _SQL_SELECT_ID = ( 22 | 'SELECT {key_column}, data, timestamp FROM {table_name} WHERE' 23 | ' {key_column} = {rowid}' 24 | ) 25 | _SQL_SELECT = ( 26 | 'SELECT {key_column}, data, timestamp FROM {table_name} ' 27 | 'ORDER BY {key_column} ASC LIMIT 1' 28 | ) 29 | _SQL_SELECT_WHERE = ( 30 | 'SELECT {key_column}, data, timestamp FROM {table_name} WHERE' 31 | ' {column} {op} %s ORDER BY {key_column} ASC LIMIT 1 ' 32 | ) 33 | _SQL_UPDATE = 'UPDATE {table_name} SET data = %s WHERE {key_column} = %s' 34 | _SQL_DELETE = 'DELETE FROM {table_name} WHERE {key_column} {op} %s' 35 | 36 | def __init__( 37 | self, 38 | host: str, 39 | user: str, 40 | passwd: str, 41 | db_name: str, 42 | name: Optional[str] = None, 43 | port: int = 3306, 44 | charset: str = 'utf8mb4', 45 | auto_commit: bool = True, 46 | serializer: Any = persistqueue.serializers.pickle, 47 | ) -> None: 48 | super(MySQLQueue, self).__init__() 49 | self.name = name if name else "sql" 50 | self.host = host 51 | self.user = user 52 | self.passwd = passwd 53 | self.db_name = db_name 54 | self.port = port 55 | self.charset = charset 56 | self._serializer = serializer 57 | self.auto_commit = auto_commit 58 | self.tran_lock = threading.Lock() 59 | self.put_event = threading.Event() 60 | self.action_lock = threading.Lock() 61 | self._connection_pool = None 62 | self._getter = None 63 | self._putter = None 64 | self._new_db_connection() 65 | self._init() 66 | 67 | def _new_db_connection(self) -> None: 68 | try: 69 | import pymysql 70 | except ImportError: 71 | print("Please install mysql library via 'pip install PyMySQL'") 72 | raise 73 | db_pool = PooledDB(pymysql, 2, 10, 5, 10, True, 74 | host=self.host, port=self.port, user=self.user, 75 | passwd=self.passwd, database=self.db_name, 76 | charset=self.charset) 77 | self._connection_pool = db_pool 78 | conn = db_pool.connection() 79 | cursor = conn.cursor() 80 | cursor.execute("SELECT VERSION()") 81 | _ = cursor.fetchone() 82 | cursor.execute(self._sql_create) 83 | conn.commit() 84 | cursor.execute("use %s" % self.db_name) 85 | self._putter = MySQLConn(queue=self) 86 | self._getter = self._putter 87 | 88 | def put(self, item: Any, block: bool = True) -> int: 89 | # block kwarg is noop and only here to align with python's queue 90 | obj = self._serializer.dumps(item) 91 | _id = self._insert_into(obj, _time.time()) 92 | self.total += 1 93 | self.put_event.set() 94 | return _id 95 | 96 | def put_nowait(self, item: Any) -> int: 97 | return self.put(item, block=False) 98 | 99 | def _init(self) -> None: 100 | self.action_lock = threading.Lock() 101 | if not self.auto_commit: 102 | head = self._select() 103 | if head: 104 | self.cursor = head[0] - 1 105 | else: 106 | self.cursor = 0 107 | self.total = self._count() 108 | 109 | def get_pooled_conn(self) -> Any: 110 | return self._connection_pool.connection() 111 | 112 | 113 | class MySQLConn: 114 | """MySqlConn defines a common structure for 115 | both mysql and sqlite3 connections. 116 | used to mitigate the interface differences between drivers/db 117 | """ 118 | 119 | def __init__(self, 120 | queue: Optional[MySQLQueue] = None, 121 | conn: Optional[Any] = None) -> None: 122 | self._queue = queue 123 | if queue is not None: 124 | self._conn = queue.get_pooled_conn() 125 | else: 126 | self._conn = conn 127 | self._cursor = None 128 | self.closed = False 129 | 130 | def __enter__(self) -> Any: 131 | self._cursor = self._conn.cursor() 132 | return self._conn 133 | 134 | def __exit__(self, 135 | exc_type: Optional[type], 136 | exc_val: Optional[BaseException], 137 | exc_tb: Optional[Any]) -> None: 138 | # do not commit() but to close() , keep same behavior 139 | # with dbutils 140 | self._cursor.close() 141 | 142 | def execute(self, *args: Any, **kwargs: Any) -> Any: 143 | if self._queue is not None: 144 | conn = self._queue.get_pooled_conn() 145 | else: 146 | conn = self._conn 147 | cursor = conn.cursor() 148 | cursor.execute(*args, **kwargs) 149 | return cursor 150 | 151 | def close(self) -> None: 152 | if not self.closed: 153 | self._conn.close() 154 | self.closed = True 155 | 156 | def commit(self) -> None: 157 | if not self.closed: 158 | self._conn.commit() 159 | -------------------------------------------------------------------------------- /persistqueue/pdict.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import sqlite3 3 | from persistqueue import sqlbase 4 | from typing import Any, Iterator 5 | 6 | log = logging.getLogger(__name__) 7 | 8 | 9 | class PDict(sqlbase.SQLiteBase, dict): 10 | _TABLE_NAME = 'dict' 11 | _KEY_COLUMN = 'key' 12 | _SQL_CREATE = ('CREATE TABLE IF NOT EXISTS {table_name} (' 13 | '{key_column} TEXT PRIMARY KEY, data BLOB)') 14 | _SQL_INSERT = 'INSERT INTO {table_name} (key, data) VALUES (?, ?)' 15 | _SQL_SELECT = ('SELECT {key_column}, data FROM {table_name} ' 16 | 'WHERE {key_column} = ?') 17 | _SQL_UPDATE = 'UPDATE {table_name} SET data = ? WHERE {key_column} = ?' 18 | _SQL_DELETE = 'DELETE FROM {table_name} WHERE {key_column} {op} ?' 19 | 20 | def __init__(self, 21 | path: str, 22 | name: str, 23 | multithreading: bool = False) -> None: 24 | # PDict is always auto_commit=True 25 | super().__init__(path, name=name, 26 | multithreading=multithreading, 27 | auto_commit=True) 28 | 29 | def __iter__(self) -> Iterator: 30 | raise NotImplementedError('Not supported.') 31 | 32 | def keys(self) -> Iterator: 33 | raise NotImplementedError('Not supported.') 34 | 35 | def iterkeys(self) -> Iterator: 36 | raise NotImplementedError('Not supported.') 37 | 38 | def values(self) -> Iterator: 39 | raise NotImplementedError('Not supported.') 40 | 41 | def itervalues(self) -> Iterator: 42 | raise NotImplementedError('Not supported.') 43 | 44 | def iteritems(self) -> Iterator: 45 | raise NotImplementedError('Not supported.') 46 | 47 | def items(self) -> Iterator: 48 | raise NotImplementedError('Not supported.') 49 | 50 | def __contains__(self, item: Any) -> bool: 51 | row = self._select(item) 52 | return row is not None 53 | 54 | def __setitem__(self, key: Any, value: Any) -> None: 55 | obj = self._serializer.dumps(value) 56 | try: 57 | self._insert_into(key, obj) 58 | except sqlite3.IntegrityError: 59 | self._update(key, obj) 60 | 61 | def __getitem__(self, item: Any) -> Any: 62 | row = self._select(item) 63 | if row: 64 | return self._serializer.loads(row[1]) 65 | else: 66 | raise KeyError('Key: {} not exists.'.format(item)) 67 | 68 | def get(self, key: Any, default: Any = None) -> Any: 69 | try: 70 | return self[key] 71 | except KeyError: 72 | return default 73 | 74 | def __delitem__(self, key: Any) -> None: 75 | self._delete(key) 76 | 77 | def __len__(self) -> int: 78 | return self._count() 79 | -------------------------------------------------------------------------------- /persistqueue/py.typed: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /persistqueue/queue.py: -------------------------------------------------------------------------------- 1 | """A thread-safe disk based persistent queue in Python.""" 2 | import logging 3 | import os 4 | import tempfile 5 | import threading 6 | from time import time as _time 7 | import persistqueue.serializers.pickle 8 | from persistqueue.exceptions import Empty, Full 9 | from typing import Any, Optional, Tuple, BinaryIO 10 | 11 | log = logging.getLogger(__name__) 12 | 13 | 14 | def _truncate(fn: str, length: int) -> None: 15 | """Truncate the file to a specified length.""" 16 | with open(fn, 'ab+') as f: 17 | f.truncate(length) 18 | 19 | 20 | def atomic_rename(src: str, dst: str) -> None: 21 | """Atomically rename a file from src to dst.""" 22 | os.replace(src, dst) 23 | 24 | 25 | class Queue: 26 | """Thread-safe, persistent queue.""" 27 | 28 | def __init__( 29 | self, 30 | path: str, 31 | maxsize: int = 0, 32 | chunksize: int = 100, 33 | tempdir: Optional[str] = None, 34 | serializer: Any = persistqueue.serializers.pickle, 35 | autosave: bool = False 36 | ) -> None: 37 | """Create a persistent queue object on a given path. 38 | 39 | The argument path indicates a directory where enqueued data should be 40 | persisted. If the directory doesn't exist, one will be created. 41 | If maxsize is <= 0, the queue size is infinite. The optional argument 42 | chunksize indicates how many entries should exist in each chunk file on 43 | disk. 44 | 45 | The tempdir parameter indicates where temporary files should be stored. 46 | The tempdir has to be located on the same disk as the enqueued data in 47 | order to obtain atomic operations. 48 | 49 | The serializer parameter controls how enqueued data is serialized. It 50 | must have methods dump(value, fp) and load(fp). The dump method must 51 | serialize value and write it to fp, and may be called for multiple 52 | values with the same fp. The load method must deserialize and return 53 | one value from fp, and may be called multiple times with the same fp 54 | to read multiple values. 55 | 56 | The autosave parameter controls when data removed from the queue is 57 | persisted. By default, (disabled), the change is only persisted when 58 | task_done() is called. If autosave is enabled, data is persisted 59 | immediately when get() is called. Adding data to the queue with put() 60 | will always persist immediately regardless of this setting. 61 | """ 62 | log.debug('Initializing File based Queue with path {}'.format(path)) 63 | self.path = path 64 | self.chunksize = chunksize 65 | self.tempdir = tempdir 66 | self.maxsize = maxsize 67 | self.serializer = serializer 68 | self.autosave = autosave 69 | self._init(maxsize) 70 | if self.tempdir: 71 | if os.stat(self.path).st_dev != os.stat(self.tempdir).st_dev: 72 | raise ValueError( 73 | "tempdir has to be located on same path filesystem") 74 | else: 75 | fd, tempdir = tempfile.mkstemp() 76 | if os.stat(self.path).st_dev != os.stat(tempdir).st_dev: 77 | self.tempdir = self.path 78 | log.warning("Default tempdir '%(dft_dir)s' is not on the " 79 | "same filesystem with queue path '%(queue_path)s'" 80 | ",defaulting to '%(new_path)s'." % { 81 | "dft_dir": tempdir, 82 | "queue_path": self.path, 83 | "new_path": self.tempdir}) 84 | os.close(fd) 85 | os.remove(tempdir) 86 | self.info = self._loadinfo() 87 | # truncate head in case it contains garbage 88 | hnum, hcnt, hoffset = self.info['head'] 89 | headfn = self._qfile(hnum) 90 | if os.path.exists(headfn): 91 | if hoffset < os.path.getsize(headfn): 92 | _truncate(headfn, hoffset) 93 | # let the head file open 94 | self.headf = self._openchunk(hnum, 'ab+') 95 | tnum, _, toffset = self.info['tail'] 96 | self.tailf = self._openchunk(tnum) 97 | self.tailf.seek(toffset) 98 | # update unfinished tasks with the current number of enqueued tasks 99 | self.unfinished_tasks = self.info['size'] 100 | self.update_info = True 101 | 102 | def _init(self, maxsize: int) -> None: 103 | self.mutex = threading.Lock() 104 | self.not_empty = threading.Condition(self.mutex) 105 | self.not_full = threading.Condition(self.mutex) 106 | self.all_tasks_done = threading.Condition( 107 | self.mutex) 108 | if not os.path.exists(self.path): 109 | os.makedirs(self.path) 110 | 111 | def join(self) -> None: 112 | with self.all_tasks_done: 113 | while self.unfinished_tasks: 114 | self.all_tasks_done.wait() 115 | 116 | def qsize(self) -> int: 117 | with self.mutex: 118 | return self._qsize() 119 | 120 | def _qsize(self) -> int: 121 | return self.info['size'] 122 | 123 | def empty(self) -> bool: 124 | return self.qsize() == 0 125 | 126 | def full(self) -> bool: 127 | return self.qsize() == self.maxsize 128 | 129 | def put(self, item: Any, block: bool = True, 130 | timeout: Optional[float] = None) -> None: 131 | self.not_full.acquire() 132 | try: 133 | if self.maxsize > 0: 134 | if not block: 135 | if self._qsize() == self.maxsize: 136 | raise Full 137 | elif timeout is None: 138 | while self._qsize() == self.maxsize: 139 | self.not_full.wait() 140 | elif timeout < 0: 141 | raise ValueError("'timeout' must be a non-negative number") 142 | else: 143 | endtime = _time() + timeout 144 | while self._qsize() == self.maxsize: 145 | remaining = endtime - _time() 146 | if remaining <= 0.0: 147 | raise Full 148 | self.not_full.wait(remaining) 149 | self._put(item) 150 | self.unfinished_tasks += 1 151 | self.not_empty.notify() 152 | finally: 153 | self.not_full.release() 154 | 155 | def _put(self, item: Any) -> None: 156 | self.serializer.dump(item, self.headf) 157 | self.headf.flush() 158 | hnum, hpos, _ = self.info['head'] 159 | hpos += 1 160 | if hpos == self.info['chunksize']: 161 | hpos = 0 162 | hnum += 1 163 | os.fsync(self.headf.fileno()) 164 | self.headf.close() 165 | self.headf = self._openchunk(hnum, 'ab+') 166 | self.info['size'] += 1 167 | self.info['head'] = [hnum, hpos, self.headf.tell()] 168 | self._saveinfo() 169 | 170 | def put_nowait(self, item: Any) -> None: 171 | self.put(item, False) 172 | 173 | def get(self, block: bool = True, timeout: Optional[float] = None) -> Any: 174 | self.not_empty.acquire() 175 | try: 176 | if not block: 177 | if not self._qsize(): 178 | raise Empty 179 | elif timeout is None: 180 | while not self._qsize(): 181 | self.not_empty.wait() 182 | elif timeout < 0: 183 | raise ValueError("'timeout' must be a non-negative number") 184 | else: 185 | endtime = _time() + timeout 186 | while not self._qsize(): 187 | remaining = endtime - _time() 188 | if remaining <= 0.0: 189 | raise Empty 190 | self.not_empty.wait(remaining) 191 | item = self._get() 192 | self.not_full.notify() 193 | return item 194 | finally: 195 | self.not_empty.release() 196 | 197 | def get_nowait(self) -> Any: 198 | return self.get(False) 199 | 200 | def _get(self) -> Any: 201 | tnum, tcnt, toffset = self.info['tail'] 202 | hnum, hcnt, _ = self.info['head'] 203 | if [tnum, tcnt] >= [hnum, hcnt]: 204 | return None 205 | data = self.serializer.load(self.tailf) 206 | toffset = self.tailf.tell() 207 | tcnt += 1 208 | if tcnt == self.info['chunksize'] and tnum <= hnum: 209 | tcnt = toffset = 0 210 | tnum += 1 211 | self.tailf.close() 212 | self.tailf = self._openchunk(tnum) 213 | self.info['size'] -= 1 214 | self.info['tail'] = [tnum, tcnt, toffset] 215 | if self.autosave: 216 | self._saveinfo() 217 | self.update_info = False 218 | else: 219 | self.update_info = True 220 | return data 221 | 222 | def task_done(self) -> None: 223 | with self.all_tasks_done: 224 | unfinished = self.unfinished_tasks - 1 225 | if unfinished <= 0: 226 | if unfinished < 0: 227 | raise ValueError("task_done() called too many times.") 228 | self.all_tasks_done.notify_all() 229 | self.unfinished_tasks = unfinished 230 | self._task_done() 231 | 232 | def _task_done(self) -> None: 233 | if self.autosave: 234 | return 235 | if self.update_info: 236 | self._saveinfo() 237 | self.update_info = False 238 | 239 | def _openchunk(self, number: int, mode: str = 'rb') -> BinaryIO: 240 | return open(self._qfile(number), mode) 241 | 242 | def _loadinfo(self) -> dict: 243 | infopath = self._infopath() 244 | if os.path.exists(infopath): 245 | with open(infopath, 'rb') as f: 246 | info = self.serializer.load(f) 247 | else: 248 | info = { 249 | 'chunksize': self.chunksize, 250 | 'size': 0, 251 | 'tail': [0, 0, 0], 252 | 'head': [0, 0, 0], 253 | } 254 | return info 255 | 256 | def _gettempfile(self) -> Tuple[int, str]: 257 | if self.tempdir: 258 | return tempfile.mkstemp(dir=self.tempdir) 259 | else: 260 | return tempfile.mkstemp() 261 | 262 | def _saveinfo(self) -> None: 263 | tmpfd, tmpfn = self._gettempfile() 264 | with os.fdopen(tmpfd, "wb") as tmpfo: 265 | self.serializer.dump(self.info, tmpfo) 266 | atomic_rename(tmpfn, self._infopath()) 267 | self._clear_tail_file() 268 | 269 | def _clear_tail_file(self) -> None: 270 | """Remove the tail files whose items were already get.""" 271 | tnum, _, _ = self.info['tail'] 272 | while tnum >= 1: 273 | tnum -= 1 274 | path = self._qfile(tnum) 275 | if os.path.exists(path): 276 | os.remove(path) 277 | else: 278 | break 279 | 280 | def _qfile(self, number: int) -> str: 281 | return os.path.join(self.path, 'q%05d' % number) 282 | 283 | def _infopath(self) -> str: 284 | return os.path.join(self.path, 'info') 285 | 286 | def __del__(self) -> None: 287 | """Handles the removal of queue.""" 288 | for to_close in self.headf, self.tailf: 289 | if to_close and not to_close.closed: 290 | to_close.close() 291 | -------------------------------------------------------------------------------- /persistqueue/serializers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/peter-wangxu/persist-queue/b194f07553379a8319ce1bd0adbb0ad4c610eba9/persistqueue/serializers/__init__.py -------------------------------------------------------------------------------- /persistqueue/serializers/cbor2.py: -------------------------------------------------------------------------------- 1 | """ 2 | A serializer that extends cbor2 to specify recommended parameters and adds a 3 | 4 byte length prefix to store multiple objects per file. 4 | """ 5 | import cbor2 6 | from struct import Struct 7 | from typing import Any, BinaryIO 8 | 9 | # Define the Struct for prefixing serialized objects with their byte length 10 | length_struct = Struct(" None: 14 | """ 15 | Serialize value as cbor2 to a byte-mode file object with a length prefix. 16 | 17 | Args: 18 | value: The Python object to serialize. 19 | fp: A file-like object supporting binary write operations. 20 | sort_keys: If True, the output of dictionaries will be sorted by key. 21 | 22 | Returns: 23 | None 24 | """ 25 | # If sorting is required and the value is a dictionary, sort it by keys 26 | if sort_keys and isinstance(value, dict): 27 | value = {key: value[key] for key in sorted(value)} 28 | packed = cbor2.dumps(value) 29 | length = length_struct.pack(len(packed)) 30 | fp.write(length) 31 | fp.write(packed) 32 | 33 | 34 | def dumps(value: Any, sort_keys: bool = False) -> bytes: 35 | """ 36 | Serialize value as cbor2 to bytes without length prefix. 37 | 38 | Args: 39 | value: The Python object to serialize. 40 | sort_keys: If True, the output of dictionaries will be sorted by key. 41 | 42 | Returns: 43 | A bytes object containing the serialized representation of the value. 44 | """ 45 | # If sorting is required and the value is a dictionary, sort it by keys 46 | if sort_keys and isinstance(value, dict): 47 | value = {key: value[key] for key in sorted(value)} 48 | return cbor2.dumps(value) 49 | 50 | 51 | def load(fp: BinaryIO) -> Any: 52 | """ 53 | Deserialize one cbor2 value from a byte-mode file object 54 | using length prefix. 55 | 56 | Args: 57 | fp: A file-like object supporting binary read operations. 58 | 59 | Returns: 60 | The deserialized Python object. 61 | """ 62 | # Read the 4-byte length prefix and determine the length of the 63 | # serialized object 64 | length = length_struct.unpack(fp.read(4))[0] 65 | # Read the serialized object using the determined length and 66 | # deserialize it 67 | return cbor2.loads(fp.read(length)) 68 | 69 | 70 | def loads(bytes_value: bytes) -> Any: 71 | """ 72 | Deserialize one cbor2 value from bytes. 73 | 74 | Args: 75 | bytes_value: The bytes object containing the serialized representation. 76 | 77 | Returns: 78 | The deserialized Python object. 79 | """ 80 | return cbor2.loads(bytes_value) 81 | -------------------------------------------------------------------------------- /persistqueue/serializers/json.py: -------------------------------------------------------------------------------- 1 | """ 2 | A serializer that extends json to use bytes and uses newlines to store 3 | multiple objects per file. 4 | """ 5 | import json 6 | from typing import Any, BinaryIO 7 | 8 | 9 | def dump(value: Any, fp: BinaryIO, sort_keys: bool = False) -> None: 10 | """Serialize value as json line to a byte-mode file object. 11 | 12 | Args: 13 | value: The Python object to serialize. 14 | fp: A file-like object supporting .write() in binary mode. 15 | sort_keys: If True, the output of dictionaries will be sorted by key. 16 | 17 | Returns: 18 | None 19 | """ 20 | fp.write(json.dumps(value, sort_keys=sort_keys).encode('utf-8')) 21 | fp.write(b"\n") 22 | 23 | 24 | def dumps(value: Any, sort_keys: bool = False) -> bytes: 25 | """Serialize value as json to bytes. 26 | 27 | Args: 28 | value: The Python object to serialize. 29 | sort_keys: If True, the output of dictionaries will be sorted by key. 30 | 31 | Returns: 32 | A json-encoded string converted to bytes. 33 | """ 34 | return json.dumps(value, sort_keys=sort_keys).encode('utf-8') 35 | 36 | 37 | def load(fp: BinaryIO) -> Any: 38 | """Deserialize one json line from a byte-mode file object. 39 | 40 | Args: 41 | fp: A file-like object supporting .readline() in binary mode. 42 | 43 | Returns: 44 | The deserialized Python object. 45 | """ 46 | return json.loads(fp.readline().decode('utf-8')) 47 | 48 | 49 | def loads(bytes_value: bytes) -> Any: 50 | """Deserialize one json value from bytes. 51 | 52 | Args: 53 | bytes_value: The json-encoded bytes to deserialize. 54 | 55 | Returns: 56 | The deserialized Python object. 57 | """ 58 | return json.loads(bytes_value.decode('utf-8')) 59 | -------------------------------------------------------------------------------- /persistqueue/serializers/msgpack.py: -------------------------------------------------------------------------------- 1 | """ 2 | A serializer that extends msgpack to specify recommended parameters and adds a 3 | 4 byte length prefix to store multiple objects per file. 4 | """ 5 | import msgpack 6 | import struct 7 | from typing import Any, BinaryIO, Dict 8 | 9 | 10 | def dump(value: Any, fp: BinaryIO, sort_keys: bool = False) -> None: 11 | """ 12 | Serialize value as msgpack to a byte-mode file object with a length prefix. 13 | 14 | Args: 15 | value: The Python object to serialize. 16 | fp: A file-like object supporting binary write operations. 17 | sort_keys: If True, the output of dictionaries will be sorted by key. 18 | 19 | Returns: 20 | None 21 | """ 22 | if sort_keys and isinstance(value, Dict): 23 | value = {key: value[key] for key in sorted(value)} 24 | packed = msgpack.packb(value, use_bin_type=True) 25 | length = struct.pack(" bytes: 31 | """ 32 | Serialize value as msgpack to bytes. 33 | 34 | Args: 35 | value: The Python object to serialize. 36 | sort_keys: If True, the output of dictionaries will be sorted by key. 37 | 38 | Returns: 39 | A bytes object containing the serialized representation of value. 40 | """ 41 | if sort_keys and isinstance(value, Dict): 42 | value = {key: value[key] for key in sorted(value)} 43 | return msgpack.packb(value, use_bin_type=True) 44 | 45 | 46 | def load(fp: BinaryIO) -> Any: 47 | """ 48 | Deserialize one msgpack value from a byte-mode file object using length 49 | prefix. 50 | 51 | Args: 52 | fp: A file-like object supporting binary read operations. 53 | 54 | Returns: 55 | The deserialized Python object. 56 | """ 57 | length = struct.unpack(" Any: 62 | """ 63 | Deserialize one msgpack value from bytes. 64 | 65 | Args: 66 | bytes_value: A bytes object containing the serialized msgpack data. 67 | 68 | Returns: 69 | The deserialized Python object. 70 | """ 71 | return msgpack.unpackb(bytes_value, use_list=False, raw=False) 72 | -------------------------------------------------------------------------------- /persistqueue/serializers/pickle.py: -------------------------------------------------------------------------------- 1 | """A serializer that extends pickle to change the default protocol.""" 2 | from typing import Any, BinaryIO, Dict 3 | import pickle 4 | import logging 5 | 6 | 7 | log = logging.getLogger(__name__) 8 | 9 | # Retrieve the selected pickle protocol from a common utility module 10 | protocol = 4 # Python 3 uses protocol version 4 or higher 11 | log.info("Selected pickle protocol: '{}'".format(protocol)) 12 | 13 | 14 | def dump(value: Any, fp: BinaryIO, sort_keys: bool = False) -> None: 15 | """ 16 | Serialize value as pickle to a byte-mode file object. 17 | 18 | Args: 19 | value: The Python object to serialize. 20 | fp: A file-like object supporting binary write operations. 21 | sort_keys: If True and if the value is a dictionary, the keys will 22 | be sorted before serialization. 23 | 24 | Returns: 25 | None 26 | """ 27 | if sort_keys and isinstance(value, Dict): 28 | # Sort the dictionary by keys if sort_keys is True 29 | value = {key: value[key] for key in sorted(value)} 30 | pickle.dump(value, fp, protocol=protocol) 31 | 32 | 33 | def dumps(value: Any, sort_keys: bool = False) -> bytes: 34 | """ 35 | Serialize value as pickle to bytes. 36 | 37 | Args: 38 | value: The Python object to serialize. 39 | sort_keys: If True and if the value is a dictionary, the keys will 40 | be sorted before serialization. 41 | 42 | Returns: 43 | A bytes object containing the serialized representation of value. 44 | """ 45 | if sort_keys and isinstance(value, Dict): 46 | # Sort the dictionary by keys if sort_keys is True 47 | value = {key: value[key] for key in sorted(value)} 48 | return pickle.dumps(value, protocol=protocol) 49 | 50 | 51 | def load(fp: BinaryIO) -> Any: 52 | """ 53 | Deserialize one pickle value from a byte-mode file object. 54 | 55 | Args: 56 | fp: A file-like object supporting binary read operations. 57 | 58 | Returns: 59 | The deserialized Python object. 60 | """ 61 | return pickle.load(fp) 62 | 63 | 64 | def loads(bytes_value: bytes) -> Any: 65 | """ 66 | Deserialize one pickle value from bytes. 67 | 68 | Args: 69 | bytes_value: A bytes object containing the serialized pickle data. 70 | 71 | Returns: 72 | The deserialized Python object. 73 | """ 74 | return pickle.loads(bytes_value) 75 | -------------------------------------------------------------------------------- /persistqueue/sqlackqueue.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import sqlite3 3 | import time as _time 4 | import threading 5 | import warnings 6 | from typing import Any, Dict, Optional, Tuple 7 | 8 | from . import sqlbase 9 | from .exceptions import Empty 10 | 11 | sqlite3.enable_callback_tracebacks(True) 12 | log = logging.getLogger(__name__) 13 | 14 | # 10 seconds interval for `wait` of event 15 | TICK_FOR_WAIT = 10 16 | 17 | 18 | class AckStatus: 19 | inited = '0' 20 | ready = '1' 21 | unack = '2' 22 | acked = '5' 23 | ack_failed = '9' 24 | 25 | 26 | class SQLiteAckQueue(sqlbase.SQLiteBase): 27 | """SQLite3 based FIFO queue with ack support.""" 28 | _TABLE_NAME = 'ack_queue' 29 | _KEY_COLUMN = '_id' # the name of the key column, used in DB CRUD 30 | _MAX_ACKED_LENGTH = 1000 # deprecated 31 | # SQL to create a table 32 | _SQL_CREATE = ( 33 | 'CREATE TABLE IF NOT EXISTS {table_name} (' 34 | '{key_column} INTEGER PRIMARY KEY AUTOINCREMENT, ' 35 | 'data BLOB, timestamp FLOAT, status INTEGER)' 36 | ) 37 | # SQL to insert a record 38 | _SQL_INSERT = ( 39 | 'INSERT INTO {table_name} (data, timestamp, status)' 40 | ' VALUES (?, ?, %s)' % AckStatus.inited 41 | ) 42 | # SQL to select a record 43 | _SQL_SELECT_ID = ( 44 | 'SELECT {key_column}, data, timestamp, status FROM {table_name} WHERE' 45 | ' {key_column} = {rowid}' 46 | ) 47 | _SQL_SELECT = ( 48 | 'SELECT {key_column}, data, timestamp, status FROM {table_name} ' 49 | 'WHERE {key_column} > {rowid} AND status < %s ' 50 | 'ORDER BY {key_column} ASC LIMIT 1' % AckStatus.unack 51 | ) 52 | _SQL_MARK_ACK_UPDATE = ( 53 | 'UPDATE {table_name} SET status = ? WHERE {key_column} = ?' 54 | ) 55 | _SQL_SELECT_WHERE = ( 56 | 'SELECT {key_column}, data, timestamp FROM {table_name}' 57 | ' WHERE {key_column} > {rowid} AND status < %s AND' 58 | ' {column} {op} ? ORDER BY {key_column} ASC' 59 | ' LIMIT 1 ' % AckStatus.unack 60 | ) 61 | _SQL_UPDATE = 'UPDATE {table_name} SET data = ? WHERE {key_column} = ?' 62 | 63 | def __init__(self, path: str, auto_resume: bool = True, **kwargs): 64 | super(SQLiteAckQueue, self).__init__(path, **kwargs) 65 | if not self.auto_commit: 66 | warnings.warn("disable auto commit is not supported in ack queue") 67 | self.auto_commit = True 68 | self._unack_cache = {} 69 | if auto_resume: 70 | self.resume_unack_tasks() 71 | 72 | def resume_unack_tasks(self) -> None: 73 | unack_count = self.unack_count() 74 | if unack_count: 75 | log.info("resume %d unack tasks", unack_count) 76 | sql = 'UPDATE {} set status = ? WHERE status = ?'.format( 77 | self._table_name) 78 | with self.tran_lock: 79 | with self._putter as tran: 80 | tran.execute(sql, (AckStatus.ready, AckStatus.unack,)) 81 | self.total = self._count() 82 | 83 | def put(self, item: Any) -> Optional[int]: 84 | obj = self._serializer.dumps(item) 85 | _id = self._insert_into(obj, _time.time()) 86 | self.total += 1 87 | self.put_event.set() 88 | return _id 89 | 90 | def _init(self) -> None: 91 | super(SQLiteAckQueue, self)._init() 92 | self.action_lock = threading.Lock() 93 | self.total = self._count() 94 | 95 | def _count(self) -> int: 96 | sql = 'SELECT COUNT({}) FROM {} WHERE status < ?'.format( 97 | self._key_column, self._table_name 98 | ) 99 | row = self._getter.execute(sql, (AckStatus.unack,)).fetchone() 100 | return row[0] if row else 0 101 | 102 | def _ack_count_via_status(self, status: str) -> int: 103 | sql = 'SELECT COUNT({}) FROM {} WHERE status = ?'.format( 104 | self._key_column, self._table_name 105 | ) 106 | row = self._getter.execute(sql, (status,)).fetchone() 107 | return row[0] if row else 0 108 | 109 | def unack_count(self) -> int: 110 | return self._ack_count_via_status(AckStatus.unack) 111 | 112 | def acked_count(self) -> int: 113 | return self._ack_count_via_status(AckStatus.acked) 114 | 115 | def ready_count(self) -> int: 116 | return self._ack_count_via_status(AckStatus.ready) 117 | 118 | def ack_failed_count(self) -> int: 119 | return self._ack_count_via_status(AckStatus.ack_failed) 120 | 121 | @sqlbase.with_conditional_transaction 122 | def _mark_ack_status(self, key: int, status: str) -> None: 123 | return self._sql_mark_ack_status, (status, key,) 124 | 125 | @sqlbase.with_conditional_transaction 126 | def clear_acked_data( 127 | self, max_delete: int = 1000, keep_latest: int = 1000, 128 | clear_ack_failed: bool = False 129 | ) -> None: 130 | acked_clear_all = '' 131 | acked_to_delete = '' 132 | acked_to_keep = '' 133 | if self._MAX_ACKED_LENGTH != 1000 and not max_delete: 134 | # Added for backward compatibility for 135 | # those that set the _MAX_ACKED_LENGTH 136 | print( 137 | "_MAX_ACKED_LENGTH has been deprecated. " 138 | "Use clear_acked_data(keep_latest=1000, max_delete=1000)" 139 | ) 140 | keep_latest = self._MAX_ACKED_LENGTH 141 | if clear_ack_failed: 142 | acked_clear_all = 'OR status = %s' % AckStatus.ack_failed 143 | if max_delete and max_delete > 0: 144 | acked_to_delete = 'LIMIT %d' % max_delete 145 | if keep_latest and keep_latest > 0: 146 | acked_to_keep = 'OFFSET %d' % keep_latest 147 | sql = """DELETE FROM {table_name} 148 | WHERE {key_column} IN ( 149 | SELECT _id FROM {table_name} 150 | WHERE status = ? {clear_ack_failed} 151 | ORDER BY {key_column} 152 | DESC {acked_to_delete} {acked_to_keep} 153 | )""".format( 154 | table_name=self._table_name, 155 | key_column=self._key_column, 156 | acked_to_delete=acked_to_delete, 157 | acked_to_keep=acked_to_keep, 158 | clear_ack_failed=acked_clear_all, 159 | ) 160 | return sql, AckStatus.acked 161 | 162 | @property 163 | def _sql_mark_ack_status(self) -> str: 164 | return self._SQL_MARK_ACK_UPDATE.format( 165 | table_name=self._table_name, key_column=self._key_column 166 | ) 167 | 168 | def _pop(self, rowid: Optional[int] = None, next_in_order: bool = False, 169 | raw: bool = False) -> Optional[Dict[str, Any]]: 170 | with self.action_lock: 171 | row = self._select(next_in_order=next_in_order, rowid=rowid) 172 | if row and row[0] is not None: 173 | self._mark_ack_status(row[0], AckStatus.unack) 174 | serialized_data = row[1] 175 | item = self._serializer.loads(serialized_data) 176 | self._unack_cache[row[0]] = item 177 | self.total -= 1 178 | if raw: 179 | return {'pqid': row[0], 'data': item, 'timestamp': row[2]} 180 | else: 181 | return item 182 | return None 183 | 184 | def _find_item_id(self, item: Any, search: bool = True) -> Optional[int]: 185 | if item is None: 186 | return None 187 | elif isinstance(item, dict) and "pqid" in item: 188 | return item.get("pqid") 189 | elif search: 190 | for key, value in self._unack_cache.items(): 191 | if value is item: 192 | return key 193 | elif isinstance(item, int) or ( 194 | isinstance(item, str) and item.isnumeric() 195 | ): 196 | return int(item) 197 | log.warning("Item id not Interger and can't find item in unack cache.") 198 | return None 199 | 200 | def _check_id(self, item: Any, id: Optional[int]) -> Tuple[Any, bool]: 201 | if id is not None and item is not None: 202 | raise ValueError("Specify an id or an item, not both.") 203 | elif id is None and item is None: 204 | raise ValueError("Specify an id or an item.") 205 | elif id is not None: 206 | search = False 207 | item = id 208 | else: 209 | search = True 210 | return item, search 211 | 212 | def ack(self, item: Any = None, 213 | id: Optional[int] = None) -> Optional[int]: 214 | 215 | item, search = self._check_id(item, id) 216 | with self.action_lock: 217 | _id = self._find_item_id(item, search) 218 | if _id is None: 219 | return None 220 | self._mark_ack_status(_id, AckStatus.acked) 221 | if _id in self._unack_cache: 222 | self._unack_cache.pop(_id) 223 | return _id 224 | 225 | def ack_failed(self, item: Any = None, 226 | id: Optional[int] = None) -> Optional[int]: 227 | item, search = self._check_id(item, id) 228 | with self.action_lock: 229 | _id = self._find_item_id(item, search) 230 | if _id is None: 231 | return None 232 | self._mark_ack_status(_id, AckStatus.ack_failed) 233 | if _id in self._unack_cache: 234 | self._unack_cache.pop(_id) 235 | return _id 236 | 237 | def nack(self, item: Any = None, 238 | id: Optional[int] = None) -> Optional[int]: 239 | item, search = self._check_id(item, id) 240 | with self.action_lock: 241 | _id = self._find_item_id(item, search) 242 | if _id is None: 243 | return None 244 | self._mark_ack_status(_id, AckStatus.ready) 245 | if _id in self._unack_cache: 246 | self._unack_cache.pop(_id) 247 | self.total += 1 248 | return _id 249 | 250 | def update(self, item: Any, id: Optional[int] = None) -> Optional[int]: 251 | _id = None 252 | if isinstance(item, dict) and "pqid" in item: 253 | _id = item.get("pqid") 254 | item = item.get("data") 255 | if id is not None: 256 | _id = id 257 | if _id is None: 258 | raise ValueError("Provide an id or raw item") 259 | obj = self._serializer.dumps(item) 260 | self._update(_id, obj) 261 | return _id 262 | 263 | def get( 264 | self, block: bool = True, timeout: Optional[float] = None, 265 | id: Optional[int] = None, next_in_order: bool = False, 266 | raw: bool = False) -> Any: 267 | rowid = self._find_item_id(id, search=False) 268 | if rowid is None and next_in_order: 269 | raise ValueError( 270 | "'next_in_order' requires the preceding 'id' be specified." 271 | ) 272 | if next_in_order and not isinstance(next_in_order, bool): 273 | raise ValueError("'next_in_order' must be a boolean (True/False)") 274 | if not block: 275 | serialized = self._pop( 276 | next_in_order=next_in_order, raw=raw, rowid=rowid 277 | ) 278 | if serialized is None: 279 | raise Empty 280 | elif timeout is None: 281 | # block until a put event. 282 | serialized = self._pop( 283 | next_in_order=next_in_order, raw=raw, rowid=rowid 284 | ) 285 | while serialized is None: 286 | self.put_event.clear() 287 | self.put_event.wait(TICK_FOR_WAIT) 288 | serialized = self._pop( 289 | next_in_order=next_in_order, raw=raw, rowid=rowid 290 | ) 291 | elif timeout < 0: 292 | raise ValueError("'timeout' must be a non-negative number") 293 | else: 294 | # block until the timeout reached 295 | endtime = _time.time() + timeout 296 | serialized = self._pop( 297 | next_in_order=next_in_order, raw=raw, rowid=rowid 298 | ) 299 | while serialized is None: 300 | self.put_event.clear() 301 | remaining = endtime - _time.time() 302 | if remaining <= 0.0: 303 | raise Empty 304 | self.put_event.wait( 305 | TICK_FOR_WAIT if TICK_FOR_WAIT < remaining else remaining 306 | ) 307 | serialized = self._pop( 308 | next_in_order=next_in_order, raw=raw, rowid=rowid 309 | ) 310 | return serialized 311 | 312 | def task_done(self) -> None: 313 | """Persist the current state if auto_commit=False.""" 314 | if not self.auto_commit: 315 | self._task_done() 316 | 317 | def queue(self) -> Any: 318 | rows = self._sql_queue() 319 | datarows = [] 320 | for row in rows: 321 | item = { 322 | 'id': row[0], 323 | 'data': self._serializer.loads(row[1]), 324 | 'timestamp': row[2], 325 | 'status': row[3], 326 | } 327 | datarows.append(item) 328 | return datarows 329 | 330 | @property 331 | def size(self) -> int: 332 | return self.total 333 | 334 | def qsize(self) -> int: 335 | return max(0, self.size) 336 | 337 | def active_size(self) -> int: 338 | return max(0, self.size + len(self._unack_cache)) 339 | 340 | def empty(self) -> bool: 341 | return self.size == 0 342 | 343 | def full(self) -> bool: 344 | return False 345 | 346 | def __len__(self) -> int: 347 | return self.size 348 | 349 | 350 | FIFOSQLiteAckQueue = SQLiteAckQueue 351 | 352 | 353 | class FILOSQLiteAckQueue(SQLiteAckQueue): 354 | """SQLite3 based FILO queue with ack support.""" 355 | _TABLE_NAME = 'ack_filo_queue' 356 | # SQL to select a record 357 | _SQL_SELECT = ( 358 | 'SELECT {key_column}, data, timestamp, status FROM {table_name} ' 359 | 'WHERE {key_column} < {rowid} and status < %s ' 360 | 'ORDER BY {key_column} DESC LIMIT 1' % AckStatus.unack 361 | ) 362 | 363 | 364 | class UniqueAckQ(SQLiteAckQueue): 365 | _TABLE_NAME = 'ack_unique_queue' 366 | _SQL_CREATE = ( 367 | 'CREATE TABLE IF NOT EXISTS {table_name} (' 368 | '{key_column} INTEGER PRIMARY KEY AUTOINCREMENT, ' 369 | 'data BLOB, timestamp FLOAT, status INTEGER, UNIQUE (data))' 370 | ) 371 | 372 | def put(self, item: Any) -> Optional[int]: 373 | obj = self._serializer.dumps(item, sort_keys=True) 374 | _id = None 375 | try: 376 | _id = self._insert_into(obj, _time.time()) 377 | except sqlite3.IntegrityError: 378 | pass 379 | else: 380 | self.total += 1 381 | self.put_event.set() 382 | return _id 383 | -------------------------------------------------------------------------------- /persistqueue/sqlbase.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import time as _time 4 | import sqlite3 5 | import threading 6 | from typing import Any, Callable, Tuple, Optional 7 | 8 | from persistqueue.exceptions import Empty 9 | import persistqueue.serializers.pickle 10 | 11 | sqlite3.enable_callback_tracebacks(True) 12 | log = logging.getLogger(__name__) 13 | 14 | # 10 seconds interval for `wait` of event 15 | TICK_FOR_WAIT = 10 16 | 17 | 18 | def with_conditional_transaction(func: Callable) -> Callable: 19 | def _execute(obj: 'SQLBase', *args: Any, **kwargs: Any) -> Any: 20 | # for MySQL, connection pool should be used since db connection is 21 | # basically not thread-safe 22 | _putter = obj._putter 23 | if str(type(obj)).find("MySQLQueue") > 0: 24 | # use fresh connection from pool not the shared one 25 | _putter = obj.get_pooled_conn() 26 | with obj.tran_lock: 27 | with _putter as tran: 28 | # For sqlite3, commit() is called automatically afterwards 29 | # but for other db API, this is not TRUE! 30 | stat, param = func(obj, *args, **kwargs) 31 | s = str(type(tran)) 32 | if s.find("Cursor") > 0: 33 | cur = tran 34 | cur.execute(stat, param) 35 | else: 36 | cur = tran.cursor() 37 | cur.execute(stat, param) 38 | cur.close() 39 | tran.commit() 40 | return cur.lastrowid 41 | 42 | return _execute 43 | 44 | 45 | def commit_ignore_error(conn: sqlite3.Connection) -> None: 46 | """Ignore the error of no transaction is active. 47 | 48 | The transaction may be already committed by user's task_done call. 49 | It's safe to ignore all errors of this kind. 50 | """ 51 | try: 52 | conn.commit() 53 | except sqlite3.OperationalError as ex: 54 | if 'no transaction is active' in str(ex): 55 | log.debug( 56 | 'Not able to commit the transaction, ' 57 | 'may already be committed.' 58 | ) 59 | else: 60 | raise 61 | 62 | 63 | class SQLBase(object): 64 | """SQL base class.""" 65 | 66 | """SQL base class.""" 67 | _TABLE_NAME = 'base' # DB table name 68 | _KEY_COLUMN = '' # the name of the key column, used in DB CRUD 69 | _SQL_CREATE = '' # SQL to create a table 70 | _SQL_UPDATE = '' # SQL to update a record 71 | _SQL_INSERT = '' # SQL to insert a record 72 | _SQL_SELECT = '' # SQL to select a record 73 | _SQL_SELECT_ID = '' # SQL to select a record with criteria 74 | _SQL_SELECT_WHERE = '' # SQL to select a record with criteria 75 | _SQL_DELETE = '' # SQL to delete a record 76 | 77 | def __init__(self) -> None: 78 | self._serializer = persistqueue.serializers.pickle 79 | self.auto_commit = True # Transaction commit behavior 80 | # SQL transaction lock 81 | self.tran_lock = threading.Lock() 82 | # Event signaling new data 83 | self.put_event = threading.Event() 84 | # Lock for atomic actions 85 | self.action_lock = threading.Lock() 86 | self.total = 0 # Total tasks 87 | self.cursor = 0 # Cursor for task processing 88 | # Connection for getting tasks 89 | self._getter = None 90 | # Connection for putting tasks 91 | self._putter = None 92 | 93 | @with_conditional_transaction 94 | def _insert_into(self, *record: Any) -> Tuple[str, Tuple[Any, ...]]: 95 | return self._sql_insert, record 96 | 97 | @with_conditional_transaction 98 | def _update(self, key: Any, *args: Any) -> Tuple[str, Tuple[Any, ...]]: 99 | args = list(args) + [key] 100 | return self._sql_update, args 101 | 102 | @with_conditional_transaction 103 | def _delete(self, key: Any, op: str = '=') -> Tuple[str, Tuple[Any, ...]]: 104 | 105 | sql = self._SQL_DELETE.format( 106 | table_name=self._table_name, key_column=self._key_column, op=op) 107 | return sql, (key,) 108 | 109 | def _pop(self, rowid: Optional[int] = None, raw: bool = False 110 | ) -> Optional[Any]: 111 | with self.action_lock: 112 | if self.auto_commit: 113 | row = self._select(rowid=rowid) 114 | # Perhaps a sqlite3 bug, sometimes (None, None) is returned 115 | # by select, below can avoid these invalid records. 116 | if row and row[0] is not None: 117 | self._delete(row[0]) 118 | self.total -= 1 119 | item = self._serializer.loads(row[1]) 120 | if raw: 121 | return { 122 | 'pqid': row[0], 123 | 'data': item, 124 | 'timestamp': row[2], 125 | } 126 | else: 127 | return item 128 | else: 129 | row = self._select( 130 | self.cursor, op=">", column=self._KEY_COLUMN, rowid=rowid 131 | ) 132 | if row and row[0] is not None: 133 | self.cursor = row[0] 134 | self.total -= 1 135 | item = self._serializer.loads(row[1]) 136 | if raw: 137 | return { 138 | 'pqid': row[0], 139 | 'data': item, 140 | 'timestamp': row[2], 141 | } 142 | else: 143 | return item 144 | return None 145 | 146 | def update(self, item: Any, id: Optional[int] = None) -> int: 147 | if isinstance(item, dict) and "pqid" in item: 148 | _id = item.get("pqid") 149 | item = item.get("data") 150 | if id is not None: 151 | _id = id 152 | if _id is None: 153 | raise ValueError("Provide an id or raw item") 154 | obj = self._serializer.dumps(item) 155 | self._update(_id, obj) 156 | return _id 157 | 158 | def get(self, block: bool = True, timeout: Optional[float] = None, 159 | id: Optional[int] = None, raw: bool = False 160 | ) -> Any: 161 | if isinstance(id, dict) and "pqid" in id: 162 | rowid = id.get("pqid") 163 | elif isinstance(id, int): 164 | rowid = id 165 | else: 166 | rowid = None 167 | if not block: 168 | serialized = self._pop(raw=raw, rowid=rowid) 169 | if serialized is None: 170 | raise Empty 171 | elif timeout is None: 172 | # block until a put event. 173 | serialized = self._pop(raw=raw, rowid=rowid) 174 | while serialized is None: 175 | self.put_event.clear() 176 | self.put_event.wait(TICK_FOR_WAIT) 177 | serialized = self._pop(raw=raw, rowid=rowid) 178 | elif timeout < 0: 179 | raise ValueError("'timeout' must be a non-negative number") 180 | else: 181 | # block until the timeout reached 182 | endtime = _time.time() + timeout 183 | serialized = self._pop(raw=raw, rowid=rowid) 184 | while serialized is None: 185 | self.put_event.clear() 186 | remaining = endtime - _time.time() 187 | if remaining <= 0.0: 188 | raise Empty 189 | self.put_event.wait( 190 | TICK_FOR_WAIT if TICK_FOR_WAIT < remaining else remaining 191 | ) 192 | serialized = self._pop(raw=raw, rowid=rowid) 193 | return serialized 194 | 195 | def get_nowait(self, id: Optional[int] = None, raw: bool = False) -> Any: 196 | return self.get(block=False, id=id, raw=raw) 197 | 198 | def task_done(self) -> None: 199 | """Persist the current state if auto_commit=False.""" 200 | if not self.auto_commit: 201 | self._delete(self.cursor, op='<=') 202 | self._task_done() 203 | 204 | def queue(self) -> Any: 205 | rows = self._sql_queue().fetchall() 206 | datarows = [] 207 | for row in rows: 208 | item = { 209 | 'id': row[0], 210 | 'data': self._serializer.loads(row[1]), 211 | 'timestamp': row[2], 212 | } 213 | datarows.append(item) 214 | return datarows 215 | 216 | @with_conditional_transaction 217 | def shrink_disk_usage(self) -> Tuple[str, Tuple[()]]: 218 | sql = """VACUUM""" 219 | return sql, () 220 | 221 | @property 222 | def size(self) -> int: 223 | return self.total 224 | 225 | def qsize(self) -> int: 226 | return max(0, self.size) 227 | 228 | def empty(self) -> bool: 229 | return self.size == 0 230 | 231 | def full(self) -> bool: 232 | return False 233 | 234 | def __len__(self) -> int: 235 | return self.size 236 | 237 | def _select(self, *args, **kwargs) -> Any: 238 | start_key = self._start_key() 239 | op = kwargs.get('op', None) 240 | column = kwargs.get('column', None) 241 | next_in_order = kwargs.get('next_in_order', False) 242 | rowid = kwargs.get('rowid') if kwargs.get('rowid', None) else start_key 243 | if not next_in_order and rowid != start_key: 244 | # Get the record by the id 245 | result = self._getter.execute( 246 | self._sql_select_id(rowid), args 247 | ).fetchone() 248 | elif op and column: 249 | # Get the next record with criteria 250 | rowid = rowid if next_in_order else start_key 251 | result = self._getter.execute( 252 | self._sql_select_where(rowid, op, column), args 253 | ).fetchone() 254 | else: 255 | # Get the next record 256 | rowid = rowid if next_in_order else start_key 257 | result = self._getter.execute( 258 | self._sql_select(rowid), args 259 | ).fetchone() 260 | if ( 261 | next_in_order 262 | and rowid != start_key 263 | and (not result or len(result) == 0) 264 | ): 265 | # sqlackqueue: if we're at the end, start over 266 | kwargs['rowid'] = start_key 267 | result = self._select(*args, **kwargs) 268 | return result 269 | 270 | def _count(self) -> int: 271 | sql = 'SELECT COUNT({}) FROM {}'.format( 272 | self._key_column, self._table_name 273 | ) 274 | row = self._getter.execute(sql).fetchone() 275 | return row[0] if row else 0 276 | 277 | def _start_key(self) -> int: 278 | if self._TABLE_NAME == 'ack_filo_queue': 279 | return 9223372036854775807 # maxsize 280 | else: 281 | return 0 282 | 283 | def _task_done(self) -> None: 284 | """Only required if auto-commit is set as False.""" 285 | commit_ignore_error(self._putter) 286 | 287 | def _sql_queue(self) -> Any: 288 | sql = 'SELECT * FROM {}'.format(self._table_name) 289 | return self._getter.execute(sql) 290 | 291 | @property 292 | def _table_name(self) -> str: 293 | return '`{}_{}`'.format(self._TABLE_NAME, self.name) 294 | 295 | @property 296 | def _key_column(self) -> str: 297 | return self._KEY_COLUMN 298 | 299 | @property 300 | def _sql_create(self) -> str: 301 | return self._SQL_CREATE.format( 302 | table_name=self._table_name, key_column=self._key_column 303 | ) 304 | 305 | @property 306 | def _sql_insert(self) -> str: 307 | return self._SQL_INSERT.format( 308 | table_name=self._table_name, key_column=self._key_column 309 | ) 310 | 311 | @property 312 | def _sql_update(self) -> str: 313 | return self._SQL_UPDATE.format( 314 | table_name=self._table_name, key_column=self._key_column 315 | ) 316 | 317 | def _sql_select_id(self, rowid) -> str: 318 | return self._SQL_SELECT_ID.format( 319 | table_name=self._table_name, 320 | key_column=self._key_column, 321 | rowid=rowid, 322 | ) 323 | 324 | def _sql_select(self, rowid) -> str: 325 | return self._SQL_SELECT.format( 326 | table_name=self._table_name, 327 | key_column=self._key_column, 328 | rowid=rowid, 329 | ) 330 | 331 | def _sql_select_where(self, rowid, op, column) -> str: 332 | return self._SQL_SELECT_WHERE.format( 333 | table_name=self._table_name, 334 | key_column=self._key_column, 335 | rowid=rowid, 336 | op=op, 337 | column=column, 338 | ) 339 | 340 | def __del__(self) -> None: 341 | """Handles sqlite connection when queue was deleted""" 342 | if self._getter: 343 | self._getter.close() 344 | if self._putter: 345 | self._putter.close() 346 | 347 | 348 | class SQLiteBase(SQLBase): 349 | """SQLite3 base class.""" 350 | _TABLE_NAME = 'base' # DB table name 351 | _KEY_COLUMN = '' # the name of the key column, used in DB CRUD 352 | _SQL_CREATE = '' # SQL to create a table 353 | _SQL_UPDATE = '' # SQL to update a record 354 | _SQL_INSERT = '' # SQL to insert a record 355 | _SQL_SELECT = '' # SQL to select a record 356 | _SQL_SELECT_ID = '' # SQL to select a record with criteria 357 | _SQL_SELECT_WHERE = '' # SQL to select a record with criteria 358 | _SQL_DELETE = '' # SQL to delete a record 359 | _MEMORY = ':memory:' # flag indicating store DB in memory 360 | 361 | def __init__(self, path: str, name: str = 'default', 362 | multithreading: bool = False, timeout: float = 10.0, 363 | auto_commit: bool = True, 364 | serializer: Any = persistqueue.serializers.pickle, 365 | db_file_name: Optional[str] = None) -> None: 366 | """Initiate a queue in sqlite3 or memory. 367 | 368 | :param path: path for storing DB file. 369 | :param name: the suffix for the table name, 370 | table name would be ${_TABLE_NAME}_${name} 371 | :param multithreading: if set to True, two db connections will be, 372 | one for **put** and one for **get**. 373 | :param timeout: timeout in second waiting for the database lock. 374 | :param auto_commit: Set to True, if commit is required on every 375 | INSERT/UPDATE action, otherwise False, whereas 376 | a **task_done** is required to persist changes 377 | after **put**. 378 | :param serializer: The serializer parameter controls how enqueued data 379 | is serialized. It must have methods dump(value, fp) 380 | and load(fp). The dump method must serialize the 381 | value and write it to fp, and may be called for 382 | multiple values with the same fp. The load method 383 | must deserialize and return one value from fp, 384 | and may be called multiple times with the same fp 385 | to read multiple values. 386 | :param db_file_name: set the db file name of the queue data, otherwise 387 | default to `data.db` 388 | """ 389 | super(SQLiteBase, self).__init__() 390 | self.memory_sql = False 391 | self.path = path 392 | self.name = name 393 | self.timeout = timeout 394 | self.multithreading = multithreading 395 | self.auto_commit = auto_commit 396 | self._serializer = serializer 397 | self.db_file_name = "data.db" 398 | if db_file_name: 399 | self.db_file_name = db_file_name 400 | self._init() 401 | 402 | def _init(self) -> None: 403 | """Initialize the tables in DB.""" 404 | if self.path == self._MEMORY: 405 | self.memory_sql = True 406 | log.debug("Initializing Sqlite3 Queue in memory.") 407 | elif not os.path.exists(self.path): 408 | os.makedirs(self.path) 409 | log.debug( 410 | 'Initializing Sqlite3 Queue with path {}'.format(self.path) 411 | ) 412 | self._conn = self._new_db_connection( 413 | self.path, self.multithreading, self.timeout 414 | ) 415 | self._getter = self._conn 416 | self._putter = self._conn 417 | 418 | self._conn.execute(self._sql_create) 419 | self._conn.commit() 420 | # Setup another session only for disk-based queue. 421 | if self.multithreading: 422 | if not self.memory_sql: 423 | self._putter = self._new_db_connection( 424 | self.path, self.multithreading, self.timeout 425 | ) 426 | self._conn.text_factory = str 427 | self._putter.text_factory = str 428 | 429 | # SQLite3 transaction lock 430 | self.tran_lock = threading.Lock() 431 | self.put_event = threading.Event() 432 | 433 | def _new_db_connection(self, path, multithreading, timeout 434 | ) -> sqlite3.Connection: 435 | conn = None 436 | if path == self._MEMORY: 437 | conn = sqlite3.connect(path, check_same_thread=not multithreading) 438 | else: 439 | conn = sqlite3.connect( 440 | '{}/{}'.format(path, self.db_file_name), 441 | timeout=timeout, 442 | check_same_thread=not multithreading, 443 | ) 444 | conn.execute('PRAGMA journal_mode=WAL;') 445 | return conn 446 | 447 | def close(self) -> None: 448 | """Closes sqlite connections""" 449 | if self._getter is not None: 450 | self._getter.close() 451 | if self._putter is not None: 452 | self._putter.close() 453 | 454 | def __del__(self) -> None: 455 | """Handles sqlite connection when queue was deleted""" 456 | self.close() 457 | -------------------------------------------------------------------------------- /persistqueue/sqlqueue.py: -------------------------------------------------------------------------------- 1 | """A thread-safe sqlite3 based persistent queue in Python.""" 2 | import logging 3 | import sqlite3 4 | import time as _time 5 | import threading 6 | from typing import Any 7 | from persistqueue import sqlbase 8 | 9 | sqlite3.enable_callback_tracebacks(True) 10 | log = logging.getLogger(__name__) 11 | 12 | 13 | class SQLiteQueue(sqlbase.SQLiteBase): 14 | """SQLite3 based FIFO queue.""" 15 | _TABLE_NAME = 'queue' 16 | _KEY_COLUMN = '_id' # the name of the key column, used in DB CRUD 17 | # SQL to create a table 18 | _SQL_CREATE = ( 19 | 'CREATE TABLE IF NOT EXISTS {table_name} (' 20 | '{key_column} INTEGER PRIMARY KEY AUTOINCREMENT, ' 21 | 'data BLOB, timestamp FLOAT)' 22 | ) 23 | # SQL to insert a record 24 | _SQL_INSERT = 'INSERT INTO {table_name} (data, timestamp) VALUES (?, ?)' 25 | # SQL to select a record 26 | _SQL_SELECT_ID = ( 27 | 'SELECT {key_column}, data, timestamp FROM {table_name} WHERE' 28 | ' {key_column} = {rowid}' 29 | ) 30 | _SQL_SELECT = ( 31 | 'SELECT {key_column}, data, timestamp FROM {table_name} ' 32 | 'ORDER BY {key_column} ASC LIMIT 1' 33 | ) 34 | _SQL_SELECT_WHERE = ( 35 | 'SELECT {key_column}, data, timestamp FROM {table_name} WHERE' 36 | ' {column} {op} ? ORDER BY {key_column} ASC LIMIT 1 ' 37 | ) 38 | _SQL_UPDATE = 'UPDATE {table_name} SET data = ? WHERE {key_column} = ?' 39 | _SQL_DELETE = 'DELETE FROM {table_name} WHERE {key_column} {op} ?' 40 | 41 | def put(self, item: Any, block: bool = True) -> int: 42 | # block kwarg is noop and only here to align with python's queue 43 | obj = self._serializer.dumps(item) 44 | _id = self._insert_into(obj, _time.time()) 45 | self.total += 1 46 | self.put_event.set() 47 | return _id 48 | 49 | def put_nowait(self, item: Any) -> int: 50 | return self.put(item, block=False) 51 | 52 | def _init(self) -> None: 53 | super(SQLiteQueue, self)._init() 54 | self.action_lock = threading.Lock() 55 | if not self.auto_commit: 56 | head = self._select() 57 | if head: 58 | self.cursor = head[0] - 1 59 | else: 60 | self.cursor = 0 61 | self.total = self._count() 62 | 63 | 64 | FIFOSQLiteQueue = SQLiteQueue 65 | 66 | 67 | class FILOSQLiteQueue(SQLiteQueue): 68 | """SQLite3 based FILO queue.""" 69 | _TABLE_NAME = 'filo_queue' 70 | # SQL to select a record 71 | _SQL_SELECT = ( 72 | 'SELECT {key_column}, data FROM {table_name} ' 73 | 'ORDER BY {key_column} DESC LIMIT 1' 74 | ) 75 | 76 | 77 | class UniqueQ(SQLiteQueue): 78 | _TABLE_NAME = 'unique_queue' 79 | _SQL_CREATE = ( 80 | 'CREATE TABLE IF NOT EXISTS {table_name} (' 81 | '{key_column} INTEGER PRIMARY KEY AUTOINCREMENT, ' 82 | 'data BLOB, timestamp FLOAT, UNIQUE (data))' 83 | ) 84 | 85 | def put(self, item: Any) -> Any: 86 | obj = self._serializer.dumps(item, sort_keys=True) 87 | _id = None 88 | try: 89 | _id = self._insert_into(obj, _time.time()) 90 | except sqlite3.IntegrityError: 91 | pass 92 | else: 93 | self.total += 1 94 | self.put_event.set() 95 | return _id 96 | -------------------------------------------------------------------------------- /persistqueue/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/peter-wangxu/persist-queue/b194f07553379a8319ce1bd0adbb0ad4c610eba9/persistqueue/tests/__init__.py -------------------------------------------------------------------------------- /persistqueue/tests/test_mysqlqueue.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | import unittest 3 | import random 4 | from threading import Thread 5 | import time 6 | import sys 7 | 8 | from persistqueue.mysqlqueue import MySQLQueue 9 | from persistqueue import Empty 10 | 11 | # db config aligned with .circleci/config.yml 12 | db_conf = { 13 | "host": "127.0.0.1", 14 | "user": "user", 15 | "passwd": "passw0rd", 16 | "db_name": "testqueue", 17 | # "name": "", 18 | "port": 3306 19 | } 20 | # for appveyor (windows ci), not able to config use the default 21 | # https://www.appveyor.com/docs/services-databases/#mysql 22 | if sys.platform.startswith('win32'): 23 | db_conf = { 24 | "host": "127.0.0.1", 25 | "user": "root", 26 | "passwd": "Password12!", 27 | "db_name": "testqueue", 28 | # "name": "", 29 | "port": 3306 30 | } 31 | 32 | 33 | class MySQLQueueTest(unittest.TestCase): 34 | """tests that focus on feature specific to mysql""" 35 | 36 | def setUp(self): 37 | _name = self.id().split(".")[-1:] 38 | _name.append(str(time.time())) 39 | self._table_name = ".".join(_name) 40 | self.queue_class = MySQLQueue 41 | self.mysql_queue = MySQLQueue(name=self._table_name, 42 | **db_conf) 43 | self.queue = self.mysql_queue 44 | 45 | def tearDown(self): 46 | pass 47 | tmp_conn = self.mysql_queue.get_pooled_conn() 48 | tmp_conn.cursor().execute( 49 | "drop table if exists %s" % self.mysql_queue._table_name) 50 | tmp_conn.commit() 51 | 52 | def test_raise_empty(self): 53 | q = self.queue 54 | 55 | q.put('first') 56 | d = q.get() 57 | self.assertEqual('first', d) 58 | self.assertRaises(Empty, q.get, block=False) 59 | self.assertRaises(Empty, q.get_nowait) 60 | 61 | # assert with timeout 62 | self.assertRaises(Empty, q.get, block=True, timeout=1.0) 63 | # assert with negative timeout 64 | self.assertRaises(ValueError, q.get, block=True, timeout=-1.0) 65 | del q 66 | 67 | def test_empty(self): 68 | q = self.queue 69 | self.assertEqual(q.empty(), True) 70 | 71 | q.put('first') 72 | self.assertEqual(q.empty(), False) 73 | 74 | q.get() 75 | self.assertEqual(q.empty(), True) 76 | 77 | def test_full(self): 78 | # SQL queue `full()` always returns `False` !! 79 | q = self.queue 80 | self.assertEqual(q.full(), False) 81 | 82 | q.put('first') 83 | self.assertEqual(q.full(), False) 84 | 85 | q.get() 86 | self.assertEqual(q.full(), False) 87 | 88 | def test_open_close_single(self): 89 | """Write 1 item, close, reopen checking if same item is there""" 90 | 91 | q = self.queue 92 | q.put(b'var1') 93 | del q 94 | q = MySQLQueue(name=self._table_name, 95 | **db_conf) 96 | self.assertEqual(1, q.qsize()) 97 | self.assertEqual(b'var1', q.get()) 98 | 99 | def test_open_close_1000(self): 100 | """Write 1000 items, close, reopen checking if all items are there""" 101 | 102 | q = self.queue 103 | for i in range(1000): 104 | q.put('var%d' % i) 105 | self.assertEqual(1000, q.qsize()) 106 | del q 107 | q = MySQLQueue(name=self._table_name, 108 | **db_conf) 109 | self.assertEqual(1000, q.qsize()) 110 | for i in range(1000): 111 | data = q.get() 112 | self.assertEqual('var%d' % i, data) 113 | # assert adding another one still works 114 | q.put('foobar') 115 | data = q.get() 116 | self.assertEqual('foobar', data) 117 | 118 | def test_random_read_write(self): 119 | """Test random read/write""" 120 | 121 | q = self.queue 122 | n = 0 123 | for _ in range(1000): 124 | if random.random() < 0.5: 125 | if n > 0: 126 | q.get() 127 | n -= 1 128 | else: 129 | self.assertRaises(Empty, q.get, block=False) 130 | else: 131 | q.put('var%d' % random.getrandbits(16)) 132 | n += 1 133 | 134 | def test_multi_threaded_parallel(self): 135 | """Create consumer and producer threads, check parallelism""" 136 | m_queue = self.queue 137 | 138 | def producer(): 139 | for i in range(1000): 140 | m_queue.put('var%d' % i) 141 | 142 | def consumer(): 143 | for i in range(1000): 144 | x = m_queue.get(block=True) 145 | self.assertEqual('var%d' % i, x) 146 | 147 | c = Thread(target=consumer) 148 | c.start() 149 | p = Thread(target=producer) 150 | p.start() 151 | p.join() 152 | c.join() 153 | self.assertEqual(0, m_queue.size) 154 | self.assertEqual(0, len(m_queue)) 155 | self.assertRaises(Empty, m_queue.get, block=False) 156 | 157 | def test_multi_threaded_multi_producer(self): 158 | """Test mysqlqueue can be used by multiple producers.""" 159 | 160 | queue = self.queue 161 | 162 | def producer(seq): 163 | for i in range(10): 164 | queue.put('var%d' % (i + (seq * 10))) 165 | 166 | def consumer(): 167 | for _ in range(100): 168 | data = queue.get(block=True) 169 | self.assertTrue('var' in data) 170 | 171 | c = Thread(target=consumer) 172 | c.start() 173 | producers = [] 174 | for seq in range(10): 175 | t = Thread(target=producer, args=(seq,)) 176 | t.start() 177 | producers.append(t) 178 | 179 | for t in producers: 180 | t.join() 181 | 182 | c.join() 183 | 184 | def test_multiple_consumers(self): 185 | """Test mysqlqueue can be used by multiple consumers.""" 186 | queue = self.queue 187 | 188 | def producer(): 189 | for x in range(1000): 190 | queue.put('var%d' % x) 191 | 192 | counter = [] 193 | # Set all to 0 194 | for _ in range(1000): 195 | counter.append(0) 196 | 197 | def consumer(t_index): 198 | for i in range(200): 199 | data = queue.get(block=True) 200 | self.assertTrue('var' in data) 201 | counter[t_index * 200 + i] = data 202 | 203 | p = Thread(target=producer) 204 | p.start() 205 | consumers = [] 206 | for index in range(5): 207 | t = Thread(target=consumer, args=(index,)) 208 | t.start() 209 | consumers.append(t) 210 | 211 | p.join() 212 | for t in consumers: 213 | t.join() 214 | 215 | self.assertEqual(0, queue.qsize()) 216 | for x in range(1000): 217 | self.assertNotEqual(0, counter[x], 218 | "not 0 for counter's index %s" % x) 219 | 220 | self.assertEqual(len(set(counter)), len(counter)) 221 | 222 | def test_task_done_with_restart(self): 223 | """Test that items are not deleted before task_done.""" 224 | 225 | q = self.queue 226 | 227 | for i in range(1, 11): 228 | q.put(i) 229 | 230 | self.assertEqual(1, q.get()) 231 | self.assertEqual(2, q.get()) 232 | # size is correct before task_done 233 | self.assertEqual(8, q.qsize()) 234 | q.task_done() 235 | # make sure the size still correct 236 | self.assertEqual(8, q.qsize()) 237 | 238 | self.assertEqual(3, q.get()) 239 | # without task done 240 | del q 241 | q = MySQLQueue(name=self._table_name, 242 | **db_conf) 243 | # After restart, the qsize and head item are the same 244 | self.assertEqual(7, q.qsize()) 245 | # After restart, the queue still works 246 | self.assertEqual(4, q.get()) 247 | self.assertEqual(6, q.qsize()) 248 | # auto_commit=False 249 | del q 250 | q = MySQLQueue(name=self._table_name, auto_commit=False, 251 | **db_conf) 252 | self.assertEqual(6, q.qsize()) 253 | # After restart, the queue still works 254 | self.assertEqual(5, q.get()) 255 | self.assertEqual(5, q.qsize()) 256 | del q 257 | q = MySQLQueue(name=self._table_name, auto_commit=False, 258 | **db_conf) 259 | # After restart, the queue still works 260 | self.assertEqual(5, q.get()) 261 | self.assertEqual(5, q.qsize()) 262 | 263 | def test_protocol_1(self): 264 | q = self.queue 265 | self.assertEqual(q._serializer.protocol, 266 | 2 if sys.version_info[0] == 2 else 4) 267 | 268 | def test_protocol_2(self): 269 | q = self.queue 270 | self.assertEqual(q._serializer.protocol, 271 | 2 if sys.version_info[0] == 2 else 4) 272 | 273 | def test_json_serializer(self): 274 | q = self.queue 275 | x = dict( 276 | a=1, 277 | b=2, 278 | c=dict( 279 | d=list(range(5)), 280 | e=[1] 281 | )) 282 | q.put(x) 283 | self.assertEqual(q.get(), x) 284 | 285 | def test_put_0(self): 286 | q = self.queue 287 | q.put(0) 288 | d = q.get(block=False) 289 | self.assertIsNotNone(d) 290 | 291 | def test_get_id(self): 292 | q = self.queue 293 | q.put("val1") 294 | val2_id = q.put("val2") 295 | q.put("val3") 296 | item = q.get(id=val2_id) 297 | # item id should be 2 298 | self.assertEqual(val2_id, 2) 299 | # item should get val2 300 | self.assertEqual(item, 'val2') 301 | 302 | def test_get_raw(self): 303 | q = self.queue 304 | q.put("val1") 305 | item = q.get(raw=True) 306 | # item should get val2 307 | self.assertEqual(True, "pqid" in item) 308 | self.assertEqual(item.get("data"), 'val1') 309 | 310 | def test_queue(self): 311 | q = self.queue 312 | q.put("val1") 313 | q.put("val2") 314 | q.put("val3") 315 | # queue should get the three items 316 | d = q.queue() 317 | self.assertEqual(len(d), 3) 318 | self.assertEqual(d[1].get("data"), "val2") 319 | 320 | def test_update(self): 321 | q = self.queue 322 | qid = q.put("val1") 323 | q.update(item="val2", id=qid) 324 | item = q.get(id=qid) 325 | self.assertEqual(item, "val2") 326 | -------------------------------------------------------------------------------- /persistqueue/tests/test_pdict.py: -------------------------------------------------------------------------------- 1 | 2 | import shutil 3 | import tempfile 4 | import unittest 5 | 6 | from persistqueue import pdict 7 | 8 | 9 | class PDictTest(unittest.TestCase): 10 | def setUp(self): 11 | self.path = tempfile.mkdtemp(suffix='pdict') 12 | 13 | def tearDown(self): 14 | shutil.rmtree(self.path, ignore_errors=True) 15 | 16 | def test_unsupported(self): 17 | pd = pdict.PDict(self.path, 'pd') 18 | pd['key_a'] = 'value_a' 19 | self.assertRaises(NotImplementedError, pd.keys) 20 | self.assertRaises(NotImplementedError, pd.iterkeys) 21 | self.assertRaises(NotImplementedError, pd.values) 22 | self.assertRaises(NotImplementedError, pd.itervalues) 23 | self.assertRaises(NotImplementedError, pd.items) 24 | self.assertRaises(NotImplementedError, pd.iteritems) 25 | 26 | def _for(): 27 | for _ in pd: 28 | pass 29 | self.assertRaises(NotImplementedError, _for) 30 | 31 | def test_add(self): 32 | pd = pdict.PDict(self.path, 'pd') 33 | pd['key_a'] = 'value_a' 34 | self.assertEqual(pd['key_a'], 'value_a') 35 | self.assertTrue('key_a' in pd) 36 | self.assertFalse('key_b' in pd) 37 | self.assertEqual(pd.get('key_a'), 'value_a') 38 | self.assertEqual(pd.get('key_b'), None) 39 | self.assertEqual(pd.get('key_b', 'absent'), 'absent') 40 | self.assertRaises(KeyError, lambda: pd['key_b']) 41 | pd['key_b'] = 'value_b' 42 | self.assertEqual(pd['key_a'], 'value_a') 43 | self.assertEqual(pd['key_b'], 'value_b') 44 | 45 | def test_set(self): 46 | pd = pdict.PDict(self.path, 'pd') 47 | pd['key_a'] = 'value_a' 48 | pd['key_b'] = 'value_b' 49 | self.assertEqual(pd['key_a'], 'value_a') 50 | self.assertEqual(pd['key_b'], 'value_b') 51 | self.assertEqual(pd.get('key_a'), 'value_a') 52 | self.assertEqual(pd.get('key_b', 'absent'), 'value_b') 53 | pd['key_a'] = 'value_aaaaaa' 54 | self.assertEqual(pd['key_a'], 'value_aaaaaa') 55 | self.assertEqual(pd['key_b'], 'value_b') 56 | 57 | def test_delete(self): 58 | pd = pdict.PDict(self.path, 'pd') 59 | pd['key_a'] = 'value_a' 60 | pd['key_b'] = 'value_b' 61 | self.assertEqual(pd['key_a'], 'value_a') 62 | self.assertEqual(pd['key_b'], 'value_b') 63 | del pd['key_a'] 64 | self.assertFalse('key_a' in pd) 65 | self.assertRaises(KeyError, lambda: pd['key_a']) 66 | self.assertEqual(pd['key_b'], 'value_b') 67 | 68 | def test_two_dicts(self): 69 | pd_1 = pdict.PDict(self.path, '1') 70 | pd_2 = pdict.PDict(self.path, '2') 71 | pd_1['key_a'] = 'value_a' 72 | pd_2['key_b'] = 'value_b' 73 | self.assertEqual(pd_1['key_a'], 'value_a') 74 | self.assertEqual(pd_2['key_b'], 'value_b') 75 | self.assertRaises(KeyError, lambda: pd_1['key_b']) 76 | self.assertRaises(KeyError, lambda: pd_2['key_a']) 77 | -------------------------------------------------------------------------------- /persistqueue/tests/test_queue.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | 3 | import os 4 | import pickle 5 | import random 6 | import shutil 7 | import sys 8 | import tempfile 9 | import unittest 10 | from collections import namedtuple 11 | from nose2.tools import params 12 | from threading import Thread 13 | 14 | from persistqueue.serializers import json as serializers_json 15 | from persistqueue.serializers import pickle as serializers_pickle 16 | from persistqueue.serializers import msgpack as serializers_msgpack 17 | from persistqueue.serializers import cbor2 as serializers_cbor2 18 | 19 | from persistqueue import Queue, Empty, Full 20 | 21 | # map keys as params for readable errors from nose 22 | serializer_params = { 23 | "serializer=default": {}, 24 | "serializer=json": {"serializer": serializers_json}, 25 | "serializer=msgpack": {"serializer": serializers_msgpack}, 26 | "serializer=cbor2": {"serializer": serializers_cbor2}, 27 | "serializer=pickle": {"serializer": serializers_pickle}, 28 | } 29 | 30 | 31 | class PersistTest(unittest.TestCase): 32 | def setUp(self): 33 | self.path = tempfile.mkdtemp(suffix='queue') 34 | 35 | def tearDown(self): 36 | shutil.rmtree(self.path, ignore_errors=True) 37 | 38 | @params(*serializer_params) 39 | def test_open_close_single(self, serializer): 40 | """Write 1 item, close, reopen checking if same item is there""" 41 | 42 | q = Queue(self.path, **serializer_params[serializer]) 43 | q.put('var1') 44 | del q 45 | q = Queue(self.path, **serializer_params[serializer]) 46 | self.assertEqual(1, q.qsize()) 47 | self.assertEqual('var1', q.get()) 48 | q.task_done() 49 | del q 50 | 51 | def test_empty(self): 52 | q = Queue(self.path) 53 | self.assertEqual(q.empty(), True) 54 | 55 | q.put('var1') 56 | self.assertEqual(q.empty(), False) 57 | 58 | q.get() 59 | self.assertEqual(q.empty(), True) 60 | 61 | def test_full(self): 62 | q = Queue(self.path, maxsize=3) 63 | 64 | for i in range(1, q.maxsize): 65 | q.put('var{}'.format(i)) 66 | self.assertEqual(q.full(), False) 67 | 68 | q.put('var{}'.format(q.maxsize)) 69 | self.assertEqual(q.full(), True) 70 | 71 | q.get() 72 | self.assertEqual(q.full(), False) 73 | 74 | @params(*serializer_params) 75 | def test_open_close_1000(self, serializer): 76 | """Write 1000 items, close, reopen checking if all items are there""" 77 | 78 | q = Queue(self.path, **serializer_params[serializer]) 79 | for i in range(1000): 80 | q.put('var%d' % i) 81 | self.assertEqual(1000, q.qsize()) 82 | del q 83 | q = Queue(self.path, **serializer_params[serializer]) 84 | self.assertEqual(1000, q.qsize()) 85 | for i in range(1000): 86 | data = q.get() 87 | self.assertEqual('var%d' % i, data) 88 | q.task_done() 89 | with self.assertRaises(Empty): 90 | q.get_nowait() 91 | # assert adding another one still works 92 | q.put('foobar') 93 | data = q.get() 94 | 95 | @params(*serializer_params) 96 | def test_partial_write(self, serializer): 97 | """Test recovery from previous crash w/ partial write""" 98 | 99 | q = Queue(self.path, **serializer_params[serializer]) 100 | for i in range(100): 101 | q.put('var%d' % i) 102 | del q 103 | with open(os.path.join(self.path, 'q00000'), 'ab') as f: 104 | pickle.dump('文字化け', f) 105 | q = Queue(self.path, **serializer_params[serializer]) 106 | self.assertEqual(100, q.qsize()) 107 | for i in range(100): 108 | self.assertEqual('var%d' % i, q.get()) 109 | q.task_done() 110 | with self.assertRaises(Empty): 111 | q.get_nowait() 112 | 113 | @params(*serializer_params) 114 | def test_random_read_write(self, serializer): 115 | """Test random read/write""" 116 | 117 | q = Queue(self.path, **serializer_params[serializer]) 118 | n = 0 119 | for i in range(1000): 120 | if random.random() < 0.5: 121 | if n > 0: 122 | q.get_nowait() 123 | q.task_done() 124 | n -= 1 125 | else: 126 | with self.assertRaises(Empty): 127 | q.get_nowait() 128 | else: 129 | q.put('var%d' % random.getrandbits(16)) 130 | n += 1 131 | 132 | @params(*serializer_params) 133 | def test_multi_threaded(self, serializer): 134 | """Create consumer and producer threads, check parallelism""" 135 | 136 | q = Queue(self.path, **serializer_params[serializer]) 137 | 138 | def producer(): 139 | for i in range(1000): 140 | q.put('var%d' % i) 141 | 142 | def consumer(): 143 | for i in range(1000): 144 | q.get() 145 | q.task_done() 146 | 147 | c = Thread(target=consumer) 148 | c.start() 149 | p = Thread(target=producer) 150 | p.start() 151 | c.join() 152 | p.join() 153 | 154 | q.join() 155 | with self.assertRaises(Empty): 156 | q.get_nowait() 157 | 158 | @params(*serializer_params) 159 | def test_garbage_on_head(self, serializer): 160 | """Adds garbage to the queue head and let the internal integrity 161 | checks fix it""" 162 | 163 | q = Queue(self.path, **serializer_params[serializer]) 164 | q.put('var1') 165 | del q 166 | 167 | with open(os.path.join(self.path, 'q00000'), 'ab') as f: 168 | f.write(b'garbage') 169 | q = Queue(self.path, **serializer_params[serializer]) 170 | q.put('var2') 171 | 172 | self.assertEqual(2, q.qsize()) 173 | self.assertEqual('var1', q.get()) 174 | q.task_done() 175 | 176 | @params(*serializer_params) 177 | def test_task_done_too_many_times(self, serializer): 178 | """Test too many task_done called.""" 179 | q = Queue(self.path, **serializer_params[serializer]) 180 | q.put('var1') 181 | q.get() 182 | q.task_done() 183 | 184 | with self.assertRaises(ValueError): 185 | q.task_done() 186 | 187 | @params(*serializer_params) 188 | def test_get_timeout_negative(self, serializer): 189 | q = Queue(self.path, **serializer_params[serializer]) 190 | q.put('var1') 191 | with self.assertRaises(ValueError): 192 | q.get(timeout=-1) 193 | 194 | @params(*serializer_params) 195 | def test_get_timeout(self, serializer): 196 | """Test when get failed within timeout.""" 197 | q = Queue(self.path, **serializer_params[serializer]) 198 | q.put('var1') 199 | q.get() 200 | with self.assertRaises(Empty): 201 | q.get(timeout=1) 202 | 203 | @params(*serializer_params) 204 | def test_put_nowait(self, serializer): 205 | """Tests the put_nowait interface.""" 206 | q = Queue(self.path, **serializer_params[serializer]) 207 | q.put_nowait('var1') 208 | self.assertEqual('var1', q.get()) 209 | q.task_done() 210 | 211 | @params(*serializer_params) 212 | def test_put_maxsize_reached(self, serializer): 213 | """Test that maxsize reached.""" 214 | q = Queue(self.path, maxsize=10, **serializer_params[serializer]) 215 | for x in range(10): 216 | q.put(x) 217 | 218 | with self.assertRaises(Full): 219 | q.put('full_now', block=False) 220 | 221 | @params(*serializer_params) 222 | def test_put_timeout_reached(self, serializer): 223 | """Test put with block and timeout.""" 224 | q = Queue(self.path, maxsize=2, **serializer_params[serializer]) 225 | for x in range(2): 226 | q.put(x) 227 | 228 | with self.assertRaises(Full): 229 | q.put('full_and_timeout', block=True, timeout=1) 230 | 231 | @params(*serializer_params) 232 | def test_put_timeout_negative(self, serializer): 233 | """Test and put with timeout < 0""" 234 | q = Queue(self.path, maxsize=1, **serializer_params[serializer]) 235 | with self.assertRaises(ValueError): 236 | q.put('var1', timeout=-1) 237 | 238 | @params(*serializer_params) 239 | def test_put_block_and_wait(self, serializer): 240 | """Test block until queue is not full.""" 241 | q = Queue(self.path, maxsize=10, **serializer_params[serializer]) 242 | 243 | def consumer(): 244 | for i in range(5): 245 | q.get() 246 | q.task_done() 247 | 248 | def producer(): 249 | for j in range(16): 250 | q.put('var%d' % j) 251 | 252 | p = Thread(target=producer) 253 | p.start() 254 | c = Thread(target=consumer) 255 | c.start() 256 | c.join() 257 | val = q.get_nowait() 258 | p.join() 259 | self.assertEqual('var5', val) 260 | 261 | @params(*serializer_params) 262 | def test_clear_tail_file(self, serializer): 263 | """Test that only remove tail file when calling task_done.""" 264 | q = Queue(self.path, chunksize=10, **serializer_params[serializer]) 265 | for i in range(35): 266 | q.put('var%d' % i) 267 | 268 | for _ in range(15): 269 | q.get() 270 | 271 | q = Queue(self.path, chunksize=10, **serializer_params[serializer]) 272 | self.assertEqual(q.qsize(), 35) 273 | 274 | for _ in range(15): 275 | q.get() 276 | # the first tail file gets removed after task_done 277 | q.task_done() 278 | for _ in range(16): 279 | q.get() 280 | # the second and third files get removed after task_done 281 | q.task_done() 282 | self.assertEqual(q.qsize(), 4) 283 | 284 | def test_protocol(self): 285 | # test that protocol is set properly 286 | expect_protocol = 2 if sys.version_info[0] == 2 else 4 287 | self.assertEqual( 288 | serializers_pickle.protocol, 289 | expect_protocol, 290 | ) 291 | 292 | # test that protocol is used properly 293 | serializer = namedtuple("Serializer", ["dump", "load"])( 294 | serializers_pickle.dump, lambda fp: fp.read()) 295 | 296 | q = Queue(path=self.path, serializer=serializer) 297 | q.put(b'a') 298 | self.assertEqual(q.get(), pickle.dumps(b'a', protocol=expect_protocol)) 299 | 300 | @params(*serializer_params) 301 | def test_del(self, serializer): 302 | """test that __del__ can be called successfully""" 303 | q = Queue(self.path, **serializer_params[serializer]) 304 | q.__del__() 305 | self.assertTrue(q.headf.closed) 306 | self.assertTrue(q.tailf.closed) 307 | 308 | @params(*serializer_params) 309 | def test_autosave_get(self, serializer): 310 | """test the autosave feature saves on get()""" 311 | q = Queue(self.path, autosave=True, **serializer_params[serializer]) 312 | q.put('var1') 313 | q.put('var2') 314 | self.assertEqual('var1', q.get()) 315 | del q 316 | # queue should save on get(), only one item should remain 317 | q = Queue(self.path, autosave=True, **serializer_params[serializer]) 318 | self.assertEqual(1, q.qsize()) 319 | self.assertEqual('var2', q.get()) 320 | del q 321 | 322 | @params(*serializer_params) 323 | def test_autosave_join(self, serializer): 324 | """Enabling autosave should still allow task_done/join behavior""" 325 | q = Queue(self.path, autosave=True, **serializer_params[serializer]) 326 | for i in range(10): 327 | q.put('var%d' % i) 328 | 329 | def consumer(): 330 | for i in range(10): 331 | q.get() 332 | # this should still 'count down' properly and allow q.join() 333 | # to finish 334 | q.task_done() 335 | 336 | c = Thread(target=consumer) 337 | c.start() 338 | q.join() 339 | with self.assertRaises(Empty): 340 | q.get_nowait() 341 | -------------------------------------------------------------------------------- /persistqueue/tests/test_sqlackqueue.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | 3 | import random 4 | import shutil 5 | import sys 6 | import tempfile 7 | import unittest 8 | from threading import Thread 9 | import uuid 10 | 11 | from persistqueue.sqlackqueue import ( 12 | SQLiteAckQueue, 13 | FILOSQLiteAckQueue, 14 | UniqueAckQ, 15 | ) 16 | from persistqueue import Empty 17 | 18 | 19 | class SQLite3AckQueueTest(unittest.TestCase): 20 | def setUp(self): 21 | self.path = tempfile.mkdtemp(suffix='sqlackqueue') 22 | self.auto_commit = True 23 | self.queue_class = SQLiteAckQueue 24 | 25 | def tearDown(self): 26 | shutil.rmtree(self.path, ignore_errors=True) 27 | 28 | def test_raise_empty(self): 29 | q = self.queue_class(self.path, auto_commit=self.auto_commit) 30 | 31 | q.put('first') 32 | d = q.get() 33 | self.assertEqual('first', d) 34 | self.assertRaises(Empty, q.get, block=False) 35 | 36 | # assert with timeout 37 | self.assertRaises(Empty, q.get, block=True, timeout=1.0) 38 | # assert with negative timeout 39 | self.assertRaises(ValueError, q.get, block=True, timeout=-1.0) 40 | 41 | def test_empty(self): 42 | q = self.queue_class(self.path, auto_commit=self.auto_commit) 43 | self.assertEqual(q.empty(), True) 44 | 45 | q.put('first') 46 | self.assertEqual(q.empty(), False) 47 | 48 | q.get() 49 | self.assertEqual(q.empty(), True) 50 | 51 | def test_full(self): 52 | # SQL queue `full()` always returns `False` !! 53 | q = self.queue_class(self.path, auto_commit=self.auto_commit) 54 | self.assertEqual(q.full(), False) 55 | 56 | q.put('first') 57 | self.assertEqual(q.full(), False) 58 | 59 | q.get() 60 | self.assertEqual(q.full(), False) 61 | 62 | def test_open_close_single(self): 63 | """Write 1 item, close, reopen checking if same item is there""" 64 | 65 | q = self.queue_class(self.path, auto_commit=self.auto_commit) 66 | q.put(b'var1') 67 | del q 68 | q = self.queue_class(self.path) 69 | self.assertEqual(1, q.qsize()) 70 | self.assertEqual(b'var1', q.get()) 71 | 72 | def test_open_close_1000(self): 73 | """Write 1000 items, close, reopen checking if all items are there""" 74 | 75 | q = self.queue_class(self.path, auto_commit=self.auto_commit) 76 | for i in range(1000): 77 | q.put('var%d' % i) 78 | 79 | self.assertEqual(1000, q.qsize()) 80 | del q 81 | q = self.queue_class(self.path) 82 | self.assertEqual(1000, q.qsize()) 83 | for i in range(1000): 84 | data = q.get() 85 | self.assertEqual('var%d' % i, data) 86 | # assert adding another one still works 87 | q.put('foobar') 88 | data = q.get() 89 | q.shrink_disk_usage() 90 | self.assertEqual('foobar', data) 91 | 92 | def test_random_read_write(self): 93 | """Test random read/write""" 94 | 95 | q = self.queue_class(self.path, auto_commit=self.auto_commit) 96 | n = 0 97 | for _ in range(1000): 98 | if random.random() < 0.5: 99 | if n > 0: 100 | q.get() 101 | n -= 1 102 | else: 103 | self.assertRaises(Empty, q.get, block=False) 104 | else: 105 | # UniqueQueue will block at get() if this is not unique 106 | # uuid.uuid4() should be unique 107 | q.put('var%s' % uuid.uuid4()) 108 | n += 1 109 | 110 | def test_multi_threaded_parallel(self): 111 | """Create consumer and producer threads, check parallelism""" 112 | 113 | # self.skipTest("Not supported multi-thread.") 114 | 115 | m_queue = self.queue_class( 116 | path=self.path, multithreading=True, auto_commit=self.auto_commit 117 | ) 118 | 119 | def producer(): 120 | for i in range(1000): 121 | m_queue.put('var%d' % i) 122 | 123 | def consumer(): 124 | for i in range(1000): 125 | x = m_queue.get(block=True) 126 | self.assertEqual('var%d' % i, x) 127 | 128 | c = Thread(target=consumer) 129 | c.start() 130 | p = Thread(target=producer) 131 | p.start() 132 | p.join() 133 | c.join() 134 | self.assertEqual(0, m_queue.size) 135 | self.assertEqual(0, len(m_queue)) 136 | self.assertRaises(Empty, m_queue.get, block=False) 137 | 138 | def test_multi_threaded_multi_producer(self): 139 | """Test sqlqueue can be used by multiple producers.""" 140 | queue = self.queue_class( 141 | path=self.path, multithreading=True, auto_commit=self.auto_commit 142 | ) 143 | 144 | def producer(seq): 145 | for i in range(10): 146 | queue.put('var%d' % (i + (seq * 10))) 147 | 148 | def consumer(): 149 | for _ in range(100): 150 | data = queue.get(block=True) 151 | self.assertTrue('var' in data) 152 | 153 | c = Thread(target=consumer) 154 | c.start() 155 | producers = [] 156 | for seq in range(10): 157 | t = Thread(target=producer, args=(seq,)) 158 | t.start() 159 | producers.append(t) 160 | 161 | for t in producers: 162 | t.join() 163 | 164 | c.join() 165 | 166 | def test_multiple_consumers(self): 167 | """Test sqlqueue can be used by multiple consumers.""" 168 | 169 | queue = self.queue_class( 170 | path=self.path, multithreading=True, auto_commit=self.auto_commit 171 | ) 172 | 173 | def producer(): 174 | for x in range(1000): 175 | queue.put('var%d' % x) 176 | 177 | counter = [] 178 | # Set all to 0 179 | for _ in range(1000): 180 | counter.append(0) 181 | 182 | def consumer(index): 183 | for i in range(200): 184 | data = queue.get(block=True) 185 | self.assertTrue('var' in data) 186 | counter[index * 200 + i] = data 187 | 188 | p = Thread(target=producer) 189 | p.start() 190 | consumers = [] 191 | for index in range(5): 192 | t = Thread(target=consumer, args=(index,)) 193 | t.start() 194 | consumers.append(t) 195 | 196 | p.join() 197 | for t in consumers: 198 | t.join() 199 | 200 | self.assertEqual(0, queue.qsize()) 201 | for x in range(1000): 202 | self.assertNotEqual( 203 | 0, counter[x], "not 0 for counter's index %s" % x 204 | ) 205 | 206 | def test_protocol_1(self): 207 | shutil.rmtree(self.path, ignore_errors=True) 208 | q = self.queue_class(path=self.path) 209 | self.assertEqual( 210 | q._serializer.protocol, 2 if sys.version_info[0] == 2 else 4 211 | ) 212 | 213 | def test_protocol_2(self): 214 | q = self.queue_class(path=self.path) 215 | self.assertEqual( 216 | q._serializer.protocol, 2 if sys.version_info[0] == 2 else 4 217 | ) 218 | 219 | def test_ack_and_clear(self): 220 | q = self.queue_class(path=self.path) 221 | ret_list = [] 222 | for _ in range(100): 223 | q.put("val%s" % _) 224 | for _ in range(100): 225 | ret_list.append(q.get()) 226 | for ret in ret_list: 227 | q.ack(ret) 228 | self.assertEqual(q.acked_count(), 100) 229 | q.clear_acked_data(keep_latest=10) 230 | self.assertEqual(q.acked_count(), 10) 231 | q.shrink_disk_usage() 232 | 233 | def test_ack_unknown_item(self): 234 | q = self.queue_class(path=self.path) 235 | q.put("val1") 236 | val1 = q.get() 237 | q.ack("val2") 238 | q.nack("val3") 239 | q.ack_failed("val4") 240 | self.assertEqual(q.qsize(), 0) 241 | self.assertEqual(q.unack_count(), 1) 242 | q.ack(val1) 243 | self.assertEqual(q.unack_count(), 0) 244 | 245 | def test_resume_unack(self): 246 | q = self.queue_class(path=self.path) 247 | q.put("val1") 248 | val1 = q.get() 249 | self.assertEqual(q.empty(), True) 250 | self.assertEqual(q.qsize(), 0) 251 | self.assertEqual(q.unack_count(), 1) 252 | self.assertEqual(q.ready_count(), 0) 253 | del q 254 | 255 | q = self.queue_class(path=self.path, auto_resume=False) 256 | self.assertEqual(q.empty(), True) 257 | self.assertEqual(q.qsize(), 0) 258 | self.assertEqual(q.unack_count(), 1) 259 | self.assertEqual(q.ready_count(), 0) 260 | q.resume_unack_tasks() 261 | self.assertEqual(q.empty(), False) 262 | self.assertEqual(q.qsize(), 1) 263 | self.assertEqual(q.unack_count(), 0) 264 | self.assertEqual(q.ready_count(), 1) 265 | self.assertEqual(val1, q.get()) 266 | del q 267 | 268 | q = self.queue_class(path=self.path, auto_resume=True) 269 | self.assertEqual(q.empty(), False) 270 | self.assertEqual(q.qsize(), 1) 271 | self.assertEqual(q.unack_count(), 0) 272 | self.assertEqual(q.ready_count(), 1) 273 | self.assertEqual(val1, q.get()) 274 | 275 | def test_ack_unack_ack_failed(self): 276 | q = self.queue_class(path=self.path) 277 | q.put("val1") 278 | q.put("val2") 279 | q.put("val3") 280 | val1 = q.get() 281 | val2 = q.get() 282 | val3 = q.get() 283 | # qsize should be zero when all item is getted from q 284 | self.assertEqual(q.qsize(), 0) 285 | self.assertEqual(q.unack_count(), 3) 286 | # active size should be equal to qsize + unack_count 287 | self.assertEqual(q.active_size(), 3) 288 | # nack will let the item requeued as ready status 289 | q.nack(val1) 290 | self.assertEqual(q.qsize(), 1) 291 | self.assertEqual(q.ready_count(), 1) 292 | # ack failed is just mark item as ack failed 293 | q.ack_failed(val3) 294 | self.assertEqual(q.ack_failed_count(), 1) 295 | # ack should not effect qsize 296 | q.ack(val2) 297 | self.assertEqual(q.acked_count(), 1) 298 | self.assertEqual(q.qsize(), 1) 299 | # all ack* related action will reduce unack count 300 | self.assertEqual(q.unack_count(), 0) 301 | # reget the nacked item 302 | ready_val = q.get() 303 | self.assertEqual(ready_val, val1) 304 | q.ack(ready_val) 305 | self.assertEqual(q.qsize(), 0) 306 | self.assertEqual(q.acked_count(), 2) 307 | self.assertEqual(q.ready_count(), 0) 308 | 309 | def test_put_0(self): 310 | q = self.queue_class(path=self.path) 311 | q.put(0) 312 | d = q.get(block=False) 313 | self.assertIsNotNone(d) 314 | 315 | def test_get_id(self): 316 | q = self.queue_class(path=self.path) 317 | q.put("val1") 318 | val2_id = q.put("val2") 319 | q.put("val3") 320 | item = q.get(id=val2_id) 321 | # item id should be 2 322 | self.assertEqual(val2_id, 2) 323 | # item should get val2 324 | self.assertEqual(item, 'val2') 325 | 326 | def test_get_next_in_order(self): 327 | q = self.queue_class(path=self.path) 328 | val1_id = q.put("val1") 329 | q.put("val2") 330 | q.put("val3") 331 | item = q.get(id=val1_id, next_in_order=True) 332 | # item id should be 1 333 | self.assertEqual(val1_id, 1) 334 | # item should get val2 335 | self.assertEqual(item, 'val2') 336 | q.nack(item) 337 | # queue should roll over to begining if next > end 338 | item = q.get(id=3, next_in_order=True, raw=True) 339 | q.nack(item) 340 | self.assertEqual(item.get("pqid"), 1) 341 | 342 | def test_get_raw(self): 343 | q = self.queue_class(path=self.path) 344 | q.put("val1") 345 | item = q.get(raw=True) 346 | q.nack(item) 347 | # item should get val2 348 | self.assertEqual(True, "pqid" in item) 349 | self.assertEqual(item.get("data"), 'val1') 350 | 351 | def test_nack_raw(self): 352 | q = self.queue_class(path=self.path) 353 | q.put("val1") 354 | item = q.get(raw=True) 355 | # nack a raw return 356 | q.nack(item) 357 | # size should be 1 after nack 358 | self.assertEqual(q.qsize(), 1) 359 | 360 | def test_ack_active_size(self): 361 | q = self.queue_class(path=self.path) 362 | q.put("val1") 363 | item = q.get(raw=True) 364 | # active_size should be 1 as it hasn't been acked 365 | self.assertEqual(q.active_size(), 1) 366 | q.ack(item) 367 | # active_size should be 0 after ack 368 | self.assertEqual(q.active_size(), 0) 369 | 370 | def test_queue(self): 371 | q = self.queue_class(path=self.path) 372 | q.put("val1") 373 | q.put("val2") 374 | q.put("val3") 375 | # queue should get the three items 376 | d = q.queue() 377 | self.assertEqual(len(d), 3) 378 | self.assertEqual(d[1].get("data"), "val2") 379 | 380 | def test_update(self): 381 | q = self.queue_class(path=self.path) 382 | qid = q.put("val1") 383 | q.update(id=qid, item="val2") 384 | item = q.get(id=qid) 385 | q.nack(item) 386 | self.assertEqual(item, "val2") 387 | 388 | 389 | class SQLite3QueueInMemory(SQLite3AckQueueTest): 390 | def setUp(self): 391 | self.path = ":memory:" 392 | self.auto_commit = True 393 | self.queue_class = SQLiteAckQueue 394 | 395 | def test_open_close_1000(self): 396 | self.skipTest('Memory based sqlite is not persistent.') 397 | 398 | def test_open_close_single(self): 399 | self.skipTest('Memory based sqlite is not persistent.') 400 | 401 | def test_multiple_consumers(self): 402 | self.skipTest( 403 | 'Skipped due to occasional crash during multithreading mode.' 404 | ) 405 | 406 | def test_multi_threaded_multi_producer(self): 407 | self.skipTest( 408 | 'Skipped due to occasional crash during multithreading mode.' 409 | ) 410 | 411 | def test_multi_threaded_parallel(self): 412 | self.skipTest( 413 | 'Skipped due to occasional crash during multithreading mode.' 414 | ) 415 | 416 | def test_task_done_with_restart(self): 417 | self.skipTest('Skipped due to not persistent.') 418 | 419 | def test_protocol_2(self): 420 | self.skipTest('In memory queue is always new.') 421 | 422 | def test_resume_unack(self): 423 | self.skipTest('Memory based sqlite is not persistent.') 424 | 425 | 426 | class FILOSQLite3AckQueueTest(SQLite3AckQueueTest): 427 | def setUp(self): 428 | self.path = tempfile.mkdtemp(suffix='filo_sqlackqueue') 429 | self.auto_commit = True 430 | self.queue_class = FILOSQLiteAckQueue 431 | 432 | def tearDown(self): 433 | shutil.rmtree(self.path, ignore_errors=True) 434 | 435 | def test_open_close_1000(self): 436 | """Write 1000 items, close, reopen checking if all items are there""" 437 | 438 | q = self.queue_class(self.path, auto_commit=self.auto_commit) 439 | for i in range(1000): 440 | q.put('var%d' % i) 441 | self.assertEqual(1000, q.qsize()) 442 | del q 443 | q = self.queue_class(self.path) 444 | self.assertEqual(1000, q.qsize()) 445 | for i in range(1000): 446 | data = q.get() 447 | self.assertEqual('var%d' % (999 - i), data) 448 | # assert adding another one still works 449 | q.put('foobar') 450 | data = q.get() 451 | q.nack(data) 452 | self.assertEqual('foobar', data) 453 | 454 | def test_multi_threaded_parallel(self): 455 | """Create consumer and producer threads, check parallelism""" 456 | 457 | # self.skipTest("Not supported multi-thread.") 458 | 459 | m_queue = self.queue_class( 460 | path=self.path, multithreading=True, auto_commit=self.auto_commit 461 | ) 462 | 463 | def producer(): 464 | for i in range(1000): 465 | m_queue.put('var%d' % i) 466 | 467 | def consumer(): 468 | # We cannot quarantee what next number will be like in FIFO 469 | for _ in range(1000): 470 | x = m_queue.get(block=True) 471 | self.assertTrue('var' in x) 472 | 473 | c = Thread(target=consumer) 474 | c.start() 475 | p = Thread(target=producer) 476 | p.start() 477 | p.join() 478 | c.join() 479 | self.assertEqual(0, m_queue.size) 480 | self.assertEqual(0, len(m_queue)) 481 | self.assertRaises(Empty, m_queue.get, block=False) 482 | 483 | def test_get_next_in_order(self): 484 | q = self.queue_class(path=self.path) 485 | val1_id = q.put("val1") 486 | q.put("val2") 487 | q.put("val3") 488 | item = q.get(id=val1_id, next_in_order=True) 489 | q.nack(item) 490 | # item id should be 1 491 | self.assertEqual(val1_id, 1) 492 | # item should get val2 493 | self.assertEqual(item, 'val3') 494 | # queue should roll over to end if next < begining 495 | item = q.get(id=1, next_in_order=True, raw=True) 496 | q.nack(item) 497 | self.assertEqual(item.get("pqid"), 3) 498 | 499 | 500 | # Note 501 | # We have to be carefull to avoid test cases from SQLite3AckQueueTest having 502 | # duplicate values in their q.put()'s. This could block the test indefinitely 503 | class SQLite3UniqueAckQueueTest(SQLite3AckQueueTest): 504 | def setUp(self): 505 | self.path = tempfile.mkdtemp(suffix='sqlackqueue') 506 | self.auto_commit = True 507 | self.queue_class = UniqueAckQ 508 | 509 | def test_add_duplicate_item(self): 510 | q = self.queue_class(self.path) 511 | q.put(1111) 512 | self.assertEqual(1, q.size) 513 | # put duplicate item 514 | q.put(1111) 515 | self.assertEqual(1, q.size) 516 | 517 | q.put(2222) 518 | self.assertEqual(2, q.size) 519 | 520 | del q 521 | q = self.queue_class(self.path) 522 | self.assertEqual(2, q.size) 523 | -------------------------------------------------------------------------------- /persistqueue/tests/test_sqlqueue.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | 3 | import random 4 | import shutil 5 | import sys 6 | import tempfile 7 | import unittest 8 | from threading import Thread 9 | 10 | from persistqueue import Empty 11 | from persistqueue import SQLiteQueue, FILOSQLiteQueue, UniqueQ 12 | from persistqueue.serializers import json as serializers_json 13 | from persistqueue.serializers import pickle as serializers_pickle 14 | from persistqueue.serializers import msgpack as serializers_msgpack 15 | from persistqueue.serializers import cbor2 as serializers_cbor2 16 | 17 | 18 | class SQLite3QueueTest(unittest.TestCase): 19 | def setUp(self): 20 | self.path = tempfile.mkdtemp(suffix='sqlqueue') 21 | self.auto_commit = True 22 | self.queue_class = SQLiteQueue 23 | 24 | def tearDown(self): 25 | shutil.rmtree(self.path, ignore_errors=True) 26 | 27 | def test_raise_empty(self): 28 | q = self.queue_class(self.path, auto_commit=self.auto_commit) 29 | 30 | q.put('first') 31 | d = q.get() 32 | self.assertEqual('first', d) 33 | self.assertRaises(Empty, q.get, block=False) 34 | self.assertRaises(Empty, q.get_nowait) 35 | 36 | # assert with timeout 37 | self.assertRaises(Empty, q.get, block=True, timeout=1.0) 38 | # assert with negative timeout 39 | self.assertRaises(ValueError, q.get, block=True, timeout=-1.0) 40 | del q 41 | 42 | def test_empty(self): 43 | q = self.queue_class(self.path, auto_commit=self.auto_commit) 44 | self.assertEqual(q.empty(), True) 45 | 46 | q.put('first') 47 | self.assertEqual(q.empty(), False) 48 | 49 | q.get() 50 | self.assertEqual(q.empty(), True) 51 | 52 | def test_full(self): 53 | # SQL queue `full()` always returns `False` !! 54 | q = self.queue_class(self.path, auto_commit=self.auto_commit) 55 | self.assertEqual(q.full(), False) 56 | 57 | q.put('first') 58 | self.assertEqual(q.full(), False) 59 | 60 | q.get() 61 | self.assertEqual(q.full(), False) 62 | 63 | def test_open_close_single(self): 64 | """Write 1 item, close, reopen checking if same item is there""" 65 | 66 | q = self.queue_class(self.path, auto_commit=self.auto_commit) 67 | q.put(b'var1') 68 | del q 69 | q = SQLiteQueue(self.path) 70 | self.assertEqual(1, q.qsize()) 71 | self.assertEqual(b'var1', q.get()) 72 | 73 | def test_open_close_1000(self): 74 | """Write 1000 items, close, reopen checking if all items are there""" 75 | 76 | q = self.queue_class(self.path, auto_commit=self.auto_commit) 77 | for i in range(1000): 78 | q.put('var%d' % i) 79 | 80 | self.assertEqual(1000, q.qsize()) 81 | del q 82 | q = SQLiteQueue(self.path) 83 | self.assertEqual(1000, q.qsize()) 84 | for i in range(1000): 85 | data = q.get() 86 | self.assertEqual('var%d' % i, data) 87 | # assert adding another one still works 88 | q.put('foobar') 89 | data = q.get() 90 | q.shrink_disk_usage() 91 | self.assertEqual('foobar', data) 92 | 93 | def test_random_read_write(self): 94 | """Test random read/write""" 95 | 96 | q = self.queue_class(self.path, auto_commit=self.auto_commit) 97 | n = 0 98 | for _ in range(1000): 99 | if random.random() < 0.5: 100 | if n > 0: 101 | q.get() 102 | n -= 1 103 | else: 104 | self.assertRaises(Empty, q.get, block=False) 105 | else: 106 | q.put('var%d' % random.getrandbits(16)) 107 | n += 1 108 | 109 | def test_multi_threaded_parallel(self): 110 | """Create consumer and producer threads, check parallelism""" 111 | 112 | # self.skipTest("Not supported multi-thread.") 113 | 114 | m_queue = SQLiteQueue(path=self.path, multithreading=True, 115 | auto_commit=self.auto_commit) 116 | 117 | def producer(): 118 | for i in range(1000): 119 | m_queue.put('var%d' % i) 120 | 121 | def consumer(): 122 | for i in range(1000): 123 | x = m_queue.get(block=True) 124 | self.assertEqual('var%d' % i, x) 125 | 126 | c = Thread(target=consumer) 127 | c.start() 128 | p = Thread(target=producer) 129 | p.start() 130 | p.join() 131 | c.join() 132 | self.assertEqual(0, m_queue.size) 133 | self.assertEqual(0, len(m_queue)) 134 | self.assertRaises(Empty, m_queue.get, block=False) 135 | 136 | def test_multi_threaded_multi_producer(self): 137 | """Test sqlqueue can be used by multiple producers.""" 138 | queue = self.queue_class(path=self.path, multithreading=True, 139 | auto_commit=self.auto_commit) 140 | 141 | def producer(seq): 142 | for i in range(10): 143 | queue.put('var%d' % (i + (seq * 10))) 144 | 145 | def consumer(): 146 | for _ in range(100): 147 | data = queue.get(block=True) 148 | self.assertTrue('var' in data) 149 | 150 | c = Thread(target=consumer) 151 | c.start() 152 | producers = [] 153 | for seq in range(10): 154 | t = Thread(target=producer, args=(seq,)) 155 | t.start() 156 | producers.append(t) 157 | 158 | for t in producers: 159 | t.join() 160 | 161 | c.join() 162 | 163 | def test_multiple_consumers(self): 164 | """Test sqlqueue can be used by multiple consumers.""" 165 | 166 | queue = self.queue_class(path=self.path, multithreading=True, 167 | auto_commit=self.auto_commit) 168 | 169 | def producer(): 170 | for x in range(1000): 171 | queue.put('var%d' % x) 172 | 173 | counter = [] 174 | # Set all to 0 175 | for _ in range(1000): 176 | counter.append(0) 177 | 178 | def consumer(index): 179 | for i in range(200): 180 | data = queue.get(block=True) 181 | self.assertTrue('var' in data) 182 | counter[index * 200 + i] = data 183 | 184 | p = Thread(target=producer) 185 | p.start() 186 | consumers = [] 187 | for index in range(5): 188 | t = Thread(target=consumer, args=(index,)) 189 | t.start() 190 | consumers.append(t) 191 | 192 | p.join() 193 | for t in consumers: 194 | t.join() 195 | 196 | self.assertEqual(0, queue.qsize()) 197 | for x in range(1000): 198 | self.assertNotEqual(0, counter[x], 199 | "not 0 for counter's index %s" % x) 200 | 201 | self.assertEqual(len(set(counter)), len(counter)) 202 | 203 | def test_task_done_with_restart(self): 204 | """Test that items are not deleted before task_done.""" 205 | 206 | q = self.queue_class(path=self.path, auto_commit=False) 207 | 208 | for i in range(1, 11): 209 | q.put(i) 210 | 211 | self.assertEqual(1, q.get()) 212 | self.assertEqual(2, q.get()) 213 | # size is correct before task_done 214 | self.assertEqual(8, q.qsize()) 215 | q.task_done() 216 | # make sure the size still correct 217 | self.assertEqual(8, q.qsize()) 218 | 219 | self.assertEqual(3, q.get()) 220 | # without task done 221 | del q 222 | q = SQLiteQueue(path=self.path, auto_commit=False) 223 | # After restart, the qsize and head item are the same 224 | self.assertEqual(8, q.qsize()) 225 | # After restart, the queue still works 226 | self.assertEqual(3, q.get()) 227 | self.assertEqual(7, q.qsize()) 228 | 229 | def test_protocol_1(self): 230 | shutil.rmtree(self.path, ignore_errors=True) 231 | q = self.queue_class(path=self.path) 232 | self.assertEqual(q._serializer.protocol, 233 | 2 if sys.version_info[0] == 2 else 4) 234 | 235 | def test_protocol_2(self): 236 | q = self.queue_class(path=self.path) 237 | self.assertEqual(q._serializer.protocol, 238 | 2 if sys.version_info[0] == 2 else 4) 239 | 240 | def test_json_serializer(self): 241 | q = self.queue_class( 242 | path=self.path, 243 | serializer=serializers_json) 244 | x = dict( 245 | a=1, 246 | b=2, 247 | c=dict( 248 | d=list(range(5)), 249 | e=[1] 250 | )) 251 | q.put(x) 252 | self.assertEqual(q.get(), x) 253 | 254 | def test_put_0(self): 255 | q = self.queue_class(path=self.path) 256 | q.put(0) 257 | d = q.get(block=False) 258 | self.assertIsNotNone(d) 259 | 260 | def test_get_id(self): 261 | q = self.queue_class(path=self.path) 262 | q.put("val1") 263 | val2_id = q.put("val2") 264 | q.put("val3") 265 | item = q.get(id=val2_id) 266 | # item id should be 2 267 | self.assertEqual(val2_id, 2) 268 | # item should get val2 269 | self.assertEqual(item, 'val2') 270 | 271 | def test_get_raw(self): 272 | q = self.queue_class(path=self.path) 273 | q.put("val1") 274 | item = q.get(raw=True) 275 | # item should get val2 276 | self.assertEqual(True, "pqid" in item) 277 | self.assertEqual(item.get("data"), 'val1') 278 | 279 | def test_queue(self): 280 | q = self.queue_class(path=self.path) 281 | q.put("val1") 282 | q.put("val2") 283 | q.put("val3") 284 | # queue should get the three items 285 | d = q.queue() 286 | self.assertEqual(len(d), 3) 287 | self.assertEqual(d[1].get("data"), "val2") 288 | 289 | def test_update(self): 290 | q = self.queue_class(path=self.path) 291 | qid = q.put("val1") 292 | q.update(item="val2", id=qid) 293 | item = q.get(id=qid) 294 | self.assertEqual(item, "val2") 295 | 296 | 297 | class SQLite3QueueNoAutoCommitTest(SQLite3QueueTest): 298 | def setUp(self): 299 | self.path = tempfile.mkdtemp(suffix='sqlqueue_auto_commit') 300 | self.auto_commit = False 301 | self.queue_class = SQLiteQueue 302 | 303 | def test_multiple_consumers(self): 304 | """ 305 | FAIL: test_multiple_consumers ( 306 | -tests.test_sqlqueue.SQLite3QueueNoAutoCommitTest) 307 | Test sqlqueue can be used by multiple consumers. 308 | ---------------------------------------------------------------------- 309 | Traceback (most recent call last): 310 | File "persist-queue\tests\test_sqlqueue.py", line 183, 311 | -in test_multiple_consumers 312 | self.assertEqual(0, queue.qsize()) 313 | AssertionError: 0 != 72 314 | :return: 315 | """ 316 | self.skipTest('Skipped due to a known bug above.') 317 | 318 | 319 | class SQLite3QueueInMemory(SQLite3QueueTest): 320 | skipstr = 'Skipped due to occasional crash during multithreading mode.' 321 | 322 | def setUp(self): 323 | self.path = ":memory:" 324 | self.auto_commit = True 325 | self.queue_class = SQLiteQueue 326 | 327 | def test_open_close_1000(self): 328 | self.skipTest('Memory based sqlite is not persistent.') 329 | 330 | def test_open_close_single(self): 331 | self.skipTest('Memory based sqlite is not persistent.') 332 | 333 | def test_multiple_consumers(self): 334 | self.skipTest(self.skipstr) 335 | 336 | def test_multi_threaded_multi_producer(self): 337 | self.skipTest(self.skipstr) 338 | 339 | def test_multi_threaded_parallel(self): 340 | self.skipTest(self.skipstr) 341 | 342 | def test_task_done_with_restart(self): 343 | self.skipTest('Skipped due to not persistent.') 344 | 345 | def test_protocol_2(self): 346 | self.skipTest('In memory queue is always new.') 347 | 348 | 349 | class FILOSQLite3QueueTest(unittest.TestCase): 350 | def setUp(self): 351 | self.path = tempfile.mkdtemp(suffix='filo_sqlqueue') 352 | self.auto_commit = True 353 | self.queue_class = SQLiteQueue 354 | 355 | def tearDown(self): 356 | shutil.rmtree(self.path, ignore_errors=True) 357 | 358 | def test_open_close_1000(self): 359 | """Write 1000 items, close, reopen checking if all items are there""" 360 | 361 | q = FILOSQLiteQueue(self.path, auto_commit=self.auto_commit) 362 | for i in range(1000): 363 | q.put('var%d' % i) 364 | self.assertEqual(1000, q.qsize()) 365 | del q 366 | q = FILOSQLiteQueue(self.path) 367 | self.assertEqual(1000, q.qsize()) 368 | for i in range(1000): 369 | data = q.get() 370 | self.assertEqual('var%d' % (999 - i), data) 371 | # assert adding another one still works 372 | q.put('foobar') 373 | data = q.get() 374 | self.assertEqual('foobar', data) 375 | 376 | 377 | class FILOSQLite3QueueNoAutoCommitTest(FILOSQLite3QueueTest): 378 | def setUp(self): 379 | self.path = tempfile.mkdtemp(suffix='filo_sqlqueue_auto_commit') 380 | self.auto_commit = False 381 | self.queue_class = FILOSQLiteQueue 382 | 383 | 384 | class SQLite3UniqueQueueTest(unittest.TestCase): 385 | def setUp(self): 386 | self.path = tempfile.mkdtemp(suffix='sqlqueue') 387 | self.auto_commit = True 388 | self.queue_class = UniqueQ 389 | 390 | def test_add_duplicate_item(self): 391 | q = UniqueQ(self.path) 392 | q.put(1111) 393 | self.assertEqual(1, q.size) 394 | # put duplicate item 395 | q.put(1111) 396 | self.assertEqual(1, q.size) 397 | 398 | q.put(2222) 399 | self.assertEqual(2, q.size) 400 | 401 | del q 402 | q = UniqueQ(self.path) 403 | self.assertEqual(2, q.size) 404 | 405 | def test_multiple_consumers(self): 406 | """Test UniqueQ can be used by multiple consumers.""" 407 | 408 | queue = UniqueQ(path=self.path, multithreading=True, 409 | auto_commit=self.auto_commit) 410 | 411 | def producer(): 412 | for x in range(1000): 413 | queue.put('var%d' % x) 414 | 415 | counter = [] 416 | # Set all to 0 417 | for _ in range(1000): 418 | counter.append(0) 419 | 420 | def consumer(index): 421 | for i in range(200): 422 | data = queue.get(block=True) 423 | self.assertTrue('var' in data) 424 | counter[index * 200 + i] = data 425 | 426 | p = Thread(target=producer) 427 | p.start() 428 | consumers = [] 429 | for index in range(5): 430 | t = Thread(target=consumer, args=(index,)) 431 | t.start() 432 | consumers.append(t) 433 | 434 | p.join() 435 | for t in consumers: 436 | t.join() 437 | 438 | self.assertEqual(0, queue.qsize()) 439 | for x in range(1000): 440 | self.assertNotEqual(0, counter[x], 441 | "not 0 for counter's index %s" % x) 442 | 443 | self.assertEqual(len(set(counter)), len(counter)) 444 | 445 | def test_unique_dictionary_serialization_pickle(self): 446 | queue = UniqueQ( 447 | path=self.path, 448 | multithreading=True, 449 | auto_commit=self.auto_commit, 450 | serializer=serializers_pickle, 451 | ) 452 | queue.put({"foo": 1, "bar": 2}) 453 | self.assertEqual(queue.total, 1) 454 | queue.put({"bar": 2, "foo": 1}) 455 | self.assertEqual(queue.total, 1) 456 | 457 | def test_unique_dictionary_serialization_msgpack(self): 458 | queue = UniqueQ( 459 | path=self.path, 460 | multithreading=True, 461 | auto_commit=self.auto_commit, 462 | serializer=serializers_msgpack 463 | ) 464 | queue.put({"foo": 1, "bar": 2}) 465 | self.assertEqual(queue.total, 1) 466 | queue.put({"bar": 2, "foo": 1}) 467 | self.assertEqual(queue.total, 1) 468 | 469 | def test_unique_dictionary_serialization_cbor2(self): 470 | queue = UniqueQ( 471 | path=self.path, 472 | multithreading=True, 473 | auto_commit=self.auto_commit, 474 | serializer=serializers_cbor2 475 | ) 476 | queue.put({"foo": 1, "bar": 2}) 477 | self.assertEqual(queue.total, 1) 478 | queue.put({"bar": 2, "foo": 1}) 479 | self.assertEqual(queue.total, 1) 480 | 481 | def test_unique_dictionary_serialization_json(self): 482 | queue = UniqueQ( 483 | path=self.path, 484 | multithreading=True, 485 | auto_commit=self.auto_commit, 486 | serializer=serializers_json 487 | ) 488 | queue.put({"foo": 1, "bar": 2}) 489 | self.assertEqual(queue.total, 1) 490 | queue.put({"bar": 2, "foo": 1}) 491 | self.assertEqual(queue.total, 1) 492 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/peter-wangxu/persist-queue/b194f07553379a8319ce1bd0adbb0ad4c610eba9/requirements.txt -------------------------------------------------------------------------------- /scripts/publish.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | BASE_DIR=`pwd` 4 | NAME=$(basename $BASE_DIR) 5 | if [[ "$NAME" != "persist-queue" ]];then 6 | echo "must run this in project root" 7 | exit 1 8 | fi 9 | rm -rf ./build/*.* ./dist/*.* 10 | python setup.py build sdist 11 | python setup.py build bdist_wheel # requires `pip install wheel` 12 | twine check ${BASE_DIR}/dist/*.tar.gz 13 | twine check ${BASE_DIR}/dist/*.whl 14 | twine upload ${BASE_DIR}/dist/* 15 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal = 0 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | 4 | from setuptools import setup, find_packages 5 | 6 | 7 | def get_extras(): 8 | return { 9 | "extra": open("extra-requirements.txt").read().splitlines() 10 | } 11 | 12 | 13 | setup( 14 | name='persist-queue', 15 | version=__import__('persistqueue').__version__, 16 | description=( 17 | 'A thread-safe disk based persistent queue in Python.' 18 | ), 19 | long_description=open('README.rst').read(), 20 | long_description_content_type='text/x-rst', 21 | author=__import__('persistqueue').__author__, 22 | author_email='wangxu198709@gmail.com', 23 | maintainer=__import__('persistqueue').__author__, 24 | maintainer_email='wangxu198709@gmail.com', 25 | license=__import__('persistqueue').__license__, 26 | packages=find_packages(), 27 | extras_require=get_extras(), 28 | platforms=["all"], 29 | url='http://github.com/peter-wangxu/persist-queue', 30 | classifiers=[ 31 | 'Development Status :: 4 - Beta', 32 | 'Operating System :: OS Independent', 33 | 'Intended Audience :: Developers', 34 | 'License :: OSI Approved :: BSD License', 35 | 'Programming Language :: Python', 36 | 'Programming Language :: Python :: Implementation', 37 | 'Programming Language :: Python :: 3', 38 | 'Programming Language :: Python :: 3.8', 39 | 'Programming Language :: Python :: 3.9', 40 | 'Programming Language :: Python :: 3.10', 41 | 'Programming Language :: Python :: 3.11', 42 | 'Programming Language :: Python :: 3.12', 43 | 'Topic :: Software Development :: Libraries' 44 | ], 45 | package_date={'persistqueue': ['py.typed']} 46 | ) 47 | -------------------------------------------------------------------------------- /test-requirements.txt: -------------------------------------------------------------------------------- 1 | tox 2 | mock>=2.0.0 3 | flake8>=3.2.1 4 | eventlet>=0.19.0 5 | msgpack>=0.5.6 6 | cbor2>=5.6.0 7 | nose2>=0.6.5 8 | coverage!=4.5 9 | cov_core>=1.15.0 10 | virtualenv>=15.1.0 11 | cryptography;sys_platform!="win32" # package only required for tests under mysql8.0&linux 12 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | 3 | minversion = 2.0 4 | skipsdist = True 5 | recreate = false 6 | envlist = py38, py39, py310, py311, py312, pep8, cover 7 | deps = -r{toxinidir}/test-requirements.txt 8 | -r{toxinidir}/extra-requirements.txt 9 | -r{toxinidir}/requirements.txt 10 | 11 | [testenv] 12 | 13 | setenv = VIRTUAL_ENV={envdir} 14 | 15 | usedevelop = True 16 | deps = -r{toxinidir}/test-requirements.txt 17 | -r{toxinidir}/extra-requirements.txt 18 | -r{toxinidir}/requirements.txt 19 | whitelist_externals = 20 | bash 21 | find 22 | commands = 23 | nose2 {posargs} 24 | 25 | [testenv:pep8] 26 | 27 | commands = 28 | flake8 ./persistqueue ./persistqueue/tests {posargs} 29 | 30 | [testenv:cover] 31 | 32 | commands = 33 | nose2 --with-coverage --coverage-report xml --coverage-report html --coverage-report term {posargs} 34 | --------------------------------------------------------------------------------