├── .codecov.yml ├── .github ├── FUNDING.yml └── workflows │ └── ci.yml ├── .gitignore ├── .pre-commit-config.yaml ├── HISTORY.rst ├── LICENSE ├── Makefile ├── README.md ├── arq ├── __init__.py ├── __main__.py ├── cli.py ├── connections.py ├── constants.py ├── cron.py ├── jobs.py ├── logs.py ├── py.typed ├── typing.py ├── utils.py ├── version.py └── worker.py ├── docs ├── Makefile ├── _templates │ └── layout.html ├── conf.py ├── examples │ ├── cron.py │ ├── custom_serialization_msgpack.py │ ├── deferred.py │ ├── job_abort.py │ ├── job_ids.py │ ├── job_results.py │ ├── main_demo.py │ ├── retry.py │ ├── slow_job.py │ ├── slow_job_output.txt │ └── sync_job.py ├── index.rst └── old-docs.zip ├── pyproject.toml ├── requirements ├── all.txt ├── docs.in ├── docs.txt ├── linting.in ├── linting.txt ├── pyproject.txt ├── testing.in └── testing.txt └── tests ├── __init__.py ├── conftest.py ├── test_cli.py ├── test_cron.py ├── test_jobs.py ├── test_main.py ├── test_utils.py └── test_worker.py /.codecov.yml: -------------------------------------------------------------------------------- 1 | coverage: 2 | precision: 2 3 | range: [95, 100] 4 | status: 5 | patch: false 6 | project: false 7 | 8 | comment: 9 | layout: 'header, diff, flags, files, footer' 10 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: samuelcolvin 2 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | tags: 8 | - '**' 9 | pull_request: {} 10 | 11 | jobs: 12 | lint: 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - uses: actions/checkout@v4 17 | 18 | - name: set up python 19 | uses: actions/setup-python@v5 20 | with: 21 | python-version: '3.11' 22 | 23 | - run: pip install -r requirements/linting.txt -r requirements/pyproject.txt pre-commit 24 | 25 | - run: pre-commit run -a --verbose 26 | env: 27 | SKIP: no-commit-to-branch 28 | 29 | docs: 30 | runs-on: ubuntu-latest 31 | 32 | steps: 33 | - uses: actions/checkout@v4 34 | 35 | - name: set up python 36 | uses: actions/setup-python@v5 37 | with: 38 | python-version: '3.11' 39 | 40 | - run: pip install -r requirements/docs.txt -r requirements/pyproject.txt 41 | - run: pip install . 42 | 43 | - run: make docs 44 | 45 | - name: Store docs site 46 | uses: actions/upload-artifact@v4 47 | with: 48 | name: docs 49 | path: docs/_build/ 50 | 51 | test: 52 | name: test py${{ matrix.python }} with redis:${{ matrix.redis }} on ${{ matrix.os }} 53 | strategy: 54 | fail-fast: false 55 | matrix: 56 | os: [ubuntu] 57 | python: ['3.8', '3.9', '3.10', '3.11', '3.12'] 58 | redis: ['5'] 59 | include: 60 | - python: '3.11' 61 | redis: '6' 62 | os: 'ubuntu' 63 | - python: '3.11' 64 | redis: '7' 65 | os: 'ubuntu' 66 | 67 | env: 68 | PYTHON: ${{ matrix.python }} 69 | OS: ${{ matrix.os }} 70 | ARQ_TEST_REDIS_VERSION: ${{ matrix.redis }} 71 | 72 | runs-on: ${{ matrix.os }}-latest 73 | 74 | steps: 75 | - uses: actions/checkout@v4 76 | 77 | - name: set up python 78 | uses: actions/setup-python@v5 79 | with: 80 | python-version: ${{ matrix.python }} 81 | 82 | - run: pip install -r requirements/testing.txt -r requirements/pyproject.txt 83 | 84 | - run: make test 85 | 86 | - run: coverage xml 87 | 88 | - uses: codecov/codecov-action@v4 89 | with: 90 | file: ./coverage.xml 91 | env_vars: PYTHON,OS 92 | 93 | check: 94 | if: always() 95 | needs: [lint, docs, test] 96 | runs-on: ubuntu-latest 97 | 98 | steps: 99 | - name: Decide whether the needed jobs succeeded or failed 100 | uses: re-actors/alls-green@release/v1 101 | id: all-green 102 | with: 103 | jobs: ${{ toJSON(needs) }} 104 | 105 | release: 106 | name: Release 107 | needs: [check] 108 | if: "success() && startsWith(github.ref, 'refs/tags/')" 109 | runs-on: ubuntu-latest 110 | environment: release 111 | 112 | permissions: 113 | id-token: write 114 | 115 | steps: 116 | - uses: actions/checkout@v4 117 | 118 | - name: get docs 119 | uses: actions/download-artifact@v4 120 | with: 121 | name: docs 122 | path: docs/_build/ 123 | 124 | - name: set up python 125 | uses: actions/setup-python@v5 126 | with: 127 | python-version: '3.11' 128 | 129 | - name: install 130 | run: pip install -U build 131 | 132 | - name: check version 133 | id: check-version 134 | uses: samuelcolvin/check-python-version@v3.2 135 | with: 136 | version_file_path: 'arq/version.py' 137 | 138 | - name: build 139 | run: python -m build 140 | 141 | - name: Upload package to PyPI 142 | uses: pypa/gh-action-pypi-publish@release/v1 143 | 144 | - name: publish docs 145 | if: '!fromJSON(steps.check-version.outputs.IS_PRERELEASE)' 146 | run: make publish-docs 147 | env: 148 | NETLIFY: ${{ secrets.netlify_token }} 149 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /env*/ 2 | /.idea 3 | __pycache__/ 4 | *.py[cod] 5 | *.cache 6 | .pytest_cache/ 7 | .coverage.* 8 | /.coverage 9 | /htmlcov/ 10 | /build 11 | /dist 12 | /demo.py 13 | *.egg-info 14 | /docs/_build/ 15 | /.mypy_cache/ 16 | /demo/tmp/ 17 | .vscode/ 18 | .venv/ 19 | /.auto-format 20 | /scratch/ 21 | .python-version 22 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | fail_fast: false 2 | 3 | repos: 4 | - repo: https://github.com/pre-commit/pre-commit-hooks 5 | rev: v4.3.0 6 | hooks: 7 | - id: no-commit-to-branch 8 | - id: check-yaml 9 | - id: check-toml 10 | - id: end-of-file-fixer 11 | - id: trailing-whitespace 12 | 13 | - repo: https://github.com/codespell-project/codespell 14 | rev: v2.2.4 15 | hooks: 16 | - id: codespell 17 | additional_dependencies: 18 | - tomli 19 | 20 | - repo: local 21 | hooks: 22 | - id: format 23 | name: Format 24 | entry: make format 25 | types: [python] 26 | language: system 27 | pass_filenames: false 28 | - id: mypy 29 | name: Mypy 30 | entry: make mypy 31 | types: [python] 32 | language: system 33 | pass_filenames: false 34 | -------------------------------------------------------------------------------- /HISTORY.rst: -------------------------------------------------------------------------------- 1 | .. :changelog: 2 | 3 | History 4 | ------- 5 | 6 | v0.26.3 (2025-01-06) 7 | .................... 8 | 9 | * Fix negative expires_ms and avoid worker freezing while using cron by @Matvey-Kuk in #479 10 | * Fix race condition on task retry by @RB387 in #487 11 | 12 | 13 | v0.26.1 (2023-08-29) 14 | .................... 15 | 16 | * Uses ``testcontainers`` to provide a redis instance for the unit tests by @chrisguidry in #470 17 | * Bump ``redis`` version from <5 to <6 by @Wh1isper in #460 18 | * Bump ``idna`` from 3.6 to 3.7 in /requirements by @dependabot in #444 19 | * Bump ``jinja2`` from 3.1.3 to 3.1.4 in /requirements by @dependabot in #452 20 | * Bump ``requests`` from 2.31.0 to 2.32.0 in /requirements by @dependabot in #461 21 | * Bump ``urllib3`` from 2.2.1 to 2.2.2 in /requirements by @dependabot in #464 22 | * Bump ``certifi`` from 2024.2.2 to 2024.7.4 in /requirements by @dependabot in #468 23 | 24 | v0.26.0 (2023-05-01) 25 | .................... 26 | 27 | No changes since v0.26.0b1. 28 | 29 | v0.26.0b1 (2023-04-01) 30 | ...................... 31 | 32 | * Prevent worker getting stuck in terminating state by @JonasKs in #370 33 | * Fix redis pipeline created and not used by @iamlikeme in #374 34 | * Bump certifi from 2022.6.15 to 2022.12.7 in ``/requirements`` by @dependabot in #373 35 | * Use instance's default queue for ``queued_jobs`` default by @phy1729 in #368 36 | * Docs: Add details about reusing a unique job id by @ross-nordstrom in #391 37 | * Delete ``setup.py`` in #398 38 | * Adding a job counter to address ``Semaphore`` issues by @rm-21 in #408 39 | * docs: add documentation on how to retrieve running jobs by @JonasKs in #377 40 | * feat: add ``job_id`` to ``JobDef``, closing #376 by @JonasKs in #378 41 | * chore: update dependencies, fixing tests by @JonasKs in #382 42 | * refactor: refactor all asserts into raise ````, close #371 by @JonasKs in #379 43 | * Fix: timezone info occasionally removed from cron job execution time by @iamlikeme in #383 44 | * 3.12 support, drop 3.7, uprev dependencies by @samuelcolvin in #439 45 | * Extend ``RedisSettings`` to include redis Retry Helper settings by @mernmic in #387 46 | * Allow to connect to Redis using a Unix socket URL… by @drygdryg in #392 47 | * Allow infinite retry by @vvmruder in #396 48 | * Allow ``max_connections`` to be set in RedisSettings by @danbox in #406 49 | * Improve ``RedisSettings`` explanation in ``main_demo.py`` by @RamonGiovane in #422 50 | * uprev to v0.26.0b1 by @samuelcolvin in #440 51 | 52 | v0.25 (2022-12-02) 53 | .................. 54 | 55 | * Allow to opt-out from logging results by @iamlikeme in #352 56 | * Add timezone support for cron jobs by @iamlikeme in #354 57 | * connections: fix pipeline usage for exists command by @utkarshgupta137 in #366 58 | * Fix race condition causing incorrect status not found by @iamlikeme in #362 59 | * Adds ``after_job_end`` hook by @AngellusMortis in #355 60 | * Raise ``ResultNotFound`` when ``Job.result()`` finds no job and no result by @iamlikeme in #364 61 | * use ``3.11`` for testing #367 62 | * Signal handler to wait for task completion before shutting down by @JonasKs in #345 63 | 64 | v0.24 (2022-09-05) 65 | .................. 66 | 67 | * Allow customisation of timezone in logs, #281 68 | * Add the ``username`` option to ``RedisSettings``, #299 69 | * Change primary branch name to ``main``, 40c8803 70 | * Add ``--custom-log-dict`` CLI option, #294 71 | * Fix error in case of pytz not being installed, #318 72 | * Support and test python 3.11, #327 73 | * Improve docs for parameter ``_expires`` in ``enqueue_job``, #313 74 | * Fix redis ssl support, #323 75 | * Fix recursion while waiting for redis connection, #311 76 | * switch from watchgod to watchfiles, #332 77 | * Simplify dependencies, drop pydantic as a dependency., #334 78 | * Allow use of ``unix_socket_path`` in ``RedisSettings``, #336 79 | * Allow user to configure a default job expiry-extra length, #303 80 | * Remove transaction around ``info`` command to support Redis 6.2.3, #338 81 | * Switch from ``setup.py`` to ``pyproject.toml``, #341 82 | * Support ``abort`` for deferred jobs, #307 83 | 84 | v0.23 (2022-08-23) 85 | .................. 86 | 87 | No changes from **v0.23a1**. 88 | 89 | v0.23a1 (2022-03-09) 90 | .................... 91 | * Fix jobs timeout by @kiriusm2 in #248 92 | * Update ``index.rst`` by @Kludex in #266 93 | * Improve some docs wording by @johtso in #285 94 | * fix error when cron jobs were terminanted by @tobymao in #273 95 | * add ``on_job_start`` and ``on_job_end`` hooks by @tobymao in #274 96 | * Update argument docstring definition by @sondrelg in #278 97 | * fix tests and uprev test dependencies, #288 98 | * Add link to WorkerSettings in documentation by @JonasKs in #279 99 | * Allow setting ``job_id`` on cron jobs by @JonasKs in #293 100 | * Fix docs typo by @johtso in #296 101 | * support aioredis v2 by @Yolley in #259 102 | * support python 3.10, #298 103 | 104 | v0.22 (2021-09-02) 105 | .................. 106 | * fix package importing in example, #261, thanks @cdpath 107 | * restrict ``aioredis`` to ``<2.0.0`` (soon we'll support ``aioredis>=2.0.0``), #258, thanks @PaxPrz 108 | * auto setting version on release, 759fe03 109 | 110 | v0.21 (2021-07-06) 111 | .................. 112 | * CI improvements #243 113 | * fix ``log_redis_info`` #255 114 | 115 | v0.20 (2021-04-26) 116 | .................. 117 | 118 | * Added ``queue_name`` attribute to ``JobResult``, #198 119 | * set ``job_deserializer``, ``job_serializer`` and ``default_queue_name`` on worker pools to better supported 120 | nested jobs, #203, #215 and #218 121 | * All job results to be kept indefinitely, #205 122 | * refactor ``cron`` jobs to prevent duplicate jobs, #200 123 | * correctly handle ``CancelledError`` in python 3.8+, #213 124 | * allow jobs to be aborted, #212 125 | * depreciate ``pole_delay`` and use correct spelling ``poll_delay``, #242 126 | * docs improvements, #207 and #232 127 | 128 | v0.19.1 (2020-10-26) 129 | .................... 130 | 131 | * fix timestamp issue in _defer_until without timezone offset, #182 132 | * add option to disable signal handler registration from running inside other frameworks, #183 133 | * add ``default_queue_name`` to ``create_redis_pool`` and ``ArqRedis``, #191 134 | * ``Worker`` can retrieve the ``queue_name`` from the connection pool, if present 135 | * fix potential race condition when starting jobs, #194 136 | * support python 3.9 and pydantic 1.7, #214 137 | 138 | v0.19.0 (2020-04-24) 139 | .................... 140 | * Python 3.8 support, #178 141 | * fix concurrency with multiple workers, #180 142 | * full mypy coverage, #181 143 | 144 | v0.18.4 (2019-12-19) 145 | .................... 146 | * Add ``py.typed`` file to tell mypy the package has type hints, #163 147 | * Added ``ssl`` option to ``RedisSettings``, #165 148 | 149 | v0.18.3 (2019-11-13) 150 | .................... 151 | * Include ``queue_name`` when for job object in response to ``enqueue_job``, #160 152 | 153 | v0.18.2 (2019-11-01) 154 | .................... 155 | * Fix cron scheduling on a specific queue, by @dmvass and @Tinche 156 | 157 | v0.18.1 (2019-10-28) 158 | .................... 159 | * add support for Redis Sentinel fix #132 160 | * fix ``Worker.abort_job`` invalid expire time error, by @dmvass 161 | 162 | v0.18 (2019-08-30) 163 | .................. 164 | * fix usage of ``max_burst_jobs``, improve coverage fix #152 165 | * stop lots of ``WatchVariableError`` errors in log, #153 166 | 167 | v0.17.1 (2019-08-21) 168 | .................... 169 | * deal better with failed job deserialization, #149 by @samuelcolvin 170 | * fix ``run_check(xmax_burst_jobs=...)`` when a jobs fails, #150 by @samuelcolvin 171 | 172 | v0.17 (2019-08-11) 173 | .................. 174 | * add ``worker.queue_read_limit``, fix #141, by @rubik 175 | * custom serializers, eg. to use msgpack rather than pickle, #143 by @rubik 176 | * add ``ArqRedis.queued_jobs`` utility method for getting queued jobs while testing, fix #145 by @samuelcolvin 177 | 178 | v0.16.1 (2019-08-02) 179 | .................... 180 | * prevent duplicate ``job_id`` when job result exists, fix #137 181 | * add "don't retry mode" via ``worker.retry_jobs = False``, fix #139 182 | * add ``worker.max_burst_jobs`` 183 | 184 | v0.16 (2019-07-30) 185 | .................. 186 | * improved error when a job is aborted (eg. function not found) 187 | 188 | v0.16.0b3 (2019-05-14) 189 | ...................... 190 | * fix semaphore on worker with many expired jobs 191 | 192 | v0.16.0b2 (2019-05-14) 193 | ...................... 194 | * add support for different queues, #127 thanks @tsutsarin 195 | 196 | v0.16.0b1 (2019-04-23) 197 | ...................... 198 | * use dicts for pickling not tuples, better handling of pickling errors, #123 199 | 200 | v0.16.0a5 (2019-04-22) 201 | ...................... 202 | * use ``pipeline`` in ``enqueue_job`` 203 | * catch any error when pickling job result 204 | * add support for python 3.6 205 | 206 | v0.16.0a4 (2019-03-15) 207 | ...................... 208 | * add ``Worker.run_check``, fix #115 209 | 210 | v0.16.0a3 (2019-03-12) 211 | ...................... 212 | * fix ``Worker`` with custom redis settings 213 | 214 | v0.16.0a2 (2019-03-06) 215 | ...................... 216 | * add ``job_try`` argument to ``enqueue_job``, #113 217 | * adding ``--watch`` mode to the worker (requires ``watchgod``), #114 218 | * allow ``ctx`` when creating Worker 219 | * add ``all_job_results`` to ``ArqRedis`` 220 | * fix python path when starting worker 221 | 222 | v0.16.0a1 (2019-03-05) 223 | ...................... 224 | * **Breaking Change:** **COMPLETE REWRITE!!!** see docs for details, #110 225 | 226 | v0.15.0 (2018-11-15) 227 | .................... 228 | * update dependencies 229 | * reconfigure ``Job``, return a job instance when enqueuing tasks #93 230 | * tweaks to docs #106 231 | 232 | v0.14.0 (2018-05-28) 233 | .................... 234 | * package updates, particularly compatibility for ``msgpack 0.5.6`` 235 | 236 | v0.13.0 (2017-11-27) 237 | .................... 238 | * **Breaking Change:** integration with aioredis >= 1.0, basic usage hasn't changed but 239 | look at aioredis's migration docs for changes in redis API #76 240 | 241 | v0.12.0 (2017-11-16) 242 | .................... 243 | * better signal handling, support ``uvloop`` #73 244 | * drain pending tasks and drain task cancellation #74 245 | * add aiohttp and docker demo ``/demo`` #75 246 | 247 | v0.11.0 (2017-08-25) 248 | .................... 249 | * extract ``create_pool_lenient`` from ``RedixMixin`` 250 | * improve redis connection traceback 251 | 252 | v0.10.4 (2017-08-22) 253 | .................... 254 | * ``RedisSettings`` repr method 255 | * add ``create_connection_timeout`` to connection pool 256 | 257 | v0.10.3 (2017-08-19) 258 | .................... 259 | * fix bug with ``RedisMixin.get_redis_pool`` creating multiple queues 260 | * tweak drain logs 261 | 262 | v0.10.2 (2017-08-17) 263 | .................... 264 | * only save job on task in drain if re-enqueuing 265 | * add semaphore timeout to drains 266 | * add key count to ``log_redis_info`` 267 | 268 | v0.10.1 (2017-08-16) 269 | .................... 270 | * correct format of ``log_redis_info`` 271 | 272 | v0.10.0 (2017-08-16) 273 | .................... 274 | * log redis version when starting worker, fix #64 275 | * log "connection success" when connecting to redis after connection failures, fix #67 276 | * add job ids, for now they're just used in logging, fix #53 277 | 278 | v0.9.0 (2017-06-23) 279 | ................... 280 | * allow set encoding in msgpack for jobs #49 281 | * cron tasks allowing scheduling of functions in the future #50 282 | * **Breaking change:** switch ``to_unix_ms`` to just return the timestamp int, add ``to_unix_ms_tz`` to 283 | return tz offset too 284 | 285 | v0.8.1 (2017-06-05) 286 | ................... 287 | * uprev setup requires 288 | * correct setup arguments 289 | 290 | v0.8.0 (2017-06-05) 291 | ................... 292 | * add ``async-timeout`` dependency 293 | * use async-timeout around ``shadow_factory`` 294 | * change logger name for control process log messages 295 | * use ``Semaphore`` rather than ``asyncio.wait(...return_when=asyncio.FIRST_COMPLETED)`` for improved performance 296 | * improve log display 297 | * add timeout and retry logic to ``RedisMixin.create_redis_pool`` 298 | 299 | v0.7.0 (2017-06-01) 300 | ................... 301 | * implementing reusable ``Drain`` which takes tasks from a redis list and allows them to be execute asynchronously. 302 | * Drain uses python 3.6 ``async yield``, therefore **python 3.5 is no longer supported**. 303 | * prevent repeated identical health check log messages 304 | 305 | v0.6.1 (2017-05-06) 306 | ................... 307 | * mypy at last passing, #30 308 | * adding trove classifiers, #29 309 | 310 | v0.6.0 (2017-04-14) 311 | ................... 312 | * add ``StopJob`` exception for cleaning ending jobs, #21 313 | * add ``flushdb`` to ``MockRedis``, #23 314 | * allow configurable length job logging via ``log_curtail`` on ``Worker``, #28 315 | 316 | v0.5.2 (2017-02-25) 317 | ................... 318 | * add ``shadow_kwargs`` method to ``BaseWorker`` to make customising actors easier. 319 | 320 | v0.5.1 (2017-02-25) 321 | ................... 322 | * reimplement worker reuse as it turned out to be useful in tests. 323 | 324 | v0.5.0 (2017-02-20) 325 | ................... 326 | * use ``gather`` rather than ``wait`` for startup and shutdown so exceptions propagate. 327 | * add ``--check`` option to confirm arq worker is running. 328 | 329 | v0.4.1 (2017-02-11) 330 | ................... 331 | * fix issue with ``Concurrent`` class binding with multiple actor instances. 332 | 333 | v0.4.0 (2017-02-10) 334 | ................... 335 | * improving naming of log handlers and formatters 336 | * upgrade numerous packages, nothing significant 337 | * add ``startup`` and ``shutdown`` methods to actors 338 | * switch ``@concurrent`` to return a ``Concurrent`` instance so the direct method is accessible via ``.direct`` 339 | 340 | v0.3.2 (2017-01-24) 341 | ................... 342 | * improved solution for preventing new jobs starting when the worker is about to stop 343 | * switch ``SIGRTMIN`` > ``SIGUSR1`` to work with mac 344 | 345 | v0.3.1 (2017-01-20) 346 | ................... 347 | * fix main process signal handling so the worker shuts down when just the main process receives a signal 348 | * re-enqueue un-started jobs popped from the queue if the worker is about to exit 349 | 350 | v0.3.0 (2017-01-19) 351 | ................... 352 | * rename settings class to ``RedisSettings`` and simplify significantly 353 | 354 | v0.2.0 (2016-12-09) 355 | ................... 356 | * add ``concurrency_enabled`` argument to aid in testing 357 | * fix conflict with unitest.mock 358 | 359 | v0.1.0 (2016-12-06) 360 | ................... 361 | * prevent logs disabling other logs 362 | 363 | v0.0.6 (2016-08-14) 364 | ................... 365 | * first proper release 366 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2017 - 2022 Samuel Colvin and other contributors 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .DEFAULT_GOAL := all 2 | sources = arq tests 3 | 4 | .PHONY: install 5 | install: 6 | pip install -U pip pre-commit pip-tools 7 | pip install -r requirements/all.txt 8 | pip install -e .[watch] 9 | pre-commit install 10 | 11 | .PHONY: refresh-lockfiles 12 | refresh-lockfiles: 13 | find requirements/ -name '*.txt' ! -name 'all.txt' -type f -delete 14 | make update-lockfiles 15 | 16 | .PHONY: update-lockfiles 17 | update-lockfiles: 18 | @echo "Updating requirements/*.txt files using pip-compile" 19 | pip-compile -q --strip-extras -o requirements/linting.txt requirements/linting.in 20 | pip-compile -q --strip-extras -o requirements/testing.txt requirements/testing.in 21 | pip-compile -q --strip-extras -o requirements/docs.txt requirements/docs.in 22 | pip-compile -q --strip-extras -o requirements/pyproject.txt pyproject.toml --all-extras 23 | pip install --dry-run -r requirements/all.txt 24 | 25 | .PHONY: format 26 | format: 27 | ruff check --fix $(sources) 28 | ruff format $(sources) 29 | 30 | .PHONY: lint 31 | lint: 32 | ruff check $(sources) 33 | ruff format --check $(sources) 34 | 35 | .PHONY: test 36 | test: 37 | coverage run -m pytest 38 | 39 | .PHONY: testcov 40 | testcov: test 41 | @echo "building coverage html" 42 | @coverage html 43 | 44 | .PHONY: mypy 45 | mypy: 46 | mypy arq 47 | 48 | .PHONY: all 49 | all: lint mypy testcov 50 | 51 | .PHONY: clean 52 | clean: 53 | rm -rf `find . -name __pycache__` 54 | rm -f `find . -type f -name '*.py[co]' ` 55 | rm -f `find . -type f -name '*~' ` 56 | rm -f `find . -type f -name '.*~' ` 57 | rm -rf .cache 58 | rm -rf .pytest_cache 59 | rm -rf .mypy_cache 60 | rm -rf htmlcov 61 | rm -rf *.egg-info 62 | rm -f .coverage 63 | rm -f .coverage.* 64 | rm -rf build 65 | make -C docs clean 66 | 67 | .PHONY: docs 68 | docs: 69 | make -C docs html 70 | rm -rf docs/_build/html/old 71 | unzip -q docs/old-docs.zip 72 | mv old-docs docs/_build/html/old 73 | @echo "open file://`pwd`/docs/_build/html/index.html" 74 | 75 | .PHONY: publish-docs 76 | publish-docs: 77 | cd docs/_build/ && cp -r html site && zip -r site.zip site 78 | @curl -H "Content-Type: application/zip" -H "Authorization: Bearer ${NETLIFY}" \ 79 | --data-binary "@docs/_build/site.zip" https://api.netlify.com/api/v1/sites/arq-docs.netlify.com/deploys 80 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # arq 2 | 3 | [![CI](https://img.shields.io/github/actions/workflow/status/samuelcolvin/arq/ci.yml?branch=main&logo=github&label=CI)](https://github.com/samuelcolvin/arq/actions?query=event%3Apush+branch%3Amain+workflow%3ACI) 4 | [![Coverage](https://codecov.io/gh/samuelcolvin/arq/branch/main/graph/badge.svg)](https://codecov.io/gh/samuelcolvin/arq) 5 | [![downloads](https://static.pepy.tech/badge/arq/month)](https://pepy.tech/project/arq) 6 | [![pypi](https://img.shields.io/pypi/v/arq.svg)](https://pypi.python.org/pypi/arq) 7 | [![versions](https://img.shields.io/pypi/pyversions/arq.svg)](https://github.com/samuelcolvin/arq) 8 | [![license](https://img.shields.io/github/license/samuelcolvin/arq.svg)](https://github.com/samuelcolvin/arq/blob/main/LICENSE) 9 | 10 | Job queues in python with asyncio and redis. 11 | 12 | See [documentation](https://arq-docs.helpmanual.io/) for more details. 13 | -------------------------------------------------------------------------------- /arq/__init__.py: -------------------------------------------------------------------------------- 1 | from .connections import ArqRedis, create_pool 2 | from .cron import cron 3 | from .version import VERSION 4 | from .worker import Retry, Worker, check_health, func, run_worker 5 | 6 | __version__ = VERSION 7 | 8 | __all__ = ( 9 | 'ArqRedis', 10 | 'create_pool', 11 | 'cron', 12 | 'VERSION', 13 | 'Retry', 14 | 'Worker', 15 | 'check_health', 16 | 'func', 17 | 'run_worker', 18 | ) 19 | -------------------------------------------------------------------------------- /arq/__main__.py: -------------------------------------------------------------------------------- 1 | from .cli import cli 2 | 3 | if __name__ == '__main__': 4 | cli() 5 | -------------------------------------------------------------------------------- /arq/cli.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import logging.config 3 | import os 4 | import sys 5 | from signal import Signals 6 | from typing import TYPE_CHECKING, cast 7 | 8 | import click 9 | 10 | from .logs import default_log_config 11 | from .utils import import_string 12 | from .version import VERSION 13 | from .worker import check_health, create_worker, run_worker 14 | 15 | if TYPE_CHECKING: 16 | from .typing import WorkerSettingsType 17 | 18 | burst_help = 'Batch mode: exit once no jobs are found in any queue.' 19 | health_check_help = 'Health Check: run a health check and exit.' 20 | watch_help = 'Watch a directory and reload the worker upon changes.' 21 | verbose_help = 'Enable verbose output.' 22 | logdict_help = "Import path for a dictionary in logdict form, to configure Arq's own logging." 23 | 24 | 25 | @click.command('arq') 26 | @click.version_option(VERSION, '-V', '--version', prog_name='arq') 27 | @click.argument('worker-settings', type=str, required=True) 28 | @click.option('--burst/--no-burst', default=None, help=burst_help) 29 | @click.option('--check', is_flag=True, help=health_check_help) 30 | @click.option('--watch', type=click.Path(exists=True, dir_okay=True, file_okay=False), help=watch_help) 31 | @click.option('-v', '--verbose', is_flag=True, help=verbose_help) 32 | @click.option('--custom-log-dict', type=str, help=logdict_help) 33 | def cli(*, worker_settings: str, burst: bool, check: bool, watch: str, verbose: bool, custom_log_dict: str) -> None: 34 | """ 35 | Job queues in python with asyncio and redis. 36 | 37 | CLI to run the arq worker. 38 | """ 39 | sys.path.append(os.getcwd()) 40 | worker_settings_ = cast('WorkerSettingsType', import_string(worker_settings)) 41 | if custom_log_dict: 42 | log_config = import_string(custom_log_dict) 43 | else: 44 | log_config = default_log_config(verbose) 45 | logging.config.dictConfig(log_config) 46 | 47 | if check: 48 | exit(check_health(worker_settings_)) 49 | else: 50 | kwargs = {} if burst is None else {'burst': burst} 51 | if watch: 52 | asyncio.run(watch_reload(watch, worker_settings_)) 53 | else: 54 | run_worker(worker_settings_, **kwargs) 55 | 56 | 57 | async def watch_reload(path: str, worker_settings: 'WorkerSettingsType') -> None: 58 | try: 59 | from watchfiles import awatch 60 | except ImportError as e: # pragma: no cover 61 | raise ImportError('watchfiles not installed, use `pip install watchfiles`') from e 62 | 63 | loop = asyncio.get_running_loop() 64 | stop_event = asyncio.Event() 65 | 66 | def worker_on_stop(s: Signals) -> None: 67 | if s != Signals.SIGUSR1: # pragma: no cover 68 | stop_event.set() 69 | 70 | worker = create_worker(worker_settings) 71 | try: 72 | worker.on_stop = worker_on_stop 73 | loop.create_task(worker.async_run()) 74 | async for _ in awatch(path, stop_event=stop_event): 75 | print('\nfiles changed, reloading arq worker...') 76 | worker.handle_sig(Signals.SIGUSR1) 77 | await worker.close() 78 | loop.create_task(worker.async_run()) 79 | finally: 80 | await worker.close() 81 | -------------------------------------------------------------------------------- /arq/connections.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import functools 3 | import logging 4 | from dataclasses import dataclass 5 | from datetime import datetime, timedelta 6 | from operator import attrgetter 7 | from typing import TYPE_CHECKING, Any, Callable, List, Optional, Tuple, Union, cast 8 | from urllib.parse import parse_qs, urlparse 9 | from uuid import uuid4 10 | 11 | from redis.asyncio import ConnectionPool, Redis 12 | from redis.asyncio.retry import Retry 13 | from redis.asyncio.sentinel import Sentinel 14 | from redis.exceptions import RedisError, WatchError 15 | 16 | from .constants import default_queue_name, expires_extra_ms, job_key_prefix, result_key_prefix 17 | from .jobs import Deserializer, Job, JobDef, JobResult, Serializer, deserialize_job, serialize_job 18 | from .utils import timestamp_ms, to_ms, to_unix_ms 19 | 20 | logger = logging.getLogger('arq.connections') 21 | 22 | 23 | @dataclass 24 | class RedisSettings: 25 | """ 26 | No-Op class used to hold redis connection redis_settings. 27 | 28 | Used by :func:`arq.connections.create_pool` and :class:`arq.worker.Worker`. 29 | """ 30 | 31 | host: Union[str, List[Tuple[str, int]]] = 'localhost' 32 | port: int = 6379 33 | unix_socket_path: Optional[str] = None 34 | database: int = 0 35 | username: Optional[str] = None 36 | password: Optional[str] = None 37 | ssl: bool = False 38 | ssl_keyfile: Optional[str] = None 39 | ssl_certfile: Optional[str] = None 40 | ssl_cert_reqs: str = 'required' 41 | ssl_ca_certs: Optional[str] = None 42 | ssl_ca_data: Optional[str] = None 43 | ssl_check_hostname: bool = False 44 | conn_timeout: int = 1 45 | conn_retries: int = 5 46 | conn_retry_delay: int = 1 47 | max_connections: Optional[int] = None 48 | 49 | sentinel: bool = False 50 | sentinel_master: str = 'mymaster' 51 | 52 | retry_on_timeout: bool = False 53 | retry_on_error: Optional[List[Exception]] = None 54 | retry: Optional[Retry] = None 55 | 56 | @classmethod 57 | def from_dsn(cls, dsn: str) -> 'RedisSettings': 58 | conf = urlparse(dsn) 59 | if conf.scheme not in {'redis', 'rediss', 'unix'}: 60 | raise RuntimeError('invalid DSN scheme') 61 | query_db = parse_qs(conf.query).get('db') 62 | if query_db: 63 | # e.g. redis://localhost:6379?db=1 64 | database = int(query_db[0]) 65 | elif conf.scheme != 'unix': 66 | database = int(conf.path.lstrip('/')) if conf.path else 0 67 | else: 68 | database = 0 69 | return RedisSettings( 70 | host=conf.hostname or 'localhost', 71 | port=conf.port or 6379, 72 | ssl=conf.scheme == 'rediss', 73 | username=conf.username, 74 | password=conf.password, 75 | database=database, 76 | unix_socket_path=conf.path if conf.scheme == 'unix' else None, 77 | ) 78 | 79 | def __repr__(self) -> str: 80 | return 'RedisSettings({})'.format(', '.join(f'{k}={v!r}' for k, v in self.__dict__.items())) 81 | 82 | 83 | if TYPE_CHECKING: 84 | BaseRedis = Redis[bytes] 85 | else: 86 | BaseRedis = Redis 87 | 88 | 89 | class ArqRedis(BaseRedis): 90 | """ 91 | Thin subclass of ``redis.asyncio.Redis`` which adds :func:`arq.connections.enqueue_job`. 92 | 93 | :param redis_settings: an instance of ``arq.connections.RedisSettings``. 94 | :param job_serializer: a function that serializes Python objects to bytes, defaults to pickle.dumps 95 | :param job_deserializer: a function that deserializes bytes into Python objects, defaults to pickle.loads 96 | :param default_queue_name: the default queue name to use, defaults to ``arq.queue``. 97 | :param expires_extra_ms: the default length of time from when a job is expected to start 98 | after which the job expires, defaults to 1 day in ms. 99 | :param kwargs: keyword arguments directly passed to ``redis.asyncio.Redis``. 100 | """ 101 | 102 | def __init__( 103 | self, 104 | pool_or_conn: Optional[ConnectionPool] = None, 105 | job_serializer: Optional[Serializer] = None, 106 | job_deserializer: Optional[Deserializer] = None, 107 | default_queue_name: str = default_queue_name, 108 | expires_extra_ms: int = expires_extra_ms, 109 | **kwargs: Any, 110 | ) -> None: 111 | self.job_serializer = job_serializer 112 | self.job_deserializer = job_deserializer 113 | self.default_queue_name = default_queue_name 114 | if pool_or_conn: 115 | kwargs['connection_pool'] = pool_or_conn 116 | self.expires_extra_ms = expires_extra_ms 117 | super().__init__(**kwargs) 118 | 119 | async def enqueue_job( 120 | self, 121 | function: str, 122 | *args: Any, 123 | _job_id: Optional[str] = None, 124 | _queue_name: Optional[str] = None, 125 | _defer_until: Optional[datetime] = None, 126 | _defer_by: Union[None, int, float, timedelta] = None, 127 | _expires: Union[None, int, float, timedelta] = None, 128 | _job_try: Optional[int] = None, 129 | **kwargs: Any, 130 | ) -> Optional[Job]: 131 | """ 132 | Enqueue a job. 133 | 134 | :param function: Name of the function to call 135 | :param args: args to pass to the function 136 | :param _job_id: ID of the job, can be used to enforce job uniqueness 137 | :param _queue_name: queue of the job, can be used to create job in different queue 138 | :param _defer_until: datetime at which to run the job 139 | :param _defer_by: duration to wait before running the job 140 | :param _expires: do not start or retry a job after this duration; 141 | defaults to 24 hours plus deferring time, if any 142 | :param _job_try: useful when re-enqueueing jobs within a job 143 | :param kwargs: any keyword arguments to pass to the function 144 | :return: :class:`arq.jobs.Job` instance or ``None`` if a job with this ID already exists 145 | """ 146 | if _queue_name is None: 147 | _queue_name = self.default_queue_name 148 | job_id = _job_id or uuid4().hex 149 | job_key = job_key_prefix + job_id 150 | if _defer_until and _defer_by: 151 | raise RuntimeError("use either 'defer_until' or 'defer_by' or neither, not both") 152 | 153 | defer_by_ms = to_ms(_defer_by) 154 | expires_ms = to_ms(_expires) 155 | 156 | async with self.pipeline(transaction=True) as pipe: 157 | await pipe.watch(job_key) 158 | if await pipe.exists(job_key, result_key_prefix + job_id): 159 | await pipe.reset() 160 | return None 161 | 162 | enqueue_time_ms = timestamp_ms() 163 | if _defer_until is not None: 164 | score = to_unix_ms(_defer_until) 165 | elif defer_by_ms: 166 | score = enqueue_time_ms + defer_by_ms 167 | else: 168 | score = enqueue_time_ms 169 | 170 | expires_ms = expires_ms or score - enqueue_time_ms + self.expires_extra_ms 171 | 172 | job = serialize_job(function, args, kwargs, _job_try, enqueue_time_ms, serializer=self.job_serializer) 173 | pipe.multi() 174 | pipe.psetex(job_key, expires_ms, job) 175 | pipe.zadd(_queue_name, {job_id: score}) 176 | try: 177 | await pipe.execute() 178 | except WatchError: 179 | # job got enqueued since we checked 'job_exists' 180 | return None 181 | return Job(job_id, redis=self, _queue_name=_queue_name, _deserializer=self.job_deserializer) 182 | 183 | async def _get_job_result(self, key: bytes) -> JobResult: 184 | job_id = key[len(result_key_prefix) :].decode() 185 | job = Job(job_id, self, _deserializer=self.job_deserializer) 186 | r = await job.result_info() 187 | if r is None: 188 | raise KeyError(f'job "{key.decode()}" not found') 189 | r.job_id = job_id 190 | return r 191 | 192 | async def all_job_results(self) -> List[JobResult]: 193 | """ 194 | Get results for all jobs in redis. 195 | """ 196 | keys = await self.keys(result_key_prefix + '*') 197 | results = await asyncio.gather(*[self._get_job_result(k) for k in keys]) 198 | return sorted(results, key=attrgetter('enqueue_time')) 199 | 200 | async def _get_job_def(self, job_id: bytes, score: int) -> JobDef: 201 | key = job_key_prefix + job_id.decode() 202 | v = await self.get(key) 203 | if v is None: 204 | raise RuntimeError(f'job "{key}" not found') 205 | jd = deserialize_job(v, deserializer=self.job_deserializer) 206 | jd.score = score 207 | jd.job_id = job_id.decode() 208 | return jd 209 | 210 | async def queued_jobs(self, *, queue_name: Optional[str] = None) -> List[JobDef]: 211 | """ 212 | Get information about queued, mostly useful when testing. 213 | """ 214 | if queue_name is None: 215 | queue_name = self.default_queue_name 216 | jobs = await self.zrange(queue_name, withscores=True, start=0, end=-1) 217 | return await asyncio.gather(*[self._get_job_def(job_id, int(score)) for job_id, score in jobs]) 218 | 219 | 220 | async def create_pool( 221 | settings_: Optional[RedisSettings] = None, 222 | *, 223 | retry: int = 0, 224 | job_serializer: Optional[Serializer] = None, 225 | job_deserializer: Optional[Deserializer] = None, 226 | default_queue_name: str = default_queue_name, 227 | expires_extra_ms: int = expires_extra_ms, 228 | ) -> ArqRedis: 229 | """ 230 | Create a new redis pool, retrying up to ``conn_retries`` times if the connection fails. 231 | 232 | Returns a :class:`arq.connections.ArqRedis` instance, thus allowing job enqueuing. 233 | """ 234 | settings: RedisSettings = RedisSettings() if settings_ is None else settings_ 235 | 236 | if isinstance(settings.host, str) and settings.sentinel: 237 | raise RuntimeError("str provided for 'host' but 'sentinel' is true; list of sentinels expected") 238 | 239 | if settings.sentinel: 240 | 241 | def pool_factory(*args: Any, **kwargs: Any) -> ArqRedis: 242 | client = Sentinel( # type: ignore[misc] 243 | *args, 244 | sentinels=settings.host, 245 | ssl=settings.ssl, 246 | **kwargs, 247 | ) 248 | redis = client.master_for(settings.sentinel_master, redis_class=ArqRedis) 249 | return cast(ArqRedis, redis) 250 | 251 | else: 252 | pool_factory = functools.partial( 253 | ArqRedis, 254 | host=settings.host, 255 | port=settings.port, 256 | unix_socket_path=settings.unix_socket_path, 257 | socket_connect_timeout=settings.conn_timeout, 258 | ssl=settings.ssl, 259 | ssl_keyfile=settings.ssl_keyfile, 260 | ssl_certfile=settings.ssl_certfile, 261 | ssl_cert_reqs=settings.ssl_cert_reqs, 262 | ssl_ca_certs=settings.ssl_ca_certs, 263 | ssl_ca_data=settings.ssl_ca_data, 264 | ssl_check_hostname=settings.ssl_check_hostname, 265 | retry=settings.retry, 266 | retry_on_timeout=settings.retry_on_timeout, 267 | retry_on_error=settings.retry_on_error, 268 | max_connections=settings.max_connections, 269 | ) 270 | 271 | while True: 272 | try: 273 | pool = pool_factory( 274 | db=settings.database, username=settings.username, password=settings.password, encoding='utf8' 275 | ) 276 | pool.job_serializer = job_serializer 277 | pool.job_deserializer = job_deserializer 278 | pool.default_queue_name = default_queue_name 279 | pool.expires_extra_ms = expires_extra_ms 280 | await pool.ping() 281 | 282 | except (ConnectionError, OSError, RedisError, asyncio.TimeoutError) as e: 283 | if retry < settings.conn_retries: 284 | logger.warning( 285 | 'redis connection error %s:%s %s %s, %s retries remaining...', 286 | settings.host, 287 | settings.port, 288 | e.__class__.__name__, 289 | e, 290 | settings.conn_retries - retry, 291 | ) 292 | await asyncio.sleep(settings.conn_retry_delay) 293 | retry = retry + 1 294 | else: 295 | raise 296 | else: 297 | if retry > 0: 298 | logger.info('redis connection successful') 299 | return pool 300 | 301 | 302 | async def log_redis_info(redis: 'Redis[bytes]', log_func: Callable[[str], Any]) -> None: 303 | async with redis.pipeline(transaction=False) as pipe: 304 | pipe.info(section='Server') 305 | pipe.info(section='Memory') 306 | pipe.info(section='Clients') 307 | pipe.dbsize() 308 | info_server, info_memory, info_clients, key_count = await pipe.execute() 309 | 310 | redis_version = info_server.get('redis_version', '?') 311 | mem_usage = info_memory.get('used_memory_human', '?') 312 | clients_connected = info_clients.get('connected_clients', '?') 313 | 314 | log_func( 315 | f'redis_version={redis_version} ' 316 | f'mem_usage={mem_usage} ' 317 | f'clients_connected={clients_connected} ' 318 | f'db_keys={key_count}' 319 | ) 320 | -------------------------------------------------------------------------------- /arq/constants.py: -------------------------------------------------------------------------------- 1 | default_queue_name = 'arq:queue' 2 | job_key_prefix = 'arq:job:' 3 | in_progress_key_prefix = 'arq:in-progress:' 4 | result_key_prefix = 'arq:result:' 5 | retry_key_prefix = 'arq:retry:' 6 | abort_jobs_ss = 'arq:abort' 7 | # age of items in the abort_key sorted set after which they're deleted 8 | abort_job_max_age = 60 9 | health_check_key_suffix = ':health-check' 10 | # how long to keep the "in_progress" key after a cron job ends to prevent the job duplication 11 | # this can be a long time since each cron job has an ID that is unique for the intended execution time 12 | keep_cronjob_progress = 60 13 | 14 | # used by `ms_to_datetime` to get the timezone 15 | timezone_env_vars = 'ARQ_TIMEZONE', 'arq_timezone', 'TIMEZONE', 'timezone' 16 | 17 | # extra time after the job is expected to start when the job key should expire, 1 day in ms 18 | expires_extra_ms = 86_400_000 19 | -------------------------------------------------------------------------------- /arq/cron.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import dataclasses 3 | from dataclasses import dataclass 4 | from datetime import datetime, timedelta 5 | from typing import Optional, Union 6 | 7 | from .typing import WEEKDAYS, OptionType, SecondsTimedelta, WeekdayOptionType, WorkerCoroutine 8 | from .utils import import_string, to_seconds 9 | 10 | 11 | @dataclass 12 | class Options: 13 | month: OptionType 14 | day: OptionType 15 | weekday: WeekdayOptionType 16 | hour: OptionType 17 | minute: OptionType 18 | second: OptionType 19 | microsecond: int 20 | 21 | 22 | def next_cron( 23 | previous_dt: datetime, 24 | *, 25 | month: OptionType = None, 26 | day: OptionType = None, 27 | weekday: WeekdayOptionType = None, 28 | hour: OptionType = None, 29 | minute: OptionType = None, 30 | second: OptionType = 0, 31 | microsecond: int = 123_456, 32 | ) -> datetime: 33 | """ 34 | Find the next datetime matching the given parameters. 35 | """ 36 | dt = previous_dt + timedelta(seconds=1) 37 | if isinstance(weekday, str): 38 | weekday = WEEKDAYS.index(weekday.lower()) 39 | options = Options( 40 | month=month, day=day, weekday=weekday, hour=hour, minute=minute, second=second, microsecond=microsecond 41 | ) 42 | 43 | while True: 44 | next_dt = _get_next_dt(dt, options) 45 | # print(dt, next_dt) 46 | if next_dt is None: 47 | return dt 48 | dt = next_dt 49 | 50 | 51 | def _get_next_dt(dt_: datetime, options: Options) -> Optional[datetime]: # noqa: C901 52 | for field, v in dataclasses.asdict(options).items(): 53 | if v is None: 54 | continue 55 | if field == 'weekday': 56 | next_v = dt_.weekday() 57 | else: 58 | next_v = getattr(dt_, field) 59 | if isinstance(v, int): 60 | mismatch = next_v != v 61 | elif isinstance(v, (set, list, tuple)): 62 | mismatch = next_v not in v 63 | else: 64 | raise RuntimeError(v) 65 | # print(field, v, next_v, mismatch) 66 | if mismatch: 67 | micro = max(dt_.microsecond - options.microsecond, 0) 68 | if field == 'month': 69 | if dt_.month == 12: 70 | return datetime(dt_.year + 1, 1, 1, tzinfo=dt_.tzinfo) 71 | else: 72 | return datetime(dt_.year, dt_.month + 1, 1, tzinfo=dt_.tzinfo) 73 | elif field in ('day', 'weekday'): 74 | return ( 75 | dt_ 76 | + timedelta(days=1) 77 | - timedelta(hours=dt_.hour, minutes=dt_.minute, seconds=dt_.second, microseconds=micro) 78 | ) 79 | elif field == 'hour': 80 | return dt_ + timedelta(hours=1) - timedelta(minutes=dt_.minute, seconds=dt_.second, microseconds=micro) 81 | elif field == 'minute': 82 | return dt_ + timedelta(minutes=1) - timedelta(seconds=dt_.second, microseconds=micro) 83 | elif field == 'second': 84 | return dt_ + timedelta(seconds=1) - timedelta(microseconds=micro) 85 | else: 86 | if field != 'microsecond': 87 | raise RuntimeError(field) 88 | return dt_ + timedelta(microseconds=options.microsecond - dt_.microsecond) 89 | return None 90 | 91 | 92 | @dataclass 93 | class CronJob: 94 | name: str 95 | coroutine: WorkerCoroutine 96 | month: OptionType 97 | day: OptionType 98 | weekday: WeekdayOptionType 99 | hour: OptionType 100 | minute: OptionType 101 | second: OptionType 102 | microsecond: int 103 | run_at_startup: bool 104 | unique: bool 105 | job_id: Optional[str] 106 | timeout_s: Optional[float] 107 | keep_result_s: Optional[float] 108 | keep_result_forever: Optional[bool] 109 | max_tries: Optional[int] 110 | next_run: Optional[datetime] = None 111 | 112 | def calculate_next(self, prev_run: datetime) -> None: 113 | self.next_run = next_cron( 114 | prev_run, 115 | month=self.month, 116 | day=self.day, 117 | weekday=self.weekday, 118 | hour=self.hour, 119 | minute=self.minute, 120 | second=self.second, 121 | microsecond=self.microsecond, 122 | ) 123 | 124 | def __repr__(self) -> str: 125 | return ''.format(' '.join(f'{k}={v}' for k, v in self.__dict__.items())) 126 | 127 | 128 | def cron( 129 | coroutine: Union[str, WorkerCoroutine], 130 | *, 131 | name: Optional[str] = None, 132 | month: OptionType = None, 133 | day: OptionType = None, 134 | weekday: WeekdayOptionType = None, 135 | hour: OptionType = None, 136 | minute: OptionType = None, 137 | second: OptionType = 0, 138 | microsecond: int = 123_456, 139 | run_at_startup: bool = False, 140 | unique: bool = True, 141 | job_id: Optional[str] = None, 142 | timeout: Optional[SecondsTimedelta] = None, 143 | keep_result: Optional[float] = 0, 144 | keep_result_forever: Optional[bool] = False, 145 | max_tries: Optional[int] = 1, 146 | ) -> CronJob: 147 | """ 148 | Create a cron job, eg. it should be executed at specific times. 149 | 150 | Workers will enqueue this job at or just after the set times. If ``unique`` is true (the default) the 151 | job will only be run once even if multiple workers are running. 152 | 153 | :param coroutine: coroutine function to run 154 | :param name: name of the job, if None, the name of the coroutine is used 155 | :param month: month(s) to run the job on, 1 - 12 156 | :param day: day(s) to run the job on, 1 - 31 157 | :param weekday: week day(s) to run the job on, 0 - 6 or mon - sun 158 | :param hour: hour(s) to run the job on, 0 - 23 159 | :param minute: minute(s) to run the job on, 0 - 59 160 | :param second: second(s) to run the job on, 0 - 59 161 | :param microsecond: microsecond(s) to run the job on, 162 | defaults to 123456 as the world is busier at the top of a second, 0 - 1e6 163 | :param run_at_startup: whether to run as worker starts 164 | :param unique: whether the job should only be executed once at each time (useful if you have multiple workers) 165 | :param job_id: ID of the job, can be used to enforce job uniqueness, spanning multiple cron schedules 166 | :param timeout: job timeout 167 | :param keep_result: how long to keep the result for 168 | :param keep_result_forever: whether to keep results forever 169 | :param max_tries: maximum number of tries for the job 170 | """ 171 | 172 | if isinstance(coroutine, str): 173 | name = name or 'cron:' + coroutine 174 | coroutine_: WorkerCoroutine = import_string(coroutine) 175 | else: 176 | coroutine_ = coroutine 177 | 178 | if not asyncio.iscoroutinefunction(coroutine_): 179 | raise RuntimeError(f'{coroutine_} is not a coroutine function') 180 | timeout = to_seconds(timeout) 181 | keep_result = to_seconds(keep_result) 182 | 183 | return CronJob( 184 | name or 'cron:' + coroutine_.__qualname__, 185 | coroutine_, 186 | month, 187 | day, 188 | weekday, 189 | hour, 190 | minute, 191 | second, 192 | microsecond, 193 | run_at_startup, 194 | unique, 195 | job_id, 196 | timeout, 197 | keep_result, 198 | keep_result_forever, 199 | max_tries, 200 | ) 201 | -------------------------------------------------------------------------------- /arq/jobs.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import logging 3 | import pickle 4 | import warnings 5 | from dataclasses import dataclass 6 | from datetime import datetime 7 | from enum import Enum 8 | from typing import Any, Callable, Dict, Optional, Tuple 9 | 10 | from redis.asyncio import Redis 11 | 12 | from .constants import abort_jobs_ss, default_queue_name, in_progress_key_prefix, job_key_prefix, result_key_prefix 13 | from .utils import ms_to_datetime, poll, timestamp_ms 14 | 15 | logger = logging.getLogger('arq.jobs') 16 | 17 | Serializer = Callable[[Dict[str, Any]], bytes] 18 | Deserializer = Callable[[bytes], Dict[str, Any]] 19 | 20 | 21 | class ResultNotFound(RuntimeError): 22 | pass 23 | 24 | 25 | class JobStatus(str, Enum): 26 | """ 27 | Enum of job statuses. 28 | """ 29 | 30 | #: job is in the queue, time it should be run not yet reached 31 | deferred = 'deferred' 32 | #: job is in the queue, time it should run has been reached 33 | queued = 'queued' 34 | #: job is in progress 35 | in_progress = 'in_progress' 36 | #: job is complete, result is available 37 | complete = 'complete' 38 | #: job not found in any way 39 | not_found = 'not_found' 40 | 41 | 42 | @dataclass 43 | class JobDef: 44 | function: str 45 | args: Tuple[Any, ...] 46 | kwargs: Dict[str, Any] 47 | job_try: int 48 | enqueue_time: datetime 49 | score: Optional[int] 50 | job_id: Optional[str] 51 | 52 | def __post_init__(self) -> None: 53 | if isinstance(self.score, float): 54 | self.score = int(self.score) 55 | 56 | 57 | @dataclass 58 | class JobResult(JobDef): 59 | success: bool 60 | result: Any 61 | start_time: datetime 62 | finish_time: datetime 63 | queue_name: str 64 | 65 | 66 | class Job: 67 | """ 68 | Holds data a reference to a job. 69 | """ 70 | 71 | __slots__ = 'job_id', '_redis', '_queue_name', '_deserializer' 72 | 73 | def __init__( 74 | self, 75 | job_id: str, 76 | redis: 'Redis[bytes]', 77 | _queue_name: str = default_queue_name, 78 | _deserializer: Optional[Deserializer] = None, 79 | ): 80 | self.job_id = job_id 81 | self._redis = redis 82 | self._queue_name = _queue_name 83 | self._deserializer = _deserializer 84 | 85 | async def result( 86 | self, timeout: Optional[float] = None, *, poll_delay: float = 0.5, pole_delay: Optional[float] = None 87 | ) -> Any: 88 | """ 89 | Get the result of the job or, if the job raised an exception, reraise it. 90 | 91 | This function waits for the result if it's not yet available and the job is 92 | present in the queue. Otherwise ``ResultNotFound`` is raised. 93 | 94 | :param timeout: maximum time to wait for the job result before raising ``TimeoutError``, will wait forever 95 | :param poll_delay: how often to poll redis for the job result 96 | :param pole_delay: deprecated, use poll_delay instead 97 | """ 98 | if pole_delay is not None: 99 | warnings.warn( 100 | '"pole_delay" is deprecated, use the correct spelling "poll_delay" instead', DeprecationWarning 101 | ) 102 | poll_delay = pole_delay 103 | 104 | async for delay in poll(poll_delay): 105 | async with self._redis.pipeline(transaction=True) as tr: 106 | tr.get(result_key_prefix + self.job_id) 107 | tr.zscore(self._queue_name, self.job_id) 108 | v, s = await tr.execute() 109 | 110 | if v: 111 | info = deserialize_result(v, deserializer=self._deserializer) 112 | if info.success: 113 | return info.result 114 | elif isinstance(info.result, (Exception, asyncio.CancelledError)): 115 | raise info.result 116 | else: 117 | raise SerializationError(info.result) 118 | elif s is None: 119 | raise ResultNotFound( 120 | 'Not waiting for job result because the job is not in queue. ' 121 | 'Is the worker function configured to keep result?' 122 | ) 123 | 124 | if timeout is not None and delay > timeout: 125 | raise asyncio.TimeoutError() 126 | 127 | async def info(self) -> Optional[JobDef]: 128 | """ 129 | All information on a job, including its result if it's available, does not wait for the result. 130 | """ 131 | info: Optional[JobDef] = await self.result_info() 132 | if not info: 133 | v = await self._redis.get(job_key_prefix + self.job_id) 134 | if v: 135 | info = deserialize_job(v, deserializer=self._deserializer) 136 | if info: 137 | s = await self._redis.zscore(self._queue_name, self.job_id) 138 | info.score = None if s is None else int(s) 139 | return info 140 | 141 | async def result_info(self) -> Optional[JobResult]: 142 | """ 143 | Information about the job result if available, does not wait for the result. Does not raise an exception 144 | even if the job raised one. 145 | """ 146 | v = await self._redis.get(result_key_prefix + self.job_id) 147 | if v: 148 | return deserialize_result(v, deserializer=self._deserializer) 149 | else: 150 | return None 151 | 152 | async def status(self) -> JobStatus: 153 | """ 154 | Status of the job. 155 | """ 156 | async with self._redis.pipeline(transaction=True) as tr: 157 | tr.exists(result_key_prefix + self.job_id) 158 | tr.exists(in_progress_key_prefix + self.job_id) 159 | tr.zscore(self._queue_name, self.job_id) 160 | is_complete, is_in_progress, score = await tr.execute() 161 | 162 | if is_complete: 163 | return JobStatus.complete 164 | elif is_in_progress: 165 | return JobStatus.in_progress 166 | elif score: 167 | return JobStatus.deferred if score > timestamp_ms() else JobStatus.queued 168 | else: 169 | return JobStatus.not_found 170 | 171 | async def abort(self, *, timeout: Optional[float] = None, poll_delay: float = 0.5) -> bool: 172 | """ 173 | Abort the job. 174 | 175 | :param timeout: maximum time to wait for the job result before raising ``TimeoutError``, 176 | will wait forever on None 177 | :param poll_delay: how often to poll redis for the job result 178 | :return: True if the job aborted properly, False otherwise 179 | """ 180 | job_info = await self.info() 181 | if job_info and job_info.score and job_info.score > timestamp_ms(): 182 | async with self._redis.pipeline(transaction=True) as tr: 183 | tr.zrem(self._queue_name, self.job_id) 184 | tr.zadd(self._queue_name, {self.job_id: 1}) 185 | await tr.execute() 186 | 187 | await self._redis.zadd(abort_jobs_ss, {self.job_id: timestamp_ms()}) 188 | 189 | try: 190 | await self.result(timeout=timeout, poll_delay=poll_delay) 191 | except asyncio.CancelledError: 192 | return True 193 | except ResultNotFound: 194 | # We do not know if the job was cancelled or not 195 | return False 196 | else: 197 | return False 198 | 199 | def __repr__(self) -> str: 200 | return f'' 201 | 202 | 203 | class SerializationError(RuntimeError): 204 | pass 205 | 206 | 207 | class DeserializationError(SerializationError): 208 | pass 209 | 210 | 211 | def serialize_job( 212 | function_name: str, 213 | args: Tuple[Any, ...], 214 | kwargs: Dict[str, Any], 215 | job_try: Optional[int], 216 | enqueue_time_ms: int, 217 | *, 218 | serializer: Optional[Serializer] = None, 219 | ) -> bytes: 220 | data = {'t': job_try, 'f': function_name, 'a': args, 'k': kwargs, 'et': enqueue_time_ms} 221 | if serializer is None: 222 | serializer = pickle.dumps 223 | try: 224 | return serializer(data) 225 | except Exception as e: 226 | raise SerializationError(f'unable to serialize job "{function_name}"') from e 227 | 228 | 229 | def serialize_result( 230 | function: str, 231 | args: Tuple[Any, ...], 232 | kwargs: Dict[str, Any], 233 | job_try: int, 234 | enqueue_time_ms: int, 235 | success: bool, 236 | result: Any, 237 | start_ms: int, 238 | finished_ms: int, 239 | ref: str, 240 | queue_name: str, 241 | job_id: str, 242 | *, 243 | serializer: Optional[Serializer] = None, 244 | ) -> Optional[bytes]: 245 | data = { 246 | 't': job_try, 247 | 'f': function, 248 | 'a': args, 249 | 'k': kwargs, 250 | 'et': enqueue_time_ms, 251 | 's': success, 252 | 'r': result, 253 | 'st': start_ms, 254 | 'ft': finished_ms, 255 | 'q': queue_name, 256 | 'id': job_id, 257 | } 258 | if serializer is None: 259 | serializer = pickle.dumps 260 | try: 261 | return serializer(data) 262 | except Exception: 263 | logger.warning('error serializing result of %s', ref, exc_info=True) 264 | 265 | # use string in case serialization fails again 266 | data.update(r='unable to serialize result', s=False) 267 | try: 268 | return serializer(data) 269 | except Exception: 270 | logger.critical('error serializing result of %s even after replacing result', ref, exc_info=True) 271 | return None 272 | 273 | 274 | def deserialize_job(r: bytes, *, deserializer: Optional[Deserializer] = None) -> JobDef: 275 | if deserializer is None: 276 | deserializer = pickle.loads 277 | try: 278 | d = deserializer(r) 279 | return JobDef( 280 | function=d['f'], 281 | args=d['a'], 282 | kwargs=d['k'], 283 | job_try=d['t'], 284 | enqueue_time=ms_to_datetime(d['et']), 285 | score=None, 286 | job_id=None, 287 | ) 288 | except Exception as e: 289 | raise DeserializationError('unable to deserialize job') from e 290 | 291 | 292 | def deserialize_job_raw( 293 | r: bytes, *, deserializer: Optional[Deserializer] = None 294 | ) -> Tuple[str, Tuple[Any, ...], Dict[str, Any], int, int]: 295 | if deserializer is None: 296 | deserializer = pickle.loads 297 | try: 298 | d = deserializer(r) 299 | return d['f'], d['a'], d['k'], d['t'], d['et'] 300 | except Exception as e: 301 | raise DeserializationError('unable to deserialize job') from e 302 | 303 | 304 | def deserialize_result(r: bytes, *, deserializer: Optional[Deserializer] = None) -> JobResult: 305 | if deserializer is None: 306 | deserializer = pickle.loads 307 | try: 308 | d = deserializer(r) 309 | return JobResult( 310 | job_try=d['t'], 311 | function=d['f'], 312 | args=d['a'], 313 | kwargs=d['k'], 314 | enqueue_time=ms_to_datetime(d['et']), 315 | score=None, 316 | success=d['s'], 317 | result=d['r'], 318 | start_time=ms_to_datetime(d['st']), 319 | finish_time=ms_to_datetime(d['ft']), 320 | queue_name=d.get('q', ''), 321 | job_id=d.get('id', ''), 322 | ) 323 | except Exception as e: 324 | raise DeserializationError('unable to deserialize job result') from e 325 | -------------------------------------------------------------------------------- /arq/logs.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict 2 | 3 | 4 | def default_log_config(verbose: bool) -> Dict[str, Any]: 5 | """ 6 | Setup default config. for dictConfig. 7 | 8 | :param verbose: level: DEBUG if True, INFO if False 9 | :return: dict suitable for ``logging.config.dictConfig`` 10 | """ 11 | log_level = 'DEBUG' if verbose else 'INFO' 12 | return { 13 | 'version': 1, 14 | 'disable_existing_loggers': False, 15 | 'handlers': { 16 | 'arq.standard': {'level': log_level, 'class': 'logging.StreamHandler', 'formatter': 'arq.standard'} 17 | }, 18 | 'formatters': {'arq.standard': {'format': '%(asctime)s: %(message)s', 'datefmt': '%H:%M:%S'}}, 19 | 'loggers': {'arq': {'handlers': ['arq.standard'], 'level': log_level}}, 20 | } 21 | -------------------------------------------------------------------------------- /arq/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/python-arq/arq/7a911f37992bad4556a27756f2ff027dd2afe655/arq/py.typed -------------------------------------------------------------------------------- /arq/typing.py: -------------------------------------------------------------------------------- 1 | from datetime import timedelta 2 | from typing import TYPE_CHECKING, Any, Dict, Literal, Optional, Protocol, Sequence, Set, Type, Union 3 | 4 | __all__ = ( 5 | 'OptionType', 6 | 'WeekdayOptionType', 7 | 'WEEKDAYS', 8 | 'SecondsTimedelta', 9 | 'WorkerCoroutine', 10 | 'StartupShutdown', 11 | 'WorkerSettingsType', 12 | ) 13 | 14 | 15 | if TYPE_CHECKING: 16 | from .cron import CronJob 17 | from .worker import Function 18 | 19 | OptionType = Union[None, Set[int], int] 20 | WEEKDAYS = 'mon', 'tues', 'wed', 'thurs', 'fri', 'sat', 'sun' 21 | WeekdayOptionType = Union[OptionType, Literal['mon', 'tues', 'wed', 'thurs', 'fri', 'sat', 'sun']] 22 | SecondsTimedelta = Union[int, float, timedelta] 23 | 24 | 25 | class WorkerCoroutine(Protocol): 26 | __qualname__: str 27 | 28 | async def __call__(self, ctx: Dict[Any, Any], *args: Any, **kwargs: Any) -> Any: # pragma: no cover 29 | pass 30 | 31 | 32 | class StartupShutdown(Protocol): 33 | __qualname__: str 34 | 35 | async def __call__(self, ctx: Dict[Any, Any]) -> Any: # pragma: no cover 36 | pass 37 | 38 | 39 | class WorkerSettingsBase(Protocol): 40 | functions: Sequence[Union[WorkerCoroutine, 'Function']] 41 | cron_jobs: Optional[Sequence['CronJob']] = None 42 | on_startup: Optional[StartupShutdown] = None 43 | on_shutdown: Optional[StartupShutdown] = None 44 | # and many more... 45 | 46 | 47 | WorkerSettingsType = Union[Dict[str, Any], Type[WorkerSettingsBase]] 48 | -------------------------------------------------------------------------------- /arq/utils.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import logging 3 | import os 4 | from datetime import datetime, timedelta, timezone 5 | from functools import lru_cache 6 | from time import time 7 | from typing import TYPE_CHECKING, Any, AsyncGenerator, Dict, Optional, Sequence, overload 8 | 9 | from .constants import timezone_env_vars 10 | 11 | try: 12 | import pytz 13 | except ImportError: # pragma: no cover 14 | pytz = None # type: ignore 15 | 16 | logger = logging.getLogger('arq.utils') 17 | 18 | if TYPE_CHECKING: 19 | from .typing import SecondsTimedelta 20 | 21 | 22 | def as_int(f: float) -> int: 23 | return int(round(f)) 24 | 25 | 26 | def timestamp_ms() -> int: 27 | return as_int(time() * 1000) 28 | 29 | 30 | def to_unix_ms(dt: datetime) -> int: 31 | """ 32 | convert a datetime to epoch with milliseconds as int 33 | """ 34 | return as_int(dt.timestamp() * 1000) 35 | 36 | 37 | @lru_cache 38 | def get_tz() -> Optional['pytz.BaseTzInfo']: 39 | if pytz: # pragma: no branch 40 | for timezone_key in timezone_env_vars: 41 | tz_name = os.getenv(timezone_key) 42 | if tz_name: 43 | try: 44 | return pytz.timezone(tz_name) 45 | except KeyError: 46 | logger.warning('unknown timezone: %r', tz_name) 47 | 48 | return None 49 | 50 | 51 | def ms_to_datetime(unix_ms: int) -> datetime: 52 | """ 53 | convert milliseconds to datetime, use the timezone in os.environ 54 | """ 55 | dt = datetime.fromtimestamp(unix_ms / 1000, tz=timezone.utc) 56 | tz = get_tz() 57 | if tz: 58 | dt = dt.astimezone(tz) 59 | return dt 60 | 61 | 62 | @overload 63 | def to_ms(td: None) -> None: 64 | pass 65 | 66 | 67 | @overload 68 | def to_ms(td: 'SecondsTimedelta') -> int: 69 | pass 70 | 71 | 72 | def to_ms(td: Optional['SecondsTimedelta']) -> Optional[int]: 73 | if td is None: 74 | return td 75 | elif isinstance(td, timedelta): 76 | td = td.total_seconds() 77 | return as_int(td * 1000) 78 | 79 | 80 | @overload 81 | def to_seconds(td: None) -> None: 82 | pass 83 | 84 | 85 | @overload 86 | def to_seconds(td: 'SecondsTimedelta') -> float: 87 | pass 88 | 89 | 90 | def to_seconds(td: Optional['SecondsTimedelta']) -> Optional[float]: 91 | if td is None: 92 | return td 93 | elif isinstance(td, timedelta): 94 | return td.total_seconds() 95 | return td 96 | 97 | 98 | async def poll(step: float = 0.5) -> AsyncGenerator[float, None]: 99 | loop = asyncio.get_event_loop() 100 | start = loop.time() 101 | while True: 102 | before = loop.time() 103 | yield before - start 104 | after = loop.time() 105 | wait = max([0, step - after + before]) 106 | await asyncio.sleep(wait) 107 | 108 | 109 | DEFAULT_CURTAIL = 80 110 | 111 | 112 | def truncate(s: str, length: int = DEFAULT_CURTAIL) -> str: 113 | """ 114 | Truncate a string and add an ellipsis (three dots) to the end if it was too long 115 | 116 | :param s: string to possibly truncate 117 | :param length: length to truncate the string to 118 | """ 119 | if len(s) > length: 120 | s = s[: length - 1] + '…' 121 | return s 122 | 123 | 124 | def args_to_string(args: Sequence[Any], kwargs: Dict[str, Any]) -> str: 125 | arguments = '' 126 | if args: 127 | arguments = ', '.join(map(repr, args)) 128 | if kwargs: 129 | if arguments: 130 | arguments += ', ' 131 | arguments += ', '.join(f'{k}={v!r}' for k, v in sorted(kwargs.items())) 132 | return truncate(arguments) 133 | 134 | 135 | def import_string(dotted_path: str) -> Any: 136 | """ 137 | Taken from pydantic.utils. 138 | """ 139 | from importlib import import_module 140 | 141 | try: 142 | module_path, class_name = dotted_path.strip(' ').rsplit('.', 1) 143 | except ValueError as e: 144 | raise ImportError(f'"{dotted_path}" doesn\'t look like a module path') from e 145 | 146 | module = import_module(module_path) 147 | try: 148 | return getattr(module, class_name) 149 | except AttributeError as e: 150 | raise ImportError(f'Module "{module_path}" does not define a "{class_name}" attribute') from e 151 | -------------------------------------------------------------------------------- /arq/version.py: -------------------------------------------------------------------------------- 1 | # Version here is used for the package version via the `[tool.hatch.version]` section of `pyproject.toml`. 2 | VERSION = '0.26.3' 3 | -------------------------------------------------------------------------------- /arq/worker.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import contextlib 3 | import inspect 4 | import logging 5 | import signal 6 | from dataclasses import dataclass 7 | from datetime import datetime, timedelta, timezone 8 | from functools import partial 9 | from signal import Signals 10 | from time import time 11 | from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Sequence, Set, Tuple, Union, cast 12 | 13 | from redis.exceptions import ResponseError, WatchError 14 | 15 | from arq.cron import CronJob 16 | from arq.jobs import Deserializer, JobResult, SerializationError, Serializer, deserialize_job_raw, serialize_result 17 | 18 | from .connections import ArqRedis, RedisSettings, create_pool, log_redis_info 19 | from .constants import ( 20 | abort_job_max_age, 21 | abort_jobs_ss, 22 | default_queue_name, 23 | expires_extra_ms, 24 | health_check_key_suffix, 25 | in_progress_key_prefix, 26 | job_key_prefix, 27 | keep_cronjob_progress, 28 | result_key_prefix, 29 | retry_key_prefix, 30 | ) 31 | from .utils import ( 32 | args_to_string, 33 | import_string, 34 | ms_to_datetime, 35 | poll, 36 | timestamp_ms, 37 | to_ms, 38 | to_seconds, 39 | to_unix_ms, 40 | truncate, 41 | ) 42 | 43 | if TYPE_CHECKING: 44 | from .typing import SecondsTimedelta, StartupShutdown, WorkerCoroutine, WorkerSettingsType 45 | 46 | logger = logging.getLogger('arq.worker') 47 | no_result = object() 48 | 49 | 50 | @dataclass 51 | class Function: 52 | name: str 53 | coroutine: 'WorkerCoroutine' 54 | timeout_s: Optional[float] 55 | keep_result_s: Optional[float] 56 | keep_result_forever: Optional[bool] 57 | max_tries: Optional[int] 58 | 59 | 60 | def func( 61 | coroutine: Union[str, Function, 'WorkerCoroutine'], 62 | *, 63 | name: Optional[str] = None, 64 | keep_result: Optional['SecondsTimedelta'] = None, 65 | timeout: Optional['SecondsTimedelta'] = None, 66 | keep_result_forever: Optional[bool] = None, 67 | max_tries: Optional[int] = None, 68 | ) -> Function: 69 | """ 70 | Wrapper for a job function which lets you configure more settings. 71 | 72 | :param coroutine: coroutine function to call, can be a string to import 73 | :param name: name for function, if None, ``coroutine.__qualname__`` is used 74 | :param keep_result: duration to keep the result for, if 0 the result is not kept 75 | :param keep_result_forever: whether to keep results forever, if None use Worker default, wins over ``keep_result`` 76 | :param timeout: maximum time the job should take 77 | :param max_tries: maximum number of tries allowed for the function, use 1 to prevent retrying 78 | """ 79 | if isinstance(coroutine, Function): 80 | return coroutine 81 | 82 | if isinstance(coroutine, str): 83 | name = name or coroutine 84 | coroutine_: 'WorkerCoroutine' = import_string(coroutine) 85 | else: 86 | coroutine_ = coroutine 87 | 88 | if not asyncio.iscoroutinefunction(coroutine_): 89 | raise RuntimeError(f'{coroutine_} is not a coroutine function') 90 | timeout = to_seconds(timeout) 91 | keep_result = to_seconds(keep_result) 92 | 93 | return Function(name or coroutine_.__qualname__, coroutine_, timeout, keep_result, keep_result_forever, max_tries) 94 | 95 | 96 | class Retry(RuntimeError): 97 | """ 98 | Special exception to retry the job (if ``max_retries`` hasn't been reached). 99 | 100 | :param defer: duration to wait before rerunning the job 101 | """ 102 | 103 | def __init__(self, defer: Optional['SecondsTimedelta'] = None): 104 | self.defer_score: Optional[int] = to_ms(defer) 105 | 106 | def __repr__(self) -> str: 107 | return f'' 108 | 109 | def __str__(self) -> str: 110 | return repr(self) 111 | 112 | 113 | class JobExecutionFailed(RuntimeError): 114 | def __eq__(self, other: Any) -> bool: 115 | if isinstance(other, JobExecutionFailed): 116 | return self.args == other.args 117 | return False 118 | 119 | 120 | class FailedJobs(RuntimeError): 121 | def __init__(self, count: int, job_results: List[JobResult]): 122 | self.count = count 123 | self.job_results = job_results 124 | 125 | def __str__(self) -> str: 126 | if self.count == 1 and self.job_results: 127 | exc = self.job_results[0].result 128 | return f'1 job failed {exc!r}' 129 | else: 130 | return f'{self.count} jobs failed:\n' + '\n'.join(repr(r.result) for r in self.job_results) 131 | 132 | def __repr__(self) -> str: 133 | return f'<{str(self)}>' 134 | 135 | 136 | class RetryJob(RuntimeError): 137 | pass 138 | 139 | 140 | class Worker: 141 | """ 142 | Main class for running jobs. 143 | 144 | :param functions: list of functions to register, can either be raw coroutine functions or the 145 | result of :func:`arq.worker.func`. 146 | :param queue_name: queue name to get jobs from 147 | :param cron_jobs: list of cron jobs to run, use :func:`arq.cron.cron` to create them 148 | :param redis_settings: settings for creating a redis connection 149 | :param redis_pool: existing redis pool, generally None 150 | :param burst: whether to stop the worker once all jobs have been run 151 | :param on_startup: coroutine function to run at startup 152 | :param on_shutdown: coroutine function to run at shutdown 153 | :param on_job_start: coroutine function to run on job start 154 | :param on_job_end: coroutine function to run on job end 155 | :param after_job_end: coroutine function to run after job has ended and results have been recorded 156 | :param handle_signals: default true, register signal handlers, 157 | set to false when running inside other async framework 158 | :param job_completion_wait: time to wait before cancelling tasks after a signal. 159 | Useful together with ``terminationGracePeriodSeconds`` in kubernetes, 160 | when you want to make the pod complete jobs before shutting down. 161 | The worker will not pick new tasks while waiting for shut down. 162 | :param max_jobs: maximum number of jobs to run at a time 163 | :param job_timeout: default job timeout (max run time) 164 | :param keep_result: default duration to keep job results for 165 | :param keep_result_forever: whether to keep results forever 166 | :param poll_delay: duration between polling the queue for new jobs 167 | :param queue_read_limit: the maximum number of jobs to pull from the queue each time it's polled. By default it 168 | equals ``max_jobs`` * 5, or 100; whichever is higher. 169 | :param max_tries: default maximum number of times to retry a job 170 | :param health_check_interval: how often to set the health check key 171 | :param health_check_key: redis key under which health check is set 172 | :param ctx: dictionary to hold extra user defined state 173 | :param retry_jobs: whether to retry jobs on Retry or CancelledError or not 174 | :param allow_abort_jobs: whether to abort jobs on a call to :func:`arq.jobs.Job.abort` 175 | :param max_burst_jobs: the maximum number of jobs to process in burst mode (disabled with negative values) 176 | :param job_serializer: a function that serializes Python objects to bytes, defaults to pickle.dumps 177 | :param job_deserializer: a function that deserializes bytes into Python objects, defaults to pickle.loads 178 | :param expires_extra_ms: the default length of time from when a job is expected to start 179 | after which the job expires, defaults to 1 day in ms. 180 | :param timezone: timezone used for evaluation of cron schedules, 181 | defaults to system timezone 182 | :param log_results: when set to true (default) results for successful jobs 183 | will be logged 184 | """ 185 | 186 | def __init__( 187 | self, 188 | functions: Sequence[Union[Function, 'WorkerCoroutine']] = (), 189 | *, 190 | queue_name: Optional[str] = default_queue_name, 191 | cron_jobs: Optional[Sequence[CronJob]] = None, 192 | redis_settings: Optional[RedisSettings] = None, 193 | redis_pool: Optional[ArqRedis] = None, 194 | burst: bool = False, 195 | on_startup: Optional['StartupShutdown'] = None, 196 | on_shutdown: Optional['StartupShutdown'] = None, 197 | on_job_start: Optional['StartupShutdown'] = None, 198 | on_job_end: Optional['StartupShutdown'] = None, 199 | after_job_end: Optional['StartupShutdown'] = None, 200 | handle_signals: bool = True, 201 | job_completion_wait: int = 0, 202 | max_jobs: int = 10, 203 | job_timeout: 'SecondsTimedelta' = 300, 204 | keep_result: 'SecondsTimedelta' = 3600, 205 | keep_result_forever: bool = False, 206 | poll_delay: 'SecondsTimedelta' = 0.5, 207 | queue_read_limit: Optional[int] = None, 208 | max_tries: int = 5, 209 | health_check_interval: 'SecondsTimedelta' = 3600, 210 | health_check_key: Optional[str] = None, 211 | ctx: Optional[Dict[Any, Any]] = None, 212 | retry_jobs: bool = True, 213 | allow_abort_jobs: bool = False, 214 | max_burst_jobs: int = -1, 215 | job_serializer: Optional[Serializer] = None, 216 | job_deserializer: Optional[Deserializer] = None, 217 | expires_extra_ms: int = expires_extra_ms, 218 | timezone: Optional[timezone] = None, 219 | log_results: bool = True, 220 | ): 221 | self.functions: Dict[str, Union[Function, CronJob]] = {f.name: f for f in map(func, functions)} 222 | if queue_name is None: 223 | if redis_pool is not None: 224 | queue_name = redis_pool.default_queue_name 225 | else: 226 | raise ValueError('If queue_name is absent, redis_pool must be present.') 227 | self.queue_name = queue_name 228 | self.cron_jobs: List[CronJob] = [] 229 | if cron_jobs is not None: 230 | if not all(isinstance(cj, CronJob) for cj in cron_jobs): 231 | raise RuntimeError('cron_jobs, must be instances of CronJob') 232 | self.cron_jobs = list(cron_jobs) 233 | self.functions.update({cj.name: cj for cj in self.cron_jobs}) 234 | if len(self.functions) == 0: 235 | raise RuntimeError('at least one function or cron_job must be registered') 236 | self.burst = burst 237 | self.on_startup = on_startup 238 | self.on_shutdown = on_shutdown 239 | self.on_job_start = on_job_start 240 | self.on_job_end = on_job_end 241 | self.after_job_end = after_job_end 242 | 243 | self.max_jobs = max_jobs 244 | self.sem = asyncio.BoundedSemaphore(max_jobs + 1) 245 | self.job_counter: int = 0 246 | 247 | self.job_timeout_s = to_seconds(job_timeout) 248 | self.keep_result_s = to_seconds(keep_result) 249 | self.keep_result_forever = keep_result_forever 250 | self.poll_delay_s = to_seconds(poll_delay) 251 | self.queue_read_limit = queue_read_limit or max(max_jobs * 5, 100) 252 | self._queue_read_offset = 0 253 | self.max_tries = max_tries 254 | self.health_check_interval = to_seconds(health_check_interval) 255 | if health_check_key is None: 256 | self.health_check_key = self.queue_name + health_check_key_suffix 257 | else: 258 | self.health_check_key = health_check_key 259 | self._pool = redis_pool 260 | if self._pool is None: 261 | self.redis_settings: Optional[RedisSettings] = redis_settings or RedisSettings() 262 | else: 263 | self.redis_settings = None 264 | # self.tasks holds references to run_job coroutines currently running 265 | self.tasks: Dict[str, asyncio.Task[Any]] = {} 266 | # self.job_tasks holds references the actual jobs running 267 | self.job_tasks: Dict[str, asyncio.Task[Any]] = {} 268 | self.main_task: Optional[asyncio.Task[None]] = None 269 | self.loop = asyncio.get_event_loop() 270 | self.ctx = ctx or {} 271 | max_timeout = max(f.timeout_s or self.job_timeout_s for f in self.functions.values()) 272 | self.in_progress_timeout_s = (max_timeout or 0) + 10 273 | self.jobs_complete = 0 274 | self.jobs_retried = 0 275 | self.jobs_failed = 0 276 | self._last_health_check: float = 0 277 | self._last_health_check_log: Optional[str] = None 278 | self._handle_signals = handle_signals 279 | self._job_completion_wait = job_completion_wait 280 | if self._handle_signals: 281 | if self._job_completion_wait: 282 | self._add_signal_handler(signal.SIGINT, self.handle_sig_wait_for_completion) 283 | self._add_signal_handler(signal.SIGTERM, self.handle_sig_wait_for_completion) 284 | else: 285 | self._add_signal_handler(signal.SIGINT, self.handle_sig) 286 | self._add_signal_handler(signal.SIGTERM, self.handle_sig) 287 | self.on_stop: Optional[Callable[[Signals], None]] = None 288 | # whether or not to retry jobs on Retry and CancelledError 289 | self.retry_jobs = retry_jobs 290 | self.allow_abort_jobs = allow_abort_jobs 291 | self.allow_pick_jobs: bool = True 292 | self.aborting_tasks: Set[str] = set() 293 | self.max_burst_jobs = max_burst_jobs 294 | self.job_serializer = job_serializer 295 | self.job_deserializer = job_deserializer 296 | self.expires_extra_ms = expires_extra_ms 297 | self.log_results = log_results 298 | 299 | # default to system timezone 300 | self.timezone = datetime.now().astimezone().tzinfo if timezone is None else timezone 301 | 302 | def run(self) -> None: 303 | """ 304 | Sync function to run the worker, finally closes worker connections. 305 | """ 306 | self.main_task = self.loop.create_task(self.main()) 307 | try: 308 | self.loop.run_until_complete(self.main_task) 309 | except asyncio.CancelledError: # pragma: no cover 310 | # happens on shutdown, fine 311 | pass 312 | finally: 313 | self.loop.run_until_complete(self.close()) 314 | 315 | async def async_run(self) -> None: 316 | """ 317 | Asynchronously run the worker, does not close connections. Useful when testing. 318 | """ 319 | self.main_task = self.loop.create_task(self.main()) 320 | await self.main_task 321 | 322 | async def run_check(self, retry_jobs: Optional[bool] = None, max_burst_jobs: Optional[int] = None) -> int: 323 | """ 324 | Run :func:`arq.worker.Worker.async_run`, check for failed jobs and raise :class:`arq.worker.FailedJobs` 325 | if any jobs have failed. 326 | 327 | :return: number of completed jobs 328 | """ 329 | if retry_jobs is not None: 330 | self.retry_jobs = retry_jobs 331 | if max_burst_jobs is not None: 332 | self.max_burst_jobs = max_burst_jobs 333 | await self.async_run() 334 | if self.jobs_failed: 335 | failed_job_results = [r for r in await self.pool.all_job_results() if not r.success] 336 | raise FailedJobs(self.jobs_failed, failed_job_results) 337 | else: 338 | return self.jobs_complete 339 | 340 | @property 341 | def pool(self) -> ArqRedis: 342 | return cast(ArqRedis, self._pool) 343 | 344 | async def main(self) -> None: 345 | if self._pool is None: 346 | self._pool = await create_pool( 347 | self.redis_settings, 348 | job_deserializer=self.job_deserializer, 349 | job_serializer=self.job_serializer, 350 | default_queue_name=self.queue_name, 351 | expires_extra_ms=self.expires_extra_ms, 352 | ) 353 | 354 | logger.info('Starting worker for %d functions: %s', len(self.functions), ', '.join(self.functions)) 355 | await log_redis_info(self.pool, logger.info) 356 | self.ctx['redis'] = self.pool 357 | if self.on_startup: 358 | await self.on_startup(self.ctx) 359 | 360 | async for _ in poll(self.poll_delay_s): 361 | await self._poll_iteration() 362 | 363 | if self.burst: 364 | if 0 <= self.max_burst_jobs <= self._jobs_started(): 365 | await asyncio.gather(*self.tasks.values()) 366 | return None 367 | queued_jobs = await self.pool.zcard(self.queue_name) 368 | if queued_jobs == 0: 369 | await asyncio.gather(*self.tasks.values()) 370 | return None 371 | 372 | async def _poll_iteration(self) -> None: 373 | """ 374 | Get ids of pending jobs from the main queue sorted-set data structure and start those jobs, remove 375 | any finished tasks from self.tasks. 376 | """ 377 | count = self.queue_read_limit 378 | if self.burst and self.max_burst_jobs >= 0: 379 | burst_jobs_remaining = self.max_burst_jobs - self._jobs_started() 380 | if burst_jobs_remaining < 1: 381 | return 382 | count = min(burst_jobs_remaining, count) 383 | if self.allow_pick_jobs: 384 | if self.job_counter < self.max_jobs: 385 | now = timestamp_ms() 386 | job_ids = await self.pool.zrangebyscore( 387 | self.queue_name, min=float('-inf'), start=self._queue_read_offset, num=count, max=now 388 | ) 389 | 390 | await self.start_jobs(job_ids) 391 | 392 | if self.allow_abort_jobs: 393 | await self._cancel_aborted_jobs() 394 | 395 | for job_id, t in list(self.tasks.items()): 396 | if t.done(): 397 | del self.tasks[job_id] 398 | # required to make sure errors in run_job get propagated 399 | t.result() 400 | 401 | await self.heart_beat() 402 | 403 | async def _cancel_aborted_jobs(self) -> None: 404 | """ 405 | Go through job_ids in the abort_jobs_ss sorted set and cancel those tasks. 406 | """ 407 | async with self.pool.pipeline(transaction=True) as pipe: 408 | pipe.zrange(abort_jobs_ss, start=0, end=-1) 409 | pipe.zremrangebyscore(abort_jobs_ss, min=timestamp_ms() + abort_job_max_age, max=float('inf')) 410 | abort_job_ids, _ = await pipe.execute() 411 | 412 | aborted: Set[str] = set() 413 | for job_id_bytes in abort_job_ids: 414 | job_id = job_id_bytes.decode() 415 | try: 416 | task = self.job_tasks[job_id] 417 | except KeyError: 418 | pass 419 | else: 420 | aborted.add(job_id) 421 | task.cancel() 422 | 423 | if aborted: 424 | self.aborting_tasks.update(aborted) 425 | await self.pool.zrem(abort_jobs_ss, *aborted) 426 | 427 | def _release_sem_dec_counter_on_complete(self) -> None: 428 | self.job_counter = self.job_counter - 1 429 | self.sem.release() 430 | 431 | async def start_jobs(self, job_ids: List[bytes]) -> None: 432 | """ 433 | For each job id, get the job definition, check it's not running and start it in a task 434 | """ 435 | for job_id_b in job_ids: 436 | await self.sem.acquire() 437 | 438 | if self.job_counter >= self.max_jobs: 439 | self.sem.release() 440 | return None 441 | 442 | self.job_counter = self.job_counter + 1 443 | 444 | job_id = job_id_b.decode() 445 | in_progress_key = in_progress_key_prefix + job_id 446 | async with self.pool.pipeline(transaction=True) as pipe: 447 | await pipe.watch(in_progress_key) 448 | ongoing_exists = await pipe.exists(in_progress_key) 449 | score = await pipe.zscore(self.queue_name, job_id) 450 | if ongoing_exists or not score or score > timestamp_ms(): 451 | # job already started elsewhere, or already finished and removed from queue 452 | # if score > ts_now, 453 | # it means probably the job was re-enqueued with a delay in another worker 454 | self.job_counter = self.job_counter - 1 455 | self.sem.release() 456 | logger.debug('job %s already running elsewhere', job_id) 457 | continue 458 | 459 | pipe.multi() 460 | pipe.psetex(in_progress_key, int(self.in_progress_timeout_s * 1000), b'1') 461 | try: 462 | await pipe.execute() 463 | except (ResponseError, WatchError): 464 | # job already started elsewhere since we got 'existing' 465 | self.job_counter = self.job_counter - 1 466 | self.sem.release() 467 | logger.debug('multi-exec error, job %s already started elsewhere', job_id) 468 | else: 469 | t = self.loop.create_task(self.run_job(job_id, int(score))) 470 | t.add_done_callback(lambda _: self._release_sem_dec_counter_on_complete()) 471 | self.tasks[job_id] = t 472 | 473 | async def run_job(self, job_id: str, score: int) -> None: # noqa: C901 474 | start_ms = timestamp_ms() 475 | async with self.pool.pipeline(transaction=True) as pipe: 476 | pipe.get(job_key_prefix + job_id) 477 | pipe.incr(retry_key_prefix + job_id) 478 | pipe.expire(retry_key_prefix + job_id, 88400) 479 | if self.allow_abort_jobs: 480 | pipe.zrem(abort_jobs_ss, job_id) 481 | v, job_try, _, abort_job = await pipe.execute() 482 | else: 483 | v, job_try, _ = await pipe.execute() 484 | abort_job = False 485 | 486 | function_name, enqueue_time_ms = '', 0 487 | args: Tuple[Any, ...] = () 488 | kwargs: Dict[Any, Any] = {} 489 | 490 | async def job_failed(exc: BaseException) -> None: 491 | self.jobs_failed += 1 492 | result_data_ = serialize_result( 493 | function=function_name, 494 | args=args, 495 | kwargs=kwargs, 496 | job_try=job_try, 497 | enqueue_time_ms=enqueue_time_ms, 498 | success=False, 499 | result=exc, 500 | start_ms=start_ms, 501 | finished_ms=timestamp_ms(), 502 | ref=f'{job_id}:{function_name}', 503 | serializer=self.job_serializer, 504 | queue_name=self.queue_name, 505 | job_id=job_id, 506 | ) 507 | await asyncio.shield(self.finish_failed_job(job_id, result_data_)) 508 | 509 | if not v: 510 | logger.warning('job %s expired', job_id) 511 | return await job_failed(JobExecutionFailed('job expired')) 512 | 513 | try: 514 | function_name, args, kwargs, enqueue_job_try, enqueue_time_ms = deserialize_job_raw( 515 | v, deserializer=self.job_deserializer 516 | ) 517 | except SerializationError as e: 518 | logger.exception('deserializing job %s failed', job_id) 519 | return await job_failed(e) 520 | 521 | if abort_job: 522 | t = (timestamp_ms() - enqueue_time_ms) / 1000 523 | logger.info('%6.2fs ⊘ %s:%s aborted before start', t, job_id, function_name) 524 | return await job_failed(asyncio.CancelledError()) 525 | 526 | try: 527 | function: Union[Function, CronJob] = self.functions[function_name] 528 | except KeyError: 529 | logger.warning('job %s, function %r not found', job_id, function_name) 530 | return await job_failed(JobExecutionFailed(f'function {function_name!r} not found')) 531 | 532 | if hasattr(function, 'next_run'): 533 | # cron_job 534 | ref = function_name 535 | keep_in_progress: Optional[float] = keep_cronjob_progress 536 | else: 537 | ref = f'{job_id}:{function_name}' 538 | keep_in_progress = None 539 | 540 | if enqueue_job_try and enqueue_job_try > job_try: 541 | job_try = enqueue_job_try 542 | await self.pool.setex(retry_key_prefix + job_id, 88400, str(job_try)) 543 | 544 | max_tries = self.max_tries if function.max_tries is None else function.max_tries 545 | if job_try > max_tries: 546 | t = (timestamp_ms() - enqueue_time_ms) / 1000 547 | logger.warning('%6.2fs ! %s max retries %d exceeded', t, ref, max_tries) 548 | self.jobs_failed += 1 549 | result_data = serialize_result( 550 | function_name, 551 | args, 552 | kwargs, 553 | job_try, 554 | enqueue_time_ms, 555 | False, 556 | JobExecutionFailed(f'max {max_tries} retries exceeded'), 557 | start_ms, 558 | timestamp_ms(), 559 | ref, 560 | self.queue_name, 561 | job_id=job_id, 562 | serializer=self.job_serializer, 563 | ) 564 | return await asyncio.shield(self.finish_failed_job(job_id, result_data)) 565 | 566 | result = no_result 567 | exc_extra = None 568 | finish = False 569 | timeout_s = self.job_timeout_s if function.timeout_s is None else function.timeout_s 570 | incr_score: Optional[int] = None 571 | job_ctx = { 572 | 'job_id': job_id, 573 | 'job_try': job_try, 574 | 'enqueue_time': ms_to_datetime(enqueue_time_ms), 575 | 'score': score, 576 | } 577 | ctx = {**self.ctx, **job_ctx} 578 | 579 | if self.on_job_start: 580 | await self.on_job_start(ctx) 581 | 582 | start_ms = timestamp_ms() 583 | success = False 584 | try: 585 | s = args_to_string(args, kwargs) 586 | extra = f' try={job_try}' if job_try > 1 else '' 587 | if (start_ms - score) > 1200: 588 | extra += f' delayed={(start_ms - score) / 1000:0.2f}s' 589 | logger.info('%6.2fs → %s(%s)%s', (start_ms - enqueue_time_ms) / 1000, ref, s, extra) 590 | self.job_tasks[job_id] = task = self.loop.create_task(function.coroutine(ctx, *args, **kwargs)) 591 | 592 | # run repr(result) and extra inside try/except as they can raise exceptions 593 | try: 594 | result = await asyncio.wait_for(task, timeout_s) 595 | except (Exception, asyncio.CancelledError) as e: 596 | exc_extra = getattr(e, 'extra', None) 597 | if callable(exc_extra): 598 | exc_extra = exc_extra() 599 | raise 600 | else: 601 | result_str = '' if result is None or not self.log_results else truncate(repr(result)) 602 | finally: 603 | del self.job_tasks[job_id] 604 | 605 | except (Exception, asyncio.CancelledError) as e: 606 | finished_ms = timestamp_ms() 607 | t = (finished_ms - start_ms) / 1000 608 | if self.retry_jobs and isinstance(e, Retry): 609 | incr_score = e.defer_score 610 | logger.info('%6.2fs ↻ %s retrying job in %0.2fs', t, ref, (e.defer_score or 0) / 1000) 611 | if e.defer_score: 612 | incr_score = e.defer_score + (timestamp_ms() - score) 613 | self.jobs_retried += 1 614 | elif job_id in self.aborting_tasks and isinstance(e, asyncio.CancelledError): 615 | logger.info('%6.2fs ⊘ %s aborted', t, ref) 616 | result = e 617 | finish = True 618 | self.aborting_tasks.remove(job_id) 619 | self.jobs_failed += 1 620 | elif self.retry_jobs and isinstance(e, (asyncio.CancelledError, RetryJob)): 621 | logger.info('%6.2fs ↻ %s cancelled, will be run again', t, ref) 622 | self.jobs_retried += 1 623 | else: 624 | logger.exception( 625 | '%6.2fs ! %s failed, %s: %s', t, ref, e.__class__.__name__, e, extra={'extra': exc_extra} 626 | ) 627 | result = e 628 | finish = True 629 | self.jobs_failed += 1 630 | else: 631 | success = True 632 | finished_ms = timestamp_ms() 633 | logger.info('%6.2fs ← %s ● %s', (finished_ms - start_ms) / 1000, ref, result_str) 634 | finish = True 635 | self.jobs_complete += 1 636 | 637 | keep_result_forever = ( 638 | self.keep_result_forever if function.keep_result_forever is None else function.keep_result_forever 639 | ) 640 | result_timeout_s = self.keep_result_s if function.keep_result_s is None else function.keep_result_s 641 | result_data = None 642 | if result is not no_result and (keep_result_forever or result_timeout_s > 0): 643 | result_data = serialize_result( 644 | function_name, 645 | args, 646 | kwargs, 647 | job_try, 648 | enqueue_time_ms, 649 | success, 650 | result, 651 | start_ms, 652 | finished_ms, 653 | ref, 654 | self.queue_name, 655 | job_id=job_id, 656 | serializer=self.job_serializer, 657 | ) 658 | 659 | if self.on_job_end: 660 | await self.on_job_end(ctx) 661 | 662 | await asyncio.shield( 663 | self.finish_job( 664 | job_id, 665 | finish, 666 | result_data, 667 | result_timeout_s, 668 | keep_result_forever, 669 | incr_score, 670 | keep_in_progress, 671 | ) 672 | ) 673 | 674 | if self.after_job_end: 675 | await self.after_job_end(ctx) 676 | 677 | async def finish_job( 678 | self, 679 | job_id: str, 680 | finish: bool, 681 | result_data: Optional[bytes], 682 | result_timeout_s: Optional[float], 683 | keep_result_forever: bool, 684 | incr_score: Optional[int], 685 | keep_in_progress: Optional[float], 686 | ) -> None: 687 | async with self.pool.pipeline(transaction=True) as tr: 688 | delete_keys = [] 689 | in_progress_key = in_progress_key_prefix + job_id 690 | if keep_in_progress is None: 691 | delete_keys += [in_progress_key] 692 | else: 693 | tr.pexpire(in_progress_key, to_ms(keep_in_progress)) 694 | 695 | if finish: 696 | if result_data: 697 | expire = None if keep_result_forever else result_timeout_s 698 | tr.set(result_key_prefix + job_id, result_data, px=to_ms(expire)) 699 | delete_keys += [retry_key_prefix + job_id, job_key_prefix + job_id] 700 | tr.zrem(abort_jobs_ss, job_id) 701 | tr.zrem(self.queue_name, job_id) 702 | elif incr_score: 703 | tr.zincrby(self.queue_name, incr_score, job_id) 704 | if delete_keys: 705 | tr.delete(*delete_keys) 706 | await tr.execute() 707 | 708 | async def finish_failed_job(self, job_id: str, result_data: Optional[bytes]) -> None: 709 | async with self.pool.pipeline(transaction=True) as tr: 710 | tr.delete( 711 | retry_key_prefix + job_id, 712 | in_progress_key_prefix + job_id, 713 | job_key_prefix + job_id, 714 | ) 715 | tr.zrem(abort_jobs_ss, job_id) 716 | tr.zrem(self.queue_name, job_id) 717 | # result_data would only be None if serializing the result fails 718 | keep_result = self.keep_result_forever or self.keep_result_s > 0 719 | if result_data is not None and keep_result: # pragma: no branch 720 | expire = 0 if self.keep_result_forever else self.keep_result_s 721 | tr.set(result_key_prefix + job_id, result_data, px=to_ms(expire)) 722 | await tr.execute() 723 | 724 | async def heart_beat(self) -> None: 725 | now = datetime.now(tz=self.timezone) 726 | await self.record_health() 727 | 728 | cron_window_size = max(self.poll_delay_s, 0.5) # Clamp the cron delay to 0.5 729 | await self.run_cron(now, cron_window_size) 730 | 731 | async def run_cron(self, n: datetime, delay: float, num_windows: int = 2) -> None: 732 | job_futures = set() 733 | 734 | cron_delay = timedelta(seconds=delay * num_windows) 735 | 736 | this_hb_cutoff = n + cron_delay 737 | 738 | for cron_job in self.cron_jobs: 739 | if cron_job.next_run is None: 740 | if cron_job.run_at_startup: 741 | cron_job.next_run = n 742 | else: 743 | cron_job.calculate_next(n) 744 | # This isn't getting run this iteration in any case. 745 | continue 746 | 747 | # We queue up the cron if the next execution time is in the next 748 | # delay * num_windows (by default 0.5 * 2 = 1 second). 749 | if cron_job.next_run < this_hb_cutoff: 750 | if cron_job.job_id: 751 | job_id: Optional[str] = cron_job.job_id 752 | else: 753 | job_id = f'{cron_job.name}:{to_unix_ms(cron_job.next_run)}' if cron_job.unique else None 754 | job_futures.add( 755 | self.pool.enqueue_job( 756 | cron_job.name, 757 | _job_id=job_id, 758 | _queue_name=self.queue_name, 759 | _defer_until=( 760 | cron_job.next_run if cron_job.next_run > datetime.now(tz=self.timezone) else None 761 | ), 762 | ) 763 | ) 764 | cron_job.calculate_next(cron_job.next_run) 765 | 766 | job_futures and await asyncio.gather(*job_futures) 767 | 768 | async def record_health(self) -> None: 769 | now_ts = time() 770 | if (now_ts - self._last_health_check) < self.health_check_interval: 771 | return 772 | self._last_health_check = now_ts 773 | pending_tasks = sum(not t.done() for t in self.tasks.values()) 774 | queued = await self.pool.zcard(self.queue_name) 775 | info = ( 776 | f'{datetime.now():%b-%d %H:%M:%S} j_complete={self.jobs_complete} j_failed={self.jobs_failed} ' 777 | f'j_retried={self.jobs_retried} j_ongoing={pending_tasks} queued={queued}' 778 | ) 779 | await self.pool.psetex( # type: ignore[no-untyped-call] 780 | self.health_check_key, int((self.health_check_interval + 1) * 1000), info.encode() 781 | ) 782 | log_suffix = info[info.index('j_complete=') :] 783 | if self._last_health_check_log and log_suffix != self._last_health_check_log: 784 | logger.info('recording health: %s', info) 785 | self._last_health_check_log = log_suffix 786 | elif not self._last_health_check_log: 787 | self._last_health_check_log = log_suffix 788 | 789 | def _add_signal_handler(self, signum: Signals, handler: Callable[[Signals], None]) -> None: 790 | try: 791 | self.loop.add_signal_handler(signum, partial(handler, signum)) 792 | except NotImplementedError: # pragma: no cover 793 | logger.debug('Windows does not support adding a signal handler to an eventloop') 794 | 795 | def _jobs_started(self) -> int: 796 | return self.jobs_complete + self.jobs_retried + self.jobs_failed + len(self.tasks) 797 | 798 | async def _sleep_until_tasks_complete(self) -> None: 799 | """ 800 | Sleeps until all tasks are done. Used together with asyncio.wait_for() 801 | """ 802 | while len(self.tasks): 803 | await asyncio.sleep(0.1) 804 | 805 | async def _wait_for_tasks_to_complete(self, signum: Signals) -> None: 806 | """ 807 | Wait for tasks to complete, until `wait_for_job_completion_on_signal_second` has been reached. 808 | """ 809 | with contextlib.suppress(asyncio.TimeoutError): 810 | await asyncio.wait_for( 811 | self._sleep_until_tasks_complete(), 812 | self._job_completion_wait, 813 | ) 814 | logger.info( 815 | 'shutdown on %s, wait complete ◆ %d jobs complete ◆ %d failed ◆ %d retries ◆ %d ongoing to cancel', 816 | signum.name, 817 | self.jobs_complete, 818 | self.jobs_failed, 819 | self.jobs_retried, 820 | sum(not t.done() for t in self.tasks.values()), 821 | ) 822 | for t in self.tasks.values(): 823 | if not t.done(): 824 | t.cancel() 825 | self.main_task and self.main_task.cancel() 826 | self.on_stop and self.on_stop(signum) 827 | 828 | def handle_sig_wait_for_completion(self, signum: Signals) -> None: 829 | """ 830 | Alternative signal handler that allow tasks to complete within a given time before shutting down the worker. 831 | Time can be configured using `wait_for_job_completion_on_signal_second`. 832 | The worker will stop picking jobs when signal has been received. 833 | """ 834 | sig = Signals(signum) 835 | logger.info('Setting allow_pick_jobs to `False`') 836 | self.allow_pick_jobs = False 837 | logger.info( 838 | 'shutdown on %s ◆ %d jobs complete ◆ %d failed ◆ %d retries ◆ %d to be completed', 839 | sig.name, 840 | self.jobs_complete, 841 | self.jobs_failed, 842 | self.jobs_retried, 843 | len(self.tasks), 844 | ) 845 | self.loop.create_task(self._wait_for_tasks_to_complete(signum=sig)) 846 | 847 | def handle_sig(self, signum: Signals) -> None: 848 | sig = Signals(signum) 849 | logger.info( 850 | 'shutdown on %s ◆ %d jobs complete ◆ %d failed ◆ %d retries ◆ %d ongoing to cancel', 851 | sig.name, 852 | self.jobs_complete, 853 | self.jobs_failed, 854 | self.jobs_retried, 855 | len(self.tasks), 856 | ) 857 | for t in self.tasks.values(): 858 | if not t.done(): 859 | t.cancel() 860 | self.main_task and self.main_task.cancel() 861 | self.on_stop and self.on_stop(sig) 862 | 863 | async def close(self) -> None: 864 | if not self._handle_signals: 865 | self.handle_sig(signal.SIGUSR1) 866 | if not self._pool: 867 | return 868 | await asyncio.gather(*self.tasks.values()) 869 | await self.pool.delete(self.health_check_key) 870 | if self.on_shutdown: 871 | await self.on_shutdown(self.ctx) 872 | await self.pool.close(close_connection_pool=True) 873 | self._pool = None 874 | 875 | def __repr__(self) -> str: 876 | return ( 877 | f'' 879 | ) 880 | 881 | 882 | def get_kwargs(settings_cls: 'WorkerSettingsType') -> Dict[str, NameError]: 883 | worker_args = set(inspect.signature(Worker).parameters.keys()) 884 | d = settings_cls if isinstance(settings_cls, dict) else settings_cls.__dict__ 885 | return {k: v for k, v in d.items() if k in worker_args} 886 | 887 | 888 | def create_worker(settings_cls: 'WorkerSettingsType', **kwargs: Any) -> Worker: 889 | return Worker(**{**get_kwargs(settings_cls), **kwargs}) 890 | 891 | 892 | def run_worker(settings_cls: 'WorkerSettingsType', **kwargs: Any) -> Worker: 893 | worker = create_worker(settings_cls, **kwargs) 894 | worker.run() 895 | return worker 896 | 897 | 898 | async def async_check_health( 899 | redis_settings: Optional[RedisSettings], health_check_key: Optional[str] = None, queue_name: Optional[str] = None 900 | ) -> int: 901 | redis_settings = redis_settings or RedisSettings() 902 | redis: ArqRedis = await create_pool(redis_settings) 903 | queue_name = queue_name or default_queue_name 904 | health_check_key = health_check_key or (queue_name + health_check_key_suffix) 905 | 906 | data = await redis.get(health_check_key) 907 | if not data: 908 | logger.warning('Health check failed: no health check sentinel value found') 909 | r = 1 910 | else: 911 | logger.info('Health check successful: %s', data) 912 | r = 0 913 | await redis.close(close_connection_pool=True) 914 | return r 915 | 916 | 917 | def check_health(settings_cls: 'WorkerSettingsType') -> int: 918 | """ 919 | Run a health check on the worker and return the appropriate exit code. 920 | :return: 0 if successful, 1 if not 921 | """ 922 | cls_kwargs = get_kwargs(settings_cls) 923 | redis_settings = cast(Optional[RedisSettings], cls_kwargs.get('redis_settings')) 924 | health_check_key = cast(Optional[str], cls_kwargs.get('health_check_key')) 925 | queue_name = cast(Optional[str], cls_kwargs.get('queue_name')) 926 | return asyncio.run(async_check_health(redis_settings, health_check_key, queue_name)) 927 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | .DEFAULT_GOAL := html 2 | 3 | # You can set these variables from the command line. 4 | SPHINXOPTS = -W 5 | SPHINXBUILD = sphinx-build 6 | PAPER = 7 | BUILDDIR = _build 8 | STATICDIR = _static 9 | 10 | # Internal variables. 11 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(SPHINXOPTS) . 12 | 13 | .PHONY: help 14 | help: 15 | @echo "Please use \`make ' where is one of" 16 | @echo " html to make standalone HTML files" 17 | @echo " linkcheck to check all external links for integrity" 18 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 19 | @echo " coverage to run coverage check of the documentation (if enabled)" 20 | 21 | .PHONY: clean 22 | clean: 23 | rm -rf $(BUILDDIR) 24 | 25 | .PHONY: html 26 | html: 27 | mkdir -p $(STATICDIR) 28 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 29 | @echo 30 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 31 | 32 | .PHONY: linkcheck 33 | linkcheck: 34 | mkdir -p $(STATICDIR) 35 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 36 | @echo 37 | @echo "Link check complete; look for any errors in the above output " \ 38 | "or in $(BUILDDIR)/linkcheck/output.txt." 39 | 40 | .PHONY: doctest 41 | doctest: 42 | mkdir -p $(STATICDIR) 43 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 44 | @echo "Testing of doctests in the sources finished, look at the " \ 45 | "results in $(BUILDDIR)/doctest/output.txt." 46 | 47 | .PHONY: coverage 48 | coverage: 49 | mkdir -p $(STATICDIR) 50 | $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage 51 | @echo "Testing of coverage in the sources finished, look at the " \ 52 | "results in $(BUILDDIR)/coverage/python.txt." 53 | -------------------------------------------------------------------------------- /docs/_templates/layout.html: -------------------------------------------------------------------------------- 1 | {% extends '!layout.html' %} 2 | 3 | {% block footer %} 4 | 12 | {% endblock %} 13 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # arq documentation build configuration file, created by 5 | # sphinx-quickstart on Sat Aug 13 12:25:33 2016. 6 | # 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | # 20 | import os 21 | import sys 22 | sys.path.append(os.path.abspath('../arq')) 23 | 24 | # -- General configuration ------------------------------------------------ 25 | 26 | # If your documentation needs a minimal Sphinx version, state it here. 27 | # 28 | # needs_sphinx = '1.0' 29 | 30 | # Add any Sphinx extension module names here, as strings. They can be 31 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 32 | # ones. 33 | extensions = [ 34 | 'sphinx.ext.autodoc', 35 | 'sphinx.ext.doctest', 36 | 'sphinx.ext.todo', 37 | 'sphinx.ext.coverage', 38 | 'sphinx.ext.ifconfig', 39 | 'sphinx.ext.viewcode', 40 | 'sphinx.ext.githubpages', 41 | ] 42 | 43 | autoclass_content = 'both' 44 | autodoc_member_order = 'bysource' 45 | 46 | # Add any paths that contain templates here, relative to this directory. 47 | templates_path = ['_templates'] 48 | 49 | # The suffix(es) of source filenames. 50 | # You can specify multiple suffix as a list of string: 51 | # 52 | # source_suffix = ['.rst', '.md'] 53 | source_suffix = '.rst' 54 | 55 | # The encoding of source files. 56 | # 57 | # source_encoding = 'utf-8-sig' 58 | 59 | # The master toctree document. 60 | master_doc = 'index' 61 | 62 | # General information about the project. 63 | project = 'arq' 64 | copyright = '2016, Samuel Colvin' 65 | author = 'Samuel Colvin' 66 | 67 | # The version info for the project you're documenting, acts as replacement for 68 | # |version| and |release|, also used in various other places throughout the 69 | # built documents. 70 | 71 | from arq.version import VERSION 72 | # The short X.Y version. Could change this if you're updating docs for a previous version. 73 | version = f'v{VERSION}' 74 | # The full version, including alpha/beta/rc tags. 75 | release = f'v{VERSION}' 76 | 77 | # The language for content autogenerated by Sphinx. Refer to documentation 78 | # for a list of supported languages. 79 | # 80 | # This is also used if you do content translation via gettext catalogs. 81 | # Usually you set "language" from the command line for these cases. 82 | language = 'en' 83 | 84 | # There are two options for replacing |today|: either, you set today to some 85 | # non-false value, then it is used: 86 | # 87 | # today = '' 88 | # 89 | # Else, today_fmt is used as the format for a strftime call. 90 | # 91 | # today_fmt = '%B %d, %Y' 92 | 93 | # List of patterns, relative to source directory, that match files and 94 | # directories to ignore when looking for source files. 95 | # This patterns also effect to html_static_path and html_extra_path 96 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 97 | 98 | # The reST default role (used for this markup: `text`) to use for all 99 | # documents. 100 | # 101 | # default_role = None 102 | 103 | # If true, '()' will be appended to :func: etc. cross-reference text. 104 | # 105 | # add_function_parentheses = True 106 | 107 | # If true, the current module name will be prepended to all description 108 | # unit titles (such as .. function::). 109 | # 110 | # add_module_names = True 111 | 112 | # If true, sectionauthor and moduleauthor directives will be shown in the 113 | # output. They are ignored by default. 114 | # 115 | # show_authors = False 116 | 117 | # The name of the Pygments (syntax highlighting) style to use. 118 | pygments_style = 'sphinx' 119 | 120 | # A list of ignored prefixes for module index sorting. 121 | # modindex_common_prefix = [] 122 | 123 | # If true, keep warnings as "system message" paragraphs in the built documents. 124 | # keep_warnings = False 125 | 126 | # If true, `todo` and `todoList` produce output, else they produce nothing. 127 | todo_include_todos = True 128 | 129 | 130 | # -- Options for HTML output ---------------------------------------------- 131 | 132 | # The theme to use for HTML and HTML Help pages. See the documentation for 133 | # a list of builtin themes. 134 | # 135 | html_theme = 'alabaster' 136 | 137 | # Theme options are theme-specific and customize the look and feel of a theme 138 | # further. For a list of options available for each theme, see the 139 | # documentation. 140 | # 141 | html_theme_options = { 142 | 'github_user': 'samuelcolvin', 143 | 'github_repo': 'arq', 144 | 'travis_button': True, 145 | 'codecov_button': True, 146 | 'page_width': '1200px', 147 | 'github_banner': True, 148 | 'github_type': 'star', 149 | } 150 | 151 | # Add any paths that contain custom themes here, relative to this directory. 152 | # html_theme_path = [] 153 | 154 | # The name for this set of Sphinx documents. 155 | # " v documentation" by default. 156 | # 157 | # html_title = 'arq v5' 158 | 159 | # A shorter title for the navigation bar. Default is the same as html_title. 160 | # 161 | # html_short_title = None 162 | 163 | # The name of an image file (relative to this directory) to place at the top 164 | # of the sidebar. 165 | # 166 | # html_logo = None 167 | 168 | # The name of an image file (relative to this directory) to use as a favicon of 169 | # the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 170 | # pixels large. 171 | # 172 | # html_favicon = None 173 | 174 | # Add any paths that contain custom static files (such as style sheets) here, 175 | # relative to this directory. They are copied after the builtin static files, 176 | # so a file named "default.css" will overwrite the builtin "default.css". 177 | html_static_path = ['_static'] 178 | 179 | # Add any extra paths that contain custom files (such as robots.txt or 180 | # .htaccess) here, relative to this directory. These files are copied 181 | # directly to the root of the documentation. 182 | # 183 | # html_extra_path = [] 184 | 185 | # If not None, a 'Last updated on:' timestamp is inserted at every page 186 | # bottom, using the given strftime format. 187 | # The empty string is equivalent to '%b %d, %Y'. 188 | # 189 | # html_last_updated_fmt = None 190 | 191 | # If true, SmartyPants will be used to convert quotes and dashes to 192 | # typographically correct entities. 193 | # 194 | # html_use_smartypants = True 195 | 196 | # Custom sidebar templates, maps document names to template names. 197 | # 198 | html_sidebars = { 199 | '**': [ 200 | 'about.html', 201 | 'localtoc.html', 202 | 'searchbox.html', 203 | ] 204 | } 205 | 206 | # Additional templates that should be rendered to pages, maps page names to 207 | # template names. 208 | # 209 | # html_additional_pages = {} 210 | 211 | # If false, no module index is generated. 212 | # 213 | # html_domain_indices = True 214 | 215 | # If false, no index is generated. 216 | # 217 | # html_use_index = True 218 | 219 | # If true, the index is split into individual pages for each letter. 220 | # 221 | # html_split_index = False 222 | 223 | # If true, links to the reST sources are added to the pages. 224 | # 225 | # html_show_sourcelink = True 226 | 227 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 228 | # 229 | # html_show_sphinx = True 230 | 231 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 232 | # 233 | # html_show_copyright = True 234 | 235 | # If true, an OpenSearch description file will be output, and all pages will 236 | # contain a tag referring to it. The value of this option must be the 237 | # base URL from which the finished HTML is served. 238 | # 239 | # html_use_opensearch = '' 240 | 241 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 242 | # html_file_suffix = None 243 | 244 | # Language to be used for generating the HTML full-text search index. 245 | # Sphinx supports the following languages: 246 | # 'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja' 247 | # 'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr', 'zh' 248 | # 249 | # html_search_language = 'en' 250 | 251 | # A dictionary with options for the search language support, empty by default. 252 | # 'ja' uses this config value. 253 | # 'zh' user can custom change `jieba` dictionary path. 254 | # 255 | # html_search_options = {'type': 'default'} 256 | 257 | # The name of a javascript file (relative to the configuration directory) that 258 | # implements a search results scorer. If empty, the default will be used. 259 | # 260 | # html_search_scorer = 'scorer.js' 261 | 262 | # Output file base name for HTML help builder. 263 | htmlhelp_basename = 'arqdoc' 264 | 265 | # -- Options for LaTeX output --------------------------------------------- 266 | 267 | latex_elements = { 268 | # The paper size ('letterpaper' or 'a4paper'). 269 | # 270 | # 'papersize': 'letterpaper', 271 | 272 | # The font size ('10pt', '11pt' or '12pt'). 273 | # 274 | # 'pointsize': '10pt', 275 | 276 | # Additional stuff for the LaTeX preamble. 277 | # 278 | # 'preamble': '', 279 | 280 | # Latex figure (float) alignment 281 | # 282 | # 'figure_align': 'htbp', 283 | } 284 | 285 | # Grouping the document tree into LaTeX files. List of tuples 286 | # (source start file, target name, title, 287 | # author, documentclass [howto, manual, or own class]). 288 | latex_documents = [ 289 | (master_doc, 'arq.tex', 'arq Documentation', 290 | 'Samuel Colvin', 'manual'), 291 | ] 292 | 293 | # The name of an image file (relative to this directory) to place at the top of 294 | # the title page. 295 | # 296 | # latex_logo = None 297 | 298 | # For "manual" documents, if this is true, then toplevel headings are parts, 299 | # not chapters. 300 | # 301 | # latex_use_parts = False 302 | 303 | # If true, show page references after internal links. 304 | # 305 | # latex_show_pagerefs = False 306 | 307 | # If true, show URL addresses after external links. 308 | # 309 | # latex_show_urls = False 310 | 311 | # Documents to append as an appendix to all manuals. 312 | # 313 | # latex_appendices = [] 314 | 315 | # It false, will not define \strong, \code, itleref, \crossref ... but only 316 | # \sphinxstrong, ..., \sphinxtitleref, ... To help avoid clash with user added 317 | # packages. 318 | # 319 | # latex_keep_old_macro_names = True 320 | 321 | # If false, no module index is generated. 322 | # 323 | # latex_domain_indices = True 324 | 325 | 326 | # -- Options for manual page output --------------------------------------- 327 | 328 | # One entry per manual page. List of tuples 329 | # (source start file, name, description, authors, manual section). 330 | man_pages = [ 331 | (master_doc, 'arq', 'arq Documentation', 332 | [author], 1) 333 | ] 334 | 335 | # If true, show URL addresses after external links. 336 | # 337 | # man_show_urls = False 338 | 339 | 340 | # -- Options for Texinfo output ------------------------------------------- 341 | 342 | # Grouping the document tree into Texinfo files. List of tuples 343 | # (source start file, target name, title, author, 344 | # dir menu entry, description, category) 345 | texinfo_documents = [ 346 | (master_doc, 'arq', 'arq Documentation', 347 | author, 'arq', 'One line description of project.', 348 | 'Miscellaneous'), 349 | ] 350 | 351 | # Documents to append as an appendix to all manuals. 352 | # 353 | # texinfo_appendices = [] 354 | 355 | # If false, no module index is generated. 356 | # 357 | # texinfo_domain_indices = True 358 | 359 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 360 | # 361 | # texinfo_show_urls = 'footnote' 362 | 363 | # If true, do not generate a @detailmenu in the "Top" node's menu. 364 | # 365 | # texinfo_no_detailmenu = False 366 | suppress_warnings = ['image.nonlocal_uri'] 367 | -------------------------------------------------------------------------------- /docs/examples/cron.py: -------------------------------------------------------------------------------- 1 | from arq import cron 2 | 3 | async def run_regularly(ctx): 4 | print('run foo job at 9.12am, 12.12pm and 6.12pm') 5 | 6 | class WorkerSettings: 7 | cron_jobs = [ 8 | cron(run_regularly, hour={9, 12, 18}, minute=12) 9 | ] 10 | -------------------------------------------------------------------------------- /docs/examples/custom_serialization_msgpack.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | import msgpack # installable with "pip install msgpack" 4 | 5 | from arq import create_pool 6 | from arq.connections import RedisSettings 7 | 8 | 9 | async def the_task(ctx): 10 | return 42 11 | 12 | 13 | async def main(): 14 | redis = await create_pool( 15 | RedisSettings(), 16 | job_serializer=msgpack.packb, 17 | job_deserializer=lambda b: msgpack.unpackb(b, raw=False), 18 | ) 19 | await redis.enqueue_job('the_task') 20 | 21 | 22 | class WorkerSettings: 23 | functions = [the_task] 24 | job_serializer = msgpack.packb 25 | # refer to MsgPack's documentation as to why raw=False is required 26 | job_deserializer = lambda b: msgpack.unpackb(b, raw=False) 27 | 28 | 29 | if __name__ == '__main__': 30 | asyncio.run(main()) 31 | -------------------------------------------------------------------------------- /docs/examples/deferred.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from datetime import datetime, timedelta 3 | 4 | from arq import create_pool 5 | from arq.connections import RedisSettings 6 | 7 | async def the_task(ctx): 8 | print('this is the tasks, delay since enqueueing:', datetime.now() - ctx['enqueue_time']) 9 | 10 | async def main(): 11 | redis = await create_pool(RedisSettings()) 12 | 13 | # deferred by 10 seconds 14 | await redis.enqueue_job('the_task', _defer_by=10) 15 | 16 | # deferred by 1 minute 17 | await redis.enqueue_job('the_task', _defer_by=timedelta(minutes=1)) 18 | 19 | # deferred until jan 28th 2032, you'll be waiting a long time for this... 20 | await redis.enqueue_job('the_task', _defer_until=datetime(2032, 1, 28)) 21 | 22 | class WorkerSettings: 23 | functions = [the_task] 24 | 25 | if __name__ == '__main__': 26 | asyncio.run(main()) 27 | -------------------------------------------------------------------------------- /docs/examples/job_abort.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from arq import create_pool 3 | from arq.connections import RedisSettings 4 | 5 | 6 | async def do_stuff(ctx): 7 | print('doing stuff...') 8 | await asyncio.sleep(10) 9 | return 'stuff done' 10 | 11 | 12 | async def main(): 13 | redis = await create_pool(RedisSettings()) 14 | job = await redis.enqueue_job('do_stuff') 15 | await asyncio.sleep(1) 16 | await job.abort() 17 | 18 | 19 | class WorkerSettings: 20 | functions = [do_stuff] 21 | allow_abort_jobs = True 22 | 23 | 24 | if __name__ == '__main__': 25 | asyncio.run(main()) 26 | -------------------------------------------------------------------------------- /docs/examples/job_ids.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | from arq import create_pool 4 | from arq.connections import RedisSettings 5 | from arq.jobs import Job 6 | 7 | 8 | async def the_task(ctx): 9 | print('running the task with id', ctx['job_id']) 10 | 11 | async def main(): 12 | redis = await create_pool(RedisSettings()) 13 | 14 | # no id, random id will be generated 15 | job1 = await redis.enqueue_job('the_task') 16 | print(job1) 17 | """ 18 | > 19 | """ 20 | 21 | # random id again, again the job will be enqueued and a job will be returned 22 | job2 = await redis.enqueue_job('the_task') 23 | print(job2) 24 | """ 25 | > 26 | """ 27 | 28 | # custom job id, job will be enqueued 29 | job3 = await redis.enqueue_job('the_task', _job_id='foobar') 30 | print(job3) 31 | """ 32 | > 33 | """ 34 | 35 | # same custom job id, job will not be enqueued and enqueue_job will return None 36 | job4 = await redis.enqueue_job('the_task', _job_id='foobar') 37 | print(job4) 38 | """ 39 | > None 40 | """ 41 | 42 | # you can retrieve jobs by using arq.jobs.Job 43 | await redis.enqueue_job('the_task', _job_id='my_job') 44 | job5 = Job(job_id='my_job', redis=redis) 45 | print(job5) 46 | """ 47 | 48 | """ 49 | 50 | class WorkerSettings: 51 | functions = [the_task] 52 | 53 | if __name__ == '__main__': 54 | asyncio.run(main()) 55 | -------------------------------------------------------------------------------- /docs/examples/job_results.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | from arq import create_pool 4 | from arq.connections import RedisSettings 5 | # requires `pip install devtools`, used for pretty printing of job info 6 | from devtools import debug 7 | 8 | async def the_task(ctx): 9 | print('running the task') 10 | return 42 11 | 12 | async def main(): 13 | redis = await create_pool(RedisSettings()) 14 | 15 | job = await redis.enqueue_job('the_task') 16 | 17 | # get the job's id 18 | print(job.job_id) 19 | """ 20 | > 68362958a244465b9be909db4b7b5ab4 (or whatever) 21 | """ 22 | 23 | # get information about the job, will include results if the job has finished, but 24 | # doesn't await the job's result 25 | debug(await job.info()) 26 | """ 27 | > docs/examples/job_results.py:23 main 28 | JobDef( 29 | function='the_task', 30 | args=(), 31 | kwargs={}, 32 | job_try=None, 33 | enqueue_time=datetime.datetime(2019, 4, 23, 13, 58, 56, 781000), 34 | score=1556027936781 35 | ) (JobDef) 36 | """ 37 | 38 | # get the Job's status 39 | print(await job.status()) 40 | """ 41 | > JobStatus.queued 42 | """ 43 | 44 | # poll redis for the job result, if the job raised an exception, 45 | # it will be raised here 46 | # (You'll need the worker running at the same time to get a result here) 47 | print(await job.result(timeout=5)) 48 | """ 49 | > 42 50 | """ 51 | 52 | class WorkerSettings: 53 | functions = [the_task] 54 | 55 | if __name__ == '__main__': 56 | asyncio.run(main()) 57 | -------------------------------------------------------------------------------- /docs/examples/main_demo.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from httpx import AsyncClient 3 | from arq import create_pool 4 | from arq.connections import RedisSettings 5 | 6 | # Here you can configure the Redis connection. 7 | # The default is to connect to localhost:6379, no password. 8 | REDIS_SETTINGS = RedisSettings() 9 | 10 | async def download_content(ctx, url): 11 | session: AsyncClient = ctx['session'] 12 | response = await session.get(url) 13 | print(f'{url}: {response.text:.80}...') 14 | return len(response.text) 15 | 16 | async def startup(ctx): 17 | ctx['session'] = AsyncClient() 18 | 19 | async def shutdown(ctx): 20 | await ctx['session'].aclose() 21 | 22 | async def main(): 23 | redis = await create_pool(REDIS_SETTINGS) 24 | for url in ('https://facebook.com', 'https://microsoft.com', 'https://github.com'): 25 | await redis.enqueue_job('download_content', url) 26 | 27 | # WorkerSettings defines the settings to use when creating the work, 28 | # It's used by the arq CLI. 29 | # redis_settings might be omitted here if using the default settings 30 | # For a list of all available settings, see https://arq-docs.helpmanual.io/#arq.worker.Worker 31 | class WorkerSettings: 32 | functions = [download_content] 33 | on_startup = startup 34 | on_shutdown = shutdown 35 | redis_settings = REDIS_SETTINGS 36 | 37 | if __name__ == '__main__': 38 | asyncio.run(main()) 39 | -------------------------------------------------------------------------------- /docs/examples/retry.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from httpx import AsyncClient 3 | from arq import create_pool, Retry 4 | from arq.connections import RedisSettings 5 | 6 | async def download_content(ctx, url): 7 | session: AsyncClient = ctx['session'] 8 | response = await session.get(url) 9 | if response.status_code != 200: 10 | # retry the job with increasing back-off 11 | # delays will be 5s, 10s, 15s, 20s 12 | # after max_tries (default 5) the job will permanently fail 13 | raise Retry(defer=ctx['job_try'] * 5) 14 | return len(response.text) 15 | 16 | async def startup(ctx): 17 | ctx['session'] = AsyncClient() 18 | 19 | async def shutdown(ctx): 20 | await ctx['session'].aclose() 21 | 22 | async def main(): 23 | redis = await create_pool(RedisSettings()) 24 | await redis.enqueue_job('download_content', 'https://httpbin.org/status/503') 25 | 26 | class WorkerSettings: 27 | functions = [download_content] 28 | on_startup = startup 29 | on_shutdown = shutdown 30 | 31 | if __name__ == '__main__': 32 | asyncio.run(main()) 33 | -------------------------------------------------------------------------------- /docs/examples/slow_job.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | from arq import create_pool 4 | from arq.connections import RedisSettings 5 | 6 | async def the_task(ctx): 7 | await asyncio.sleep(5) 8 | 9 | async def main(): 10 | redis = await create_pool(RedisSettings()) 11 | await redis.enqueue_job('the_task') 12 | 13 | class WorkerSettings: 14 | functions = [the_task] 15 | 16 | if __name__ == '__main__': 17 | asyncio.run(main()) 18 | -------------------------------------------------------------------------------- /docs/examples/slow_job_output.txt: -------------------------------------------------------------------------------- 1 | ➤ arq slow_job.WorkerSettings 2 | 12:42:38: Starting worker for 1 functions: the_task 3 | 12:42:38: redis_version=4.0.9 mem_usage=904.50K clients_connected=4 db_keys=3 4 | 12:42:38: 10.23s → c3dd4acc171541b9ac10b1d791750cde:the_task() delayed=10.23s 5 | ^C12:42:40: shutdown on SIGINT ◆ 0 jobs complete ◆ 0 failed ◆ 0 retries ◆ 1 ongoing to cancel 6 | 12:42:40: 1.16s ↻ c3dd4acc171541b9ac10b1d791750cde:the_task cancelled, will be run again 7 | 8 | 9 | ➤ arq slow_job.WorkerSettings 10 | 12:42:50: Starting worker for 1 functions: the_task 11 | 12:42:50: redis_version=4.0.9 mem_usage=904.61K clients_connected=4 db_keys=4 12 | 12:42:50: 21.78s → c3dd4acc171541b9ac10b1d791750cde:the_task() try=2 delayed=21.78s 13 | 12:42:55: 5.00s ← c3dd4acc171541b9ac10b1d791750cde:the_task ● 14 | ^C12:42:57: shutdown on SIGINT ◆ 1 jobs complete ◆ 0 failed ◆ 0 retries ◆ 0 ongoing to cancel 15 | -------------------------------------------------------------------------------- /docs/examples/sync_job.py: -------------------------------------------------------------------------------- 1 | import time 2 | import functools 3 | import asyncio 4 | from concurrent import futures 5 | 6 | def sync_task(t): 7 | return time.sleep(t) 8 | 9 | async def the_task(ctx, t): 10 | blocking = functools.partial(sync_task, t) 11 | loop = asyncio.get_running_loop() 12 | return await loop.run_in_executor(ctx['pool'], blocking) 13 | 14 | async def startup(ctx): 15 | ctx['pool'] = futures.ProcessPoolExecutor() 16 | 17 | class WorkerSettings: 18 | functions = [the_task] 19 | on_startup = startup 20 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | arq 2 | === 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | 7 | |pypi| |license| 8 | 9 | Current Version: |version| 10 | 11 | Job queues and RPC in python with asyncio and redis. 12 | 13 | *arq* was conceived as a simple, modern and performant successor to rq_. 14 | 15 | .. warning:: 16 | 17 | In ``v0.16`` *arq* was **COMPLETELY REWRITTEN** to use an entirely different approach to registering workers, 18 | enqueueing jobs and processing jobs. You will need to either keep using ``v0.15`` or entirely rewrite your *arq* 19 | integration to use ``v0.16``. 20 | 21 | See `here <./old/index.html>`_ for old docs. 22 | 23 | Why use *arq*? 24 | 25 | **non-blocking** 26 | *arq* is built using python 3's asyncio_ allowing 27 | non-blocking job enqueuing and execution. Multiple jobs (potentially hundreds) can be run simultaneously 28 | using a pool of *asyncio* ``Tasks``. 29 | 30 | **powerful-features** 31 | Deferred execution, easy retrying of jobs, and pessimistic execution (:ref:`see below `) 32 | means *arq* is great for critical jobs that **must** be completed. 33 | 34 | **fast** 35 | Asyncio and no forking make *arq* around 7x faster than 36 | *rq* for short jobs with no io. With io that might increase to around 40x 37 | faster. (TODO) 38 | 39 | **elegant** 40 | I'm a long time contributor to and user of `rq`_, *arq* is designed to be simpler, clearer and more powerful. 41 | 42 | **small** 43 | and easy to reason with - currently *arq* is only about 700 lines, that won't change significantly. 44 | 45 | Install 46 | ------- 47 | 48 | Just:: 49 | 50 | pip install arq 51 | 52 | Redesigned to be less elegant? 53 | ------------------------------ 54 | 55 | The approach used in *arq* ``v0.16`` of enqueueing jobs by name rather than "just calling a function" and knowing it 56 | will be called on the worker (as used in *arq* ``<= v0.15``, rq, celery et al.) might seem less elegant, 57 | but it's for good reason. 58 | 59 | This approach means your frontend (calling the worker) doesn't need access to the worker code, 60 | meaning better code separation and possibly smaller images etc. 61 | 62 | .. _usage: 63 | 64 | Usage 65 | ----- 66 | 67 | .. warning:: 68 | 69 | **Jobs may be called more than once!** 70 | 71 | *arq* v0.16 has what I'm calling "pessimistic execution": jobs aren't removed from the queue until they've either 72 | succeeded or failed. If the worker shuts down, the job will be cancelled immediately and will remain in the queue 73 | to be run again when the worker starts up again (or run by another worker which is still running). 74 | 75 | (This differs from other similar libraries like *arq* ``<= v0.15``, rq, celery et al. where jobs generally don't get 76 | rerun when a worker shuts down. This in turn requires complex logic to try and let jobs finish before 77 | shutting down (I wrote the ``HerokuWorker`` for rq), however this never really works unless either: all jobs take 78 | less than 6 seconds or your worker never shuts down when a job is running (impossible).) 79 | 80 | All *arq* jobs should therefore be designed to cope with being called repeatedly if they're cancelled, 81 | eg. use database transactions, idempotency keys or redis to mark when an API request or similar has succeeded 82 | to avoid making it twice. 83 | 84 | **In summary:** sometimes *exactly once* can be hard or impossible, *arq* favours multiple times over zero times. 85 | 86 | Simple Usage 87 | ............ 88 | 89 | .. literalinclude:: examples/main_demo.py 90 | 91 | (This script is complete, it should run "as is" both to enqueue jobs and run them) 92 | 93 | To enqueue the jobs, simply run the script:: 94 | 95 | python demo.py 96 | 97 | To execute the jobs, either after running ``demo.py`` or before/during:: 98 | 99 | arq demo.WorkerSettings 100 | 101 | Append ``--burst`` to stop the worker once all jobs have finished. See :class:`arq.worker.Worker` for more available 102 | properties of ``WorkerSettings``. 103 | 104 | You can also watch for changes and reload the worker when the source changes:: 105 | 106 | arq demo.WorkerSettings --watch path/to/src 107 | 108 | This requires watchfiles_ to be installed (``pip install watchfiles``). 109 | 110 | For details on the *arq* CLI:: 111 | 112 | arq --help 113 | 114 | Startup & Shutdown coroutines 115 | ............................. 116 | 117 | The ``on_startup`` and ``on_shutdown`` coroutines are provided as a convenient way to run logic as the worker 118 | starts and finishes, see :class:`arq.worker.Worker`. 119 | 120 | For example, in the above example ``session`` is created once when the work starts up and is then used in subsequent 121 | jobs. 122 | 123 | Deferring Jobs 124 | .............. 125 | 126 | By default, when a job is enqueued it will run as soon as possible (provided a worker is running). However 127 | you can schedule jobs to run in the future, either by a given duration (``_defer_by``) or 128 | at a particular time ``_defer_until``, see :func:`arq.connections.ArqRedis.enqueue_job`. 129 | 130 | .. literalinclude:: examples/deferred.py 131 | 132 | Job Uniqueness 133 | .............. 134 | 135 | Sometimes you want a job to only be run once at a time (eg. a backup) or once for a given parameter (eg. generating 136 | invoices for a particular company). 137 | 138 | *arq* supports this via custom job ids, see :func:`arq.connections.ArqRedis.enqueue_job`. It guarantees 139 | that a job with a particular ID cannot be enqueued again until its execution has finished and its result has cleared. To control when a finished job's result clears, you can use the `keep_result` setting on your worker, see :func:`arq.worker.func`. 140 | 141 | .. literalinclude:: examples/job_ids.py 142 | 143 | The check of ``job_id`` uniqueness in the queue is performed using a redis transaction so you can be certain jobs 144 | with the same id won't be enqueued twice (or overwritten) even if they're enqueued at exactly the same time. 145 | 146 | Job Results 147 | ........... 148 | 149 | You can access job information, status and job results using the :class:`arq.jobs.Job` instance returned from 150 | :func:`arq.connections.ArqRedis.enqueue_job`. 151 | 152 | .. literalinclude:: examples/job_results.py 153 | 154 | Retrying jobs and cancellation 155 | .............................. 156 | 157 | As described above, when an arq worker shuts down, any ongoing jobs are cancelled immediately 158 | (via vanilla ``task.cancel()``, so a ``CancelledError`` will be raised). You can see this by running a slow job 159 | (eg. add ``await asyncio.sleep(5)``) and hitting ``Ctrl+C`` once it's started. 160 | 161 | You'll get something like. 162 | 163 | .. literalinclude:: examples/slow_job_output.txt 164 | 165 | You can also retry jobs by raising the :class:`arq.worker.Retry` exception from within a job, 166 | optionally with a duration to defer rerunning the jobs by: 167 | 168 | .. literalinclude:: examples/retry.py 169 | 170 | To abort a job, call :func:`arq.job.Job.abort`. (Note for the :func:`arq.job.Job.abort` method to 171 | have any effect, you need to set ``allow_abort_jobs`` to ``True`` on the worker, this is for performance reason. 172 | ``allow_abort_jobs=True`` may become the default in future) 173 | 174 | :func:`arq.job.Job.abort` will abort a job if it's already running or prevent it being run if it's currently 175 | in the queue. 176 | 177 | .. literalinclude:: examples/job_abort.py 178 | 179 | Health checks 180 | ............. 181 | 182 | *arq* will automatically record some info about its current state in redis every ``health_check_interval`` seconds. 183 | That key/value will expire after ``health_check_interval + 1`` seconds so you can be sure if the variable exists *arq* 184 | is alive and kicking (technically you can be sure it was alive and kicking ``health_check_interval`` seconds ago). 185 | 186 | You can run a health check with the CLI (assuming you're using the first example above):: 187 | 188 | arq --check demo.WorkerSettings 189 | 190 | The command will output the value of the health check if found; 191 | then exit ``0`` if the key was found and ``1`` if it was not. 192 | 193 | A health check value takes the following form:: 194 | 195 | Mar-01 17:41:22 j_complete=0 j_failed=0 j_retried=0 j_ongoing=0 queued=0 196 | 197 | Where the items have the following meaning: 198 | 199 | * ``j_complete`` the number of jobs completed 200 | * ``j_failed`` the number of jobs which have failed eg. raised an exception 201 | * ``j_ongoing`` the number of jobs currently being performed 202 | * ``j_retried`` the number of jobs retries run 203 | 204 | Cron Jobs 205 | ......... 206 | 207 | Functions can be scheduled to be run periodically at specific times. See :func:`arq.cron.cron`. 208 | 209 | .. literalinclude:: examples/cron.py 210 | 211 | Usage roughly shadows `cron `_ except ``None`` is equivalent on ``*`` in crontab. 212 | As per the example sets can be used to run at multiple of the given unit. 213 | 214 | Note that ``second`` defaults to ``0`` so you don't in inadvertently run jobs every second and ``microsecond`` 215 | defaults to ``123456`` so you don't inadvertently run jobs every microsecond and so *arq* avoids enqueuing jobs 216 | at the top of a second when the world is generally slightly busier. 217 | 218 | Synchronous Jobs 219 | ................ 220 | 221 | Functions that can block the loop for extended periods should be run in an executor like 222 | ``concurrent.futures.ThreadPoolExecutor`` or ``concurrent.futures.ProcessPoolExecutor`` using 223 | ``loop.run_in_executor`` as shown below. 224 | 225 | .. literalinclude:: examples/sync_job.py 226 | 227 | Custom job serializers 228 | ...................... 229 | 230 | By default, *arq* will use the built-in ``pickle`` module to serialize and deserialize jobs. If you wish to 231 | use an alternative serialization methods, you can do so by specifying them when creating the connection pool 232 | and the worker settings. A serializer function takes a Python object and returns a binary representation 233 | encoded in a ``bytes`` object. A deserializer function, on the other hand, creates Python objects out of 234 | a ``bytes`` sequence. 235 | 236 | .. warning:: 237 | It is essential that the serialization functions used by :func:`arq.connections.create_pool` and 238 | :class:`arq.worker.Worker` are the same, otherwise jobs created by the former cannot be executed by the 239 | latter. This also applies when you update your serialization functions: you need to ensure that your new 240 | functions are backward compatible with the old jobs, or that there are no jobs with the older serialization 241 | scheme in the queue. 242 | 243 | Here is an example with `MsgPack `_, an efficient binary serialization format that 244 | may enable significant memory improvements over pickle: 245 | 246 | .. literalinclude:: examples/custom_serialization_msgpack.py 247 | 248 | 249 | Reference 250 | --------- 251 | 252 | .. automodule:: arq.connections 253 | :members: 254 | 255 | .. automodule:: arq.worker 256 | :members: func, Retry, Worker 257 | 258 | .. automodule:: arq.cron 259 | :members: cron 260 | 261 | .. automodule:: arq.jobs 262 | :members: JobStatus, Job 263 | 264 | .. include:: ../HISTORY.rst 265 | 266 | .. |pypi| image:: https://img.shields.io/pypi/v/arq.svg 267 | :target: https://pypi.python.org/pypi/arq 268 | .. |license| image:: https://img.shields.io/pypi/l/arq.svg 269 | :target: https://github.com/samuelcolvin/arq 270 | .. _asyncio: https://docs.python.org/3/library/asyncio.html 271 | .. _watchfiles: https://pypi.org/project/watchfiles/ 272 | .. _rq: http://python-rq.org/ 273 | -------------------------------------------------------------------------------- /docs/old-docs.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/python-arq/arq/7a911f37992bad4556a27756f2ff027dd2afe655/docs/old-docs.zip -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ['hatchling'] 3 | build-backend = 'hatchling.build' 4 | 5 | [tool.hatch.version] 6 | path = 'arq/version.py' 7 | 8 | [project] 9 | name = 'arq' 10 | description = 'Job queues in python with asyncio and redis' 11 | authors = [{name = 'Samuel Colvin', email = 's@muelcolvin.com'}] 12 | license = { text = 'MIT' } 13 | readme = 'README.md' 14 | classifiers = [ 15 | 'Development Status :: 5 - Production/Stable', 16 | 'Environment :: Console', 17 | 'Framework :: AsyncIO', 18 | 'Intended Audience :: Developers', 19 | 'Intended Audience :: Information Technology', 20 | 'Intended Audience :: System Administrators', 21 | 'License :: OSI Approved :: MIT License', 22 | 'Operating System :: Unix', 23 | 'Operating System :: POSIX :: Linux', 24 | 'Programming Language :: Python', 25 | 'Programming Language :: Python :: 3', 26 | 'Programming Language :: Python :: 3 :: Only', 27 | 'Programming Language :: Python :: 3.8', 28 | 'Programming Language :: Python :: 3.9', 29 | 'Programming Language :: Python :: 3.10', 30 | 'Programming Language :: Python :: 3.11', 31 | 'Programming Language :: Python :: 3.12', 32 | 'Topic :: Internet', 33 | 'Topic :: Software Development :: Libraries :: Python Modules', 34 | 'Topic :: System :: Clustering', 35 | 'Topic :: System :: Distributed Computing', 36 | 'Topic :: System :: Monitoring', 37 | 'Topic :: System :: Systems Administration', 38 | ] 39 | requires-python = '>=3.8' 40 | dependencies = [ 41 | 'redis[hiredis]>=4.2.0,<6', 42 | 'click>=8.0', 43 | ] 44 | optional-dependencies = {watch = ['watchfiles>=0.16'] } 45 | dynamic = ['version'] 46 | 47 | [project.scripts] 48 | arq = 'arq.cli:cli' 49 | 50 | [project.urls] 51 | Homepage = 'https://github.com/samuelcolvin/arq' 52 | Documentation = 'https://arq-docs.helpmanual.io' 53 | Funding = 'https://github.com/sponsors/samuelcolvin' 54 | Source = 'https://github.com/samuelcolvin/arq' 55 | Changelog = 'https://github.com/samuelcolvin/arq/releases' 56 | 57 | [tool.pytest.ini_options] 58 | testpaths = 'tests' 59 | filterwarnings = ['error'] 60 | asyncio_mode = 'auto' 61 | timeout = 10 62 | 63 | [tool.coverage.run] 64 | source = ['arq'] 65 | branch = true 66 | omit = ['arq/__main__.py'] 67 | 68 | [tool.coverage.report] 69 | precision = 2 70 | exclude_lines = [ 71 | 'pragma: no cover', 72 | 'raise NotImplementedError', 73 | 'raise NotImplemented', 74 | 'if TYPE_CHECKING:', 75 | '@overload', 76 | ] 77 | 78 | [tool.ruff] 79 | line-length = 120 80 | 81 | [tool.ruff.lint] 82 | extend-select = ['Q', 'RUF100', 'C90', 'UP', 'I'] 83 | flake8-quotes = {inline-quotes = 'single', multiline-quotes = 'double'} 84 | mccabe = { max-complexity = 13 } 85 | 86 | [tool.ruff.format] 87 | quote-style = 'single' 88 | 89 | [tool.mypy] 90 | strict = true 91 | -------------------------------------------------------------------------------- /requirements/all.txt: -------------------------------------------------------------------------------- 1 | -r ./docs.txt 2 | -r ./linting.txt 3 | -r ./testing.txt 4 | -r ./pyproject.txt 5 | -------------------------------------------------------------------------------- /requirements/docs.in: -------------------------------------------------------------------------------- 1 | Sphinx>=5,<6 2 | -------------------------------------------------------------------------------- /requirements/docs.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with Python 3.12 3 | # by the following command: 4 | # 5 | # pip-compile --output-file=requirements/docs.txt --strip-extras requirements/docs.in 6 | # 7 | alabaster==0.7.16 8 | # via sphinx 9 | babel==2.14.0 10 | # via sphinx 11 | certifi==2024.7.4 12 | # via requests 13 | charset-normalizer==3.3.2 14 | # via requests 15 | docutils==0.19 16 | # via sphinx 17 | idna==3.7 18 | # via requests 19 | imagesize==1.4.1 20 | # via sphinx 21 | jinja2==3.1.4 22 | # via sphinx 23 | markupsafe==2.1.5 24 | # via jinja2 25 | packaging==24.0 26 | # via sphinx 27 | pygments==2.17.2 28 | # via sphinx 29 | requests==2.32.3 30 | # via sphinx 31 | snowballstemmer==2.2.0 32 | # via sphinx 33 | sphinx==5.3.0 34 | # via -r docs.in 35 | sphinxcontrib-applehelp==1.0.8 36 | # via sphinx 37 | sphinxcontrib-devhelp==1.0.6 38 | # via sphinx 39 | sphinxcontrib-htmlhelp==2.0.5 40 | # via sphinx 41 | sphinxcontrib-jsmath==1.0.1 42 | # via sphinx 43 | sphinxcontrib-qthelp==1.0.7 44 | # via sphinx 45 | sphinxcontrib-serializinghtml==1.1.10 46 | # via sphinx 47 | urllib3==2.2.2 48 | # via requests 49 | -------------------------------------------------------------------------------- /requirements/linting.in: -------------------------------------------------------------------------------- 1 | ruff 2 | mypy 3 | types-pytz 4 | types_redis 5 | -------------------------------------------------------------------------------- /requirements/linting.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with Python 3.12 3 | # by the following command: 4 | # 5 | # pip-compile --output-file=requirements/linting.txt --strip-extras requirements/linting.in 6 | # 7 | cffi==1.16.0 8 | # via cryptography 9 | cryptography==42.0.5 10 | # via 11 | # types-pyopenssl 12 | # types-redis 13 | mypy==1.9.0 14 | # via -r requirements/linting.in 15 | mypy-extensions==1.0.0 16 | # via mypy 17 | pycparser==2.22 18 | # via cffi 19 | ruff==0.3.4 20 | # via -r requirements/linting.in 21 | types-pyopenssl==24.0.0.20240311 22 | # via types-redis 23 | types-pytz==2024.1.0.20240203 24 | # via -r requirements/linting.in 25 | types-redis==4.6.0.20240311 26 | # via -r requirements/linting.in 27 | typing-extensions==4.10.0 28 | # via mypy 29 | -------------------------------------------------------------------------------- /requirements/pyproject.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with Python 3.12 3 | # by the following command: 4 | # 5 | # pip-compile --all-extras --output-file=requirements/pyproject.txt --strip-extras pyproject.toml 6 | # 7 | anyio==4.3.0 8 | # via watchfiles 9 | click==8.1.7 10 | # via arq (pyproject.toml) 11 | hiredis==2.3.2 12 | # via redis 13 | idna==3.7 14 | # via anyio 15 | redis==4.6.0 16 | # via arq (pyproject.toml) 17 | sniffio==1.3.1 18 | # via anyio 19 | watchfiles==0.21.0 20 | # via arq (pyproject.toml) 21 | -------------------------------------------------------------------------------- /requirements/testing.in: -------------------------------------------------------------------------------- 1 | coverage[toml] 2 | dirty-equals 3 | msgpack 4 | pydantic 5 | pytest 6 | pytest-asyncio 7 | pytest-mock 8 | pytest-pretty 9 | pytest-timeout 10 | pytz 11 | testcontainers<4 # until we remove 3.8 support 12 | -------------------------------------------------------------------------------- /requirements/testing.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with Python 3.9 3 | # by the following command: 4 | # 5 | # pip-compile --output-file=requirements/testing.txt --strip-extras requirements/testing.in 6 | # 7 | annotated-types==0.6.0 8 | # via pydantic 9 | certifi==2024.7.4 10 | # via requests 11 | charset-normalizer==3.3.2 12 | # via requests 13 | coverage==7.4.4 14 | # via -r requirements/testing.in 15 | deprecation==2.1.0 16 | # via testcontainers 17 | dirty-equals==0.7.1.post0 18 | # via -r requirements/testing.in 19 | docker==7.1.0 20 | # via testcontainers 21 | exceptiongroup==1.2.2 22 | # via pytest 23 | idna==3.7 24 | # via requests 25 | iniconfig==2.0.0 26 | # via pytest 27 | markdown-it-py==3.0.0 28 | # via rich 29 | mdurl==0.1.2 30 | # via markdown-it-py 31 | msgpack==1.0.8 32 | # via -r requirements/testing.in 33 | packaging==24.0 34 | # via 35 | # deprecation 36 | # pytest 37 | pluggy==1.4.0 38 | # via pytest 39 | pydantic==2.6.4 40 | # via -r requirements/testing.in 41 | pydantic-core==2.16.3 42 | # via pydantic 43 | pygments==2.17.2 44 | # via rich 45 | pytest==8.1.1 46 | # via 47 | # -r requirements/testing.in 48 | # pytest-asyncio 49 | # pytest-mock 50 | # pytest-pretty 51 | # pytest-timeout 52 | pytest-asyncio==0.23.6 53 | # via -r requirements/testing.in 54 | pytest-mock==3.14.0 55 | # via -r requirements/testing.in 56 | pytest-pretty==1.2.0 57 | # via -r requirements/testing.in 58 | pytest-timeout==2.3.1 59 | # via -r requirements/testing.in 60 | pytz==2024.1 61 | # via 62 | # -r requirements/testing.in 63 | # dirty-equals 64 | requests==2.32.3 65 | # via docker 66 | rich==13.7.1 67 | # via pytest-pretty 68 | testcontainers==3.7.1 69 | # via -r requirements/testing.in 70 | tomli==2.0.1 71 | # via 72 | # coverage 73 | # pytest 74 | typing-extensions==4.10.0 75 | # via 76 | # pydantic 77 | # pydantic-core 78 | urllib3==2.2.2 79 | # via 80 | # docker 81 | # requests 82 | wrapt==1.16.0 83 | # via testcontainers 84 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/python-arq/arq/7a911f37992bad4556a27756f2ff027dd2afe655/tests/__init__.py -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import functools 3 | import os 4 | import sys 5 | from typing import Generator 6 | 7 | import msgpack 8 | import pytest 9 | import redis.exceptions 10 | from redis.asyncio.retry import Retry 11 | from redis.backoff import NoBackoff 12 | from testcontainers.redis import RedisContainer 13 | 14 | from arq.connections import ArqRedis, RedisSettings, create_pool 15 | from arq.worker import Worker 16 | 17 | 18 | @pytest.fixture(name='loop') 19 | def _fix_loop(event_loop: asyncio.AbstractEventLoop) -> asyncio.AbstractEventLoop: 20 | return event_loop 21 | 22 | 23 | @pytest.fixture(scope='session') 24 | def redis_version() -> str: 25 | return os.getenv('ARQ_TEST_REDIS_VERSION', 'latest') 26 | 27 | 28 | @pytest.fixture(scope='session') 29 | def redis_container(redis_version: str) -> Generator[RedisContainer, None, None]: 30 | with RedisContainer(f'redis:{redis_version}') as redis: 31 | yield redis 32 | 33 | 34 | @pytest.fixture(scope='session') 35 | def test_redis_host(redis_container: RedisContainer) -> str: 36 | return redis_container.get_container_host_ip() 37 | 38 | 39 | @pytest.fixture(scope='session') 40 | def test_redis_port(redis_container: RedisContainer) -> int: 41 | return redis_container.get_exposed_port(redis_container.port_to_expose) 42 | 43 | 44 | @pytest.fixture(scope='session') 45 | def test_redis_settings(test_redis_host: str, test_redis_port: int) -> RedisSettings: 46 | return RedisSettings(host=test_redis_host, port=test_redis_port) 47 | 48 | 49 | @pytest.fixture 50 | async def arq_redis(test_redis_host: str, test_redis_port: int): 51 | redis_ = ArqRedis( 52 | host=test_redis_host, 53 | port=test_redis_port, 54 | encoding='utf-8', 55 | ) 56 | 57 | await redis_.flushall() 58 | 59 | yield redis_ 60 | 61 | await redis_.close(close_connection_pool=True) 62 | 63 | 64 | @pytest.fixture 65 | async def arq_redis_msgpack(test_redis_host: str, test_redis_port: int): 66 | redis_ = ArqRedis( 67 | host=test_redis_host, 68 | port=test_redis_port, 69 | encoding='utf-8', 70 | job_serializer=msgpack.packb, 71 | job_deserializer=functools.partial(msgpack.unpackb, raw=False), 72 | ) 73 | await redis_.flushall() 74 | yield redis_ 75 | await redis_.close(close_connection_pool=True) 76 | 77 | 78 | @pytest.fixture 79 | async def arq_redis_retry(test_redis_host: str, test_redis_port: int): 80 | redis_ = ArqRedis( 81 | host=test_redis_host, 82 | port=test_redis_port, 83 | encoding='utf-8', 84 | retry=Retry(backoff=NoBackoff(), retries=3), 85 | retry_on_timeout=True, 86 | retry_on_error=[redis.exceptions.ConnectionError], 87 | ) 88 | await redis_.flushall() 89 | yield redis_ 90 | await redis_.close(close_connection_pool=True) 91 | 92 | 93 | @pytest.fixture 94 | async def worker(arq_redis): 95 | worker_: Worker = None 96 | 97 | def create(functions=[], burst=True, poll_delay=0, max_jobs=10, arq_redis=arq_redis, **kwargs): 98 | nonlocal worker_ 99 | worker_ = Worker( 100 | functions=functions, redis_pool=arq_redis, burst=burst, poll_delay=poll_delay, max_jobs=max_jobs, **kwargs 101 | ) 102 | return worker_ 103 | 104 | yield create 105 | 106 | if worker_: 107 | await worker_.close() 108 | 109 | 110 | @pytest.fixture 111 | async def worker_retry(arq_redis_retry): 112 | worker_retry_: Worker = None 113 | 114 | def create(functions=[], burst=True, poll_delay=0, max_jobs=10, arq_redis=arq_redis_retry, **kwargs): 115 | nonlocal worker_retry_ 116 | worker_retry_ = Worker( 117 | functions=functions, 118 | redis_pool=arq_redis, 119 | burst=burst, 120 | poll_delay=poll_delay, 121 | max_jobs=max_jobs, 122 | **kwargs, 123 | ) 124 | return worker_retry_ 125 | 126 | yield create 127 | 128 | if worker_retry_: 129 | await worker_retry_.close() 130 | 131 | 132 | @pytest.fixture(name='create_pool') 133 | async def fix_create_pool(loop): 134 | pools = [] 135 | 136 | async def create_pool_(settings, *args, **kwargs): 137 | pool = await create_pool(settings, *args, **kwargs) 138 | pools.append(pool) 139 | return pool 140 | 141 | yield create_pool_ 142 | 143 | await asyncio.gather(*[p.close(close_connection_pool=True) for p in pools]) 144 | 145 | 146 | @pytest.fixture(name='cancel_remaining_task') 147 | def fix_cancel_remaining_task(loop): 148 | async def cancel_remaining_task(): 149 | tasks = asyncio.all_tasks(loop) 150 | cancelled = [] 151 | for task in tasks: 152 | # in repr works in 3.7 where get_coro() is not available 153 | if 'cancel_remaining_task()' not in repr(task): 154 | cancelled.append(task) 155 | task.cancel() 156 | if cancelled: 157 | print(f'Cancelled {len(cancelled)} ongoing tasks', file=sys.stderr) 158 | await asyncio.gather(*cancelled, return_exceptions=True) 159 | 160 | yield 161 | 162 | loop.run_until_complete(cancel_remaining_task()) 163 | 164 | 165 | class SetEnv: 166 | def __init__(self): 167 | self.envars = set() 168 | 169 | def set(self, name, value): 170 | self.envars.add(name) 171 | os.environ[name] = value 172 | 173 | def clear(self): 174 | for n in self.envars: 175 | os.environ.pop(n) 176 | 177 | 178 | @pytest.fixture 179 | def env(): 180 | setenv = SetEnv() 181 | 182 | yield setenv 183 | 184 | setenv.clear() 185 | -------------------------------------------------------------------------------- /tests/test_cli.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from click.testing import CliRunner 3 | 4 | from arq import logs 5 | from arq.cli import cli 6 | from arq.connections import RedisSettings 7 | 8 | 9 | async def foobar(ctx): 10 | return 42 11 | 12 | 13 | class WorkerSettings: 14 | burst = True 15 | functions = [foobar] 16 | 17 | 18 | @pytest.fixture(scope='module', autouse=True) 19 | def setup_worker_connection(test_redis_host: str, test_redis_port: int): 20 | WorkerSettings.redis_settings = RedisSettings(host=test_redis_host, port=test_redis_port) 21 | 22 | 23 | def test_help(): 24 | runner = CliRunner() 25 | result = runner.invoke(cli, ['--help']) 26 | assert result.exit_code == 0 27 | assert result.output.startswith('Usage: arq [OPTIONS] WORKER_SETTINGS\n') 28 | 29 | 30 | def test_run(cancel_remaining_task, mocker, loop): 31 | mocker.patch('asyncio.get_event_loop', lambda: loop) 32 | runner = CliRunner() 33 | result = runner.invoke(cli, ['tests.test_cli.WorkerSettings']) 34 | assert result.exit_code == 0 35 | assert 'Starting worker for 1 functions: foobar' in result.output 36 | 37 | 38 | def test_check(loop): 39 | runner = CliRunner() 40 | result = runner.invoke(cli, ['tests.test_cli.WorkerSettings', '--check']) 41 | assert result.exit_code == 1 42 | assert 'Health check failed: no health check sentinel value found' in result.output 43 | 44 | 45 | async def mock_awatch(): 46 | yield [1] 47 | 48 | 49 | def test_run_watch(mocker, cancel_remaining_task): 50 | mocker.patch('watchfiles.awatch', return_value=mock_awatch()) 51 | runner = CliRunner() 52 | result = runner.invoke(cli, ['tests.test_cli.WorkerSettings', '--watch', 'tests']) 53 | assert result.exit_code == 0 54 | assert '1 files changes, reloading arq worker...' 55 | 56 | 57 | custom_log_dict = { 58 | 'version': 1, 59 | 'handlers': {'custom': {'level': 'ERROR', 'class': 'logging.StreamHandler', 'formatter': 'custom'}}, 60 | 'formatters': {'custom': {'format': '%(asctime)s: %(message)s', 'datefmt': '%H:%M:%S'}}, 61 | 'loggers': {'arq': {'handlers': ['custom'], 'level': 'ERROR'}}, 62 | } 63 | 64 | 65 | @pytest.mark.parametrize( 66 | 'cli_argument,log_dict_to_use', 67 | [ 68 | (None, logs.default_log_config(verbose=False)), 69 | ('--custom-log-dict=tests.test_cli.custom_log_dict', custom_log_dict), 70 | ], 71 | ) 72 | def test_custom_log_dict(mocker, loop, cli_argument, log_dict_to_use): 73 | mocker.patch('asyncio.get_event_loop', lambda: loop) 74 | mock_dictconfig = mocker.MagicMock() 75 | mocker.patch('logging.config.dictConfig', mock_dictconfig) 76 | arq_arguments = ['tests.test_cli.WorkerSettings'] 77 | if cli_argument is not None: 78 | arq_arguments.append(cli_argument) 79 | 80 | runner = CliRunner() 81 | result = runner.invoke(cli, arq_arguments) 82 | assert result.exit_code == 0 83 | mock_dictconfig.assert_called_with(log_dict_to_use) 84 | -------------------------------------------------------------------------------- /tests/test_cron.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import logging 3 | import re 4 | from datetime import datetime, timedelta, timezone 5 | from random import random 6 | 7 | import pytest 8 | from pytest_mock import MockerFixture 9 | 10 | import arq 11 | from arq import Worker 12 | from arq.constants import in_progress_key_prefix 13 | from arq.cron import cron, next_cron 14 | 15 | tz = timezone(offset=timedelta(hours=3)) 16 | 17 | 18 | @pytest.mark.parametrize( 19 | 'previous,expected,kwargs', 20 | [ 21 | ( 22 | datetime(2016, 6, 1, 12, 10, 10, tzinfo=tz), 23 | datetime(2016, 6, 1, 12, 10, 20, microsecond=123_456, tzinfo=tz), 24 | dict(second=20), 25 | ), 26 | ( 27 | datetime(2016, 6, 1, 12, 10, 10, tzinfo=tz), 28 | datetime(2016, 6, 1, 12, 11, 0, microsecond=123_456, tzinfo=tz), 29 | dict(minute=11), 30 | ), 31 | ( 32 | datetime(2016, 6, 1, 12, 10, 10, tzinfo=tz), 33 | datetime(2016, 6, 1, 12, 10, 20, tzinfo=tz), 34 | dict(second=20, microsecond=0), 35 | ), 36 | ( 37 | datetime(2016, 6, 1, 12, 10, 10, tzinfo=tz), 38 | datetime(2016, 6, 1, 12, 11, 0, tzinfo=tz), 39 | dict(minute=11, microsecond=0), 40 | ), 41 | ( 42 | datetime(2016, 6, 1, 12, 10, 11, tzinfo=tz), 43 | datetime(2017, 6, 1, 12, 10, 10, microsecond=123_456, tzinfo=tz), 44 | dict(month=6, day=1, hour=12, minute=10, second=10), 45 | ), 46 | ( 47 | datetime(2016, 6, 1, 12, 10, 10, microsecond=1, tzinfo=tz), 48 | datetime(2016, 7, 1, 12, 10, 10, tzinfo=tz), 49 | dict(day=1, hour=12, minute=10, second=10, microsecond=0), 50 | ), 51 | ( 52 | datetime(2032, 1, 31, 0, 0, 0, tzinfo=tz), 53 | datetime(2032, 2, 28, 0, 0, 0, microsecond=123_456, tzinfo=tz), 54 | dict(day=28), 55 | ), 56 | ( 57 | datetime(2032, 1, 1, 0, 5, tzinfo=tz), 58 | datetime(2032, 1, 1, 4, 0, microsecond=123_456, tzinfo=tz), 59 | dict(hour=4), 60 | ), 61 | ( 62 | datetime(2032, 1, 1, 0, 0, tzinfo=tz), 63 | datetime(2032, 1, 1, 4, 2, microsecond=123_456, tzinfo=tz), 64 | dict(hour=4, minute={2, 4, 6}), 65 | ), 66 | ( 67 | datetime(2032, 1, 1, 0, 5, tzinfo=tz), 68 | datetime(2032, 1, 1, 4, 2, microsecond=123_456, tzinfo=tz), 69 | dict(hour=4, minute={2, 4, 6}), 70 | ), 71 | ( 72 | datetime(2032, 2, 5, 0, 0, 0, tzinfo=tz), 73 | datetime(2032, 3, 31, 0, 0, 0, microsecond=123_456, tzinfo=tz), 74 | dict(day=31), 75 | ), 76 | ( 77 | datetime(2001, 1, 1, 0, 0, 0, tzinfo=tz), # Monday 78 | datetime(2001, 1, 7, 0, 0, 0, microsecond=123_456, tzinfo=tz), 79 | dict(weekday='Sun'), # Sunday 80 | ), 81 | ( 82 | datetime(2001, 1, 1, 0, 0, 0, tzinfo=tz), 83 | datetime(2001, 1, 7, 0, 0, 0, microsecond=123_456, tzinfo=tz), 84 | dict(weekday=6), 85 | ), # Sunday 86 | ( 87 | datetime(2001, 1, 1, 0, 0, 0, tzinfo=tz), 88 | datetime(2001, 11, 7, 0, 0, 0, microsecond=123_456, tzinfo=tz), 89 | dict(month=11, weekday=2), 90 | ), 91 | ( 92 | datetime(2001, 1, 1, 0, 0, 0, tzinfo=tz), 93 | datetime(2001, 1, 3, 0, 0, 0, microsecond=123_456, tzinfo=tz), 94 | dict(weekday='wed'), 95 | ), 96 | ], 97 | ) 98 | def test_next_cron(previous, expected, kwargs): 99 | start = datetime.now() 100 | assert next_cron(previous, **kwargs) == expected 101 | diff = datetime.now() - start 102 | print(f'{diff.total_seconds() * 1000:0.3f}ms') 103 | 104 | 105 | def test_next_cron_preserves_tzinfo(): 106 | previous = datetime.fromisoformat('2016-06-01T12:10:10+02:00') 107 | assert previous.tzinfo is not None 108 | assert next_cron(previous, second=20).tzinfo is previous.tzinfo 109 | 110 | 111 | def test_next_cron_invalid(): 112 | with pytest.raises(ValueError): 113 | next_cron(datetime(2001, 1, 1, 0, 0, 0), weekday='monday') 114 | 115 | 116 | @pytest.mark.parametrize( 117 | 'max_previous,kwargs,expected', 118 | [ 119 | (1, dict(microsecond=59867), datetime(2001, 1, 1, 0, 0, microsecond=59867)), 120 | (59, dict(second=28, microsecond=0), datetime(2023, 1, 1, 1, 59, 28)), 121 | (3600, dict(minute=10, second=20), datetime(2016, 6, 1, 12, 10, 20, microsecond=123_456)), 122 | (68400, dict(hour=3), datetime(2032, 1, 1, 3, 0, 0, microsecond=123_456)), 123 | (68400 * 60, dict(day=31, minute=59), datetime(2032, 3, 31, 0, 59, 0, microsecond=123_456)), 124 | (68400 * 7, dict(weekday='tues', minute=59), datetime(2032, 3, 30, 0, 59, 0, microsecond=123_456)), 125 | ( 126 | 68400 * 175, # previous friday the 13th is February 127 | dict(day=13, weekday='fri', microsecond=1), 128 | datetime(2032, 8, 13, 0, 0, 0, microsecond=1), 129 | ), 130 | (68400 * 365, dict(month=10, day=4, hour=23), datetime(2032, 10, 4, 23, 0, 0, microsecond=123_456)), 131 | ( 132 | 1, 133 | dict(month=1, day=1, hour=0, minute=0, second=0, microsecond=69875), 134 | datetime(2001, 1, 1, 0, 0, microsecond=69875), 135 | ), 136 | ], 137 | ) 138 | def test_next_cron_random(max_previous, kwargs, expected): 139 | for i in range(100): 140 | previous = expected - timedelta(seconds=0.9 + random() * max_previous) 141 | v = next_cron(previous, **kwargs) 142 | diff = v - previous 143 | if diff > timedelta(seconds=1): 144 | print(f'previous: {previous}, expected: {expected}, diff: {v - previous}') 145 | assert v == expected 146 | 147 | 148 | async def foobar(ctx): 149 | return 42 150 | 151 | 152 | @pytest.mark.parametrize('poll_delay', [0.0, 0.5, 0.9]) 153 | async def test_job_successful(worker, caplog, arq_redis, poll_delay): 154 | caplog.set_level(logging.INFO) 155 | worker: Worker = worker(cron_jobs=[cron(foobar, hour=1, run_at_startup=True)], poll_delay=poll_delay) 156 | await worker.main() 157 | assert worker.jobs_complete == 1 158 | assert worker.jobs_failed == 0 159 | assert worker.jobs_retried == 0 160 | 161 | log = re.sub(r'(\d+).\d\ds', r'\1.XXs', '\n'.join(r.message for r in caplog.records)) 162 | assert ' 0.XXs → cron:foobar()\n 0.XXs ← cron:foobar ● 42' in log 163 | 164 | # Assert the in-progress key still exists. 165 | keys = await arq_redis.keys(in_progress_key_prefix + '*') 166 | assert len(keys) == 1 167 | assert await arq_redis.pttl(keys[0]) > 0.0 168 | 169 | 170 | async def test_calculate_next_is_called_with_aware_datetime(worker, mocker: MockerFixture): 171 | worker: Worker = worker(cron_jobs=[cron(foobar, hour=1)]) 172 | spy_run_cron = mocker.spy(worker, worker.run_cron.__name__) 173 | 174 | await worker.main() 175 | 176 | assert spy_run_cron.call_args[0][0].tzinfo is not None 177 | assert worker.cron_jobs[0].next_run.tzinfo is not None 178 | 179 | 180 | async def test_job_successful_on_specific_queue(worker, caplog): 181 | caplog.set_level(logging.INFO) 182 | worker: Worker = worker( 183 | queue_name='arq:test-cron-queue', cron_jobs=[cron(foobar, hour=1, run_at_startup=True)], poll_delay=0.5 184 | ) 185 | await worker.main() 186 | assert worker.jobs_complete == 1 187 | assert worker.jobs_failed == 0 188 | assert worker.jobs_retried == 0 189 | 190 | log = re.sub(r'(\d+).\d\ds', r'\1.XXs', '\n'.join(r.message for r in caplog.records)) 191 | assert ' 0.XXs → cron:foobar()\n 0.XXs ← cron:foobar ● 42' in log 192 | 193 | 194 | async def test_not_run(worker, caplog): 195 | caplog.set_level(logging.INFO) 196 | worker: Worker = worker(cron_jobs=[cron(foobar, hour=1, run_at_startup=False)]) 197 | await worker.main() 198 | assert worker.jobs_complete == 0 199 | assert worker.jobs_failed == 0 200 | assert worker.jobs_retried == 0 201 | 202 | log = '\n'.join(r.message for r in caplog.records) 203 | assert 'cron:foobar()' not in log 204 | 205 | 206 | async def test_repr(): 207 | cj = cron(foobar, hour=1, run_at_startup=True) 208 | assert str(cj).startswith('' 26 | 27 | 28 | async def test_unknown(arq_redis: ArqRedis): 29 | j = Job('foobar', arq_redis) 30 | assert JobStatus.not_found == await j.status() 31 | info = await j.info() 32 | assert info is None 33 | 34 | 35 | async def test_result_timeout(arq_redis: ArqRedis): 36 | j = await arq_redis.enqueue_job('foobar') 37 | with pytest.raises(asyncio.TimeoutError): 38 | await j.result(0.1, poll_delay=0) 39 | 40 | 41 | async def test_result_not_found(arq_redis: ArqRedis): 42 | j = Job('foobar', arq_redis) 43 | with pytest.raises(ResultNotFound): 44 | await j.result() 45 | 46 | 47 | async def test_result_when_job_does_not_keep_result(arq_redis: ArqRedis, worker): 48 | async def foobar(ctx): 49 | pass 50 | 51 | worker: Worker = worker(functions=[func(foobar, name='foobar', keep_result=0)]) 52 | j = await arq_redis.enqueue_job('foobar') 53 | 54 | result_call = asyncio.Task(j.result()) 55 | 56 | _, pending = await asyncio.wait([result_call], timeout=0.1) 57 | assert pending == {result_call} 58 | 59 | await worker.main() 60 | 61 | with pytest.raises(ResultNotFound): 62 | # Job has completed and did not store any result 63 | await asyncio.wait_for(result_call, timeout=5) 64 | 65 | 66 | async def test_enqueue_job(arq_redis: ArqRedis, worker, queue_name=default_queue_name): 67 | async def foobar(ctx, *args, **kwargs): 68 | return 42 69 | 70 | j = await arq_redis.enqueue_job('foobar', 1, 2, c=3, _queue_name=queue_name) 71 | assert isinstance(j, Job) 72 | assert JobStatus.queued == await j.status() 73 | worker: Worker = worker(functions=[func(foobar, name='foobar')], queue_name=queue_name) 74 | await worker.main() 75 | r = await j.result(poll_delay=0) 76 | assert r == 42 77 | assert JobStatus.complete == await j.status() 78 | info = await j.info() 79 | expected_queue_name = queue_name or arq_redis.default_queue_name 80 | assert info == JobResult( 81 | job_try=1, 82 | function='foobar', 83 | args=(1, 2), 84 | kwargs={'c': 3}, 85 | enqueue_time=IsNow(tz='utc'), 86 | success=True, 87 | result=42, 88 | start_time=IsNow(tz='utc'), 89 | finish_time=IsNow(tz='utc'), 90 | score=None, 91 | queue_name=expected_queue_name, 92 | job_id=IsStr(), 93 | ) 94 | results = await arq_redis.all_job_results() 95 | assert results == [ 96 | JobResult( 97 | function='foobar', 98 | args=(1, 2), 99 | kwargs={'c': 3}, 100 | job_try=1, 101 | enqueue_time=IsNow(tz='utc'), 102 | success=True, 103 | result=42, 104 | start_time=IsNow(tz='utc'), 105 | finish_time=IsNow(tz='utc'), 106 | score=None, 107 | queue_name=expected_queue_name, 108 | job_id=j.job_id, 109 | ) 110 | ] 111 | 112 | 113 | async def test_enqueue_job_alt_queue(arq_redis: ArqRedis, worker): 114 | await test_enqueue_job(arq_redis, worker, queue_name='custom_queue') 115 | 116 | 117 | async def test_enqueue_job_nondefault_queue(test_redis_settings: RedisSettings, worker): 118 | """Test initializing arq_redis with a queue name, and the worker using it.""" 119 | arq_redis = await create_pool(test_redis_settings, default_queue_name='test_queue') 120 | await test_enqueue_job( 121 | arq_redis, 122 | lambda functions, **_: worker(functions=functions, arq_redis=arq_redis, queue_name=None), 123 | queue_name=None, 124 | ) 125 | 126 | 127 | async def test_cant_unpickle_at_all(): 128 | class Foobar: 129 | def __getstate__(self): 130 | raise TypeError("this doesn't pickle") 131 | 132 | r1 = serialize_result('foobar', (1,), {}, 1, 123, True, Foobar(), 123, 123, 'testing', 'test-queue', 'job_1') 133 | assert isinstance(r1, bytes) 134 | r2 = serialize_result('foobar', (Foobar(),), {}, 1, 123, True, Foobar(), 123, 123, 'testing', 'test-queue', 'job_1') 135 | assert r2 is None 136 | 137 | 138 | async def test_custom_serializer(): 139 | class Foobar: 140 | def __getstate__(self): 141 | raise TypeError("this doesn't pickle") 142 | 143 | def custom_serializer(x): 144 | return b'0123456789' 145 | 146 | r1 = serialize_result( 147 | 'foobar', 148 | (1,), 149 | {}, 150 | 1, 151 | 123, 152 | True, 153 | Foobar(), 154 | 123, 155 | 123, 156 | 'testing', 157 | 'test-queue', 158 | 'job_1', 159 | serializer=custom_serializer, 160 | ) 161 | assert r1 == b'0123456789' 162 | r2 = serialize_result( 163 | 'foobar', 164 | (Foobar(),), 165 | {}, 166 | 1, 167 | 123, 168 | True, 169 | Foobar(), 170 | 123, 171 | 123, 172 | 'testing', 173 | 'test-queue', 174 | 'job_1', 175 | serializer=custom_serializer, 176 | ) 177 | assert r2 == b'0123456789' 178 | 179 | 180 | async def test_deserialize_result(arq_redis: ArqRedis, worker): 181 | async def foobar(ctx, a, b): 182 | return a + b 183 | 184 | j = await arq_redis.enqueue_job('foobar', 1, 2) 185 | assert JobStatus.queued == await j.status() 186 | worker: Worker = worker(functions=[func(foobar, name='foobar')]) 187 | await worker.run_check() 188 | assert await j.result(poll_delay=0) == 3 189 | assert await j.result(poll_delay=0) == 3 190 | info = await j.info() 191 | assert info.args == (1, 2) 192 | await arq_redis.set(result_key_prefix + j.job_id, b'invalid pickle data') 193 | with pytest.raises(DeserializationError, match='unable to deserialize job result'): 194 | assert await j.result(poll_delay=0) == 3 195 | 196 | 197 | async def test_deserialize_info(arq_redis: ArqRedis): 198 | j = await arq_redis.enqueue_job('foobar', 1, 2) 199 | assert JobStatus.queued == await j.status() 200 | await arq_redis.set(job_key_prefix + j.job_id, b'invalid pickle data') 201 | 202 | with pytest.raises(DeserializationError, match='unable to deserialize job'): 203 | assert await j.info() 204 | 205 | 206 | async def test_deserialize_job_raw(): 207 | assert deserialize_job_raw(pickle.dumps({'f': 1, 'a': 2, 'k': 3, 't': 4, 'et': 5})) == (1, 2, 3, 4, 5) 208 | with pytest.raises(DeserializationError, match='unable to deserialize job'): 209 | deserialize_job_raw(b'123') 210 | 211 | 212 | async def test_get_job_result(arq_redis: ArqRedis): 213 | with pytest.raises(KeyError, match='job "foobar" not found'): 214 | await arq_redis._get_job_result(b'foobar') 215 | 216 | 217 | async def test_result_pole_delay_dep(arq_redis: ArqRedis): 218 | j = Job('foobar', arq_redis) 219 | r = serialize_result('foobar', (1,), {}, 1, 123, True, 42, 123, 123, 'testing', 'test-queue', 'job_1') 220 | await arq_redis.set(result_key_prefix + j.job_id, r) 221 | with pytest.warns( 222 | DeprecationWarning, match='"pole_delay" is deprecated, use the correct spelling "poll_delay" instead' 223 | ): 224 | assert await j.result(pole_delay=0) == 42 225 | -------------------------------------------------------------------------------- /tests/test_main.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import dataclasses 3 | import functools 4 | import logging 5 | from collections import Counter 6 | from datetime import datetime, timezone 7 | from random import shuffle 8 | from time import time 9 | 10 | import msgpack 11 | import pytest 12 | from dirty_equals import IsInt, IsNow 13 | 14 | from arq.connections import ArqRedis, RedisSettings 15 | from arq.constants import default_queue_name 16 | from arq.jobs import Job, JobDef, SerializationError 17 | from arq.utils import timestamp_ms 18 | from arq.worker import Retry, Worker, func 19 | 20 | 21 | async def test_enqueue_job(arq_redis: ArqRedis, worker): 22 | async def foobar(ctx): 23 | return 42 24 | 25 | j = await arq_redis.enqueue_job('foobar') 26 | worker: Worker = worker(functions=[func(foobar, name='foobar')]) 27 | await worker.main() 28 | r = await j.result(poll_delay=0) 29 | assert r == 42 # 1 30 | 31 | 32 | async def test_enqueue_job_different_queues(arq_redis: ArqRedis, worker): 33 | async def foobar(ctx): 34 | return 42 35 | 36 | j1 = await arq_redis.enqueue_job('foobar', _queue_name='arq:queue1') 37 | j2 = await arq_redis.enqueue_job('foobar', _queue_name='arq:queue2') 38 | worker1: Worker = worker(functions=[func(foobar, name='foobar')], queue_name='arq:queue1') 39 | worker2: Worker = worker(functions=[func(foobar, name='foobar')], queue_name='arq:queue2') 40 | 41 | await worker1.main() 42 | await worker2.main() 43 | r1 = await j1.result(poll_delay=0) 44 | r2 = await j2.result(poll_delay=0) 45 | assert r1 == 42 # 1 46 | assert r2 == 42 # 2 47 | 48 | 49 | async def test_enqueue_job_nested(arq_redis: ArqRedis, worker): 50 | async def foobar(ctx): 51 | return 42 52 | 53 | async def parent_job(ctx): 54 | inner_job = await ctx['redis'].enqueue_job('foobar') 55 | return inner_job.job_id 56 | 57 | job = await arq_redis.enqueue_job('parent_job') 58 | worker: Worker = worker(functions=[func(parent_job, name='parent_job'), func(foobar, name='foobar')]) 59 | 60 | await worker.main() 61 | result = await job.result(poll_delay=0) 62 | assert result is not None 63 | inner_job = Job(result, arq_redis) 64 | inner_result = await inner_job.result(poll_delay=0) 65 | assert inner_result == 42 66 | 67 | 68 | async def test_enqueue_job_nested_custom_serializer( 69 | arq_redis_msgpack: ArqRedis, test_redis_settings: RedisSettings, worker 70 | ): 71 | async def foobar(ctx): 72 | return 42 73 | 74 | async def parent_job(ctx): 75 | inner_job = await ctx['redis'].enqueue_job('foobar') 76 | return inner_job.job_id 77 | 78 | job = await arq_redis_msgpack.enqueue_job('parent_job') 79 | 80 | worker: Worker = worker( 81 | functions=[func(parent_job, name='parent_job'), func(foobar, name='foobar')], 82 | arq_redis=None, 83 | redis_settings=test_redis_settings, 84 | job_serializer=msgpack.packb, 85 | job_deserializer=functools.partial(msgpack.unpackb, raw=False), 86 | ) 87 | 88 | await worker.main() 89 | result = await job.result(poll_delay=0) 90 | assert result is not None 91 | inner_job = Job(result, arq_redis_msgpack, _deserializer=functools.partial(msgpack.unpackb, raw=False)) 92 | inner_result = await inner_job.result(poll_delay=0) 93 | assert inner_result == 42 94 | 95 | 96 | async def test_enqueue_job_custom_queue(arq_redis: ArqRedis, test_redis_settings: RedisSettings, worker): 97 | async def foobar(ctx): 98 | return 42 99 | 100 | async def parent_job(ctx): 101 | inner_job = await ctx['redis'].enqueue_job('foobar') 102 | return inner_job.job_id 103 | 104 | job = await arq_redis.enqueue_job('parent_job', _queue_name='spanner') 105 | 106 | worker: Worker = worker( 107 | functions=[func(parent_job, name='parent_job'), func(foobar, name='foobar')], 108 | arq_redis=None, 109 | redis_settings=test_redis_settings, 110 | queue_name='spanner', 111 | ) 112 | 113 | await worker.main() 114 | inner_job_id = await job.result(poll_delay=0) 115 | assert inner_job_id is not None 116 | inner_job = Job(inner_job_id, arq_redis, _queue_name='spanner') 117 | inner_result = await inner_job.result(poll_delay=0) 118 | assert inner_result == 42 119 | 120 | 121 | async def test_job_error(arq_redis: ArqRedis, worker): 122 | async def foobar(ctx): 123 | raise RuntimeError('foobar error') 124 | 125 | j = await arq_redis.enqueue_job('foobar') 126 | worker: Worker = worker(functions=[func(foobar, name='foobar')]) 127 | await worker.main() 128 | 129 | with pytest.raises(RuntimeError, match='foobar error'): 130 | await j.result(poll_delay=0) 131 | 132 | 133 | async def test_job_info(arq_redis: ArqRedis): 134 | t_before = time() 135 | j = await arq_redis.enqueue_job('foobar', 123, a=456) 136 | info = await j.info() 137 | assert info.enqueue_time == IsNow(tz='utc') 138 | assert info.job_try is None 139 | assert info.function == 'foobar' 140 | assert info.args == (123,) 141 | assert info.kwargs == {'a': 456} 142 | assert abs(t_before * 1000 - info.score) < 1000 143 | 144 | 145 | async def test_repeat_job(arq_redis: ArqRedis): 146 | j1 = await arq_redis.enqueue_job('foobar', _job_id='job_id') 147 | assert isinstance(j1, Job) 148 | j2 = await arq_redis.enqueue_job('foobar', _job_id='job_id') 149 | assert j2 is None 150 | 151 | 152 | async def test_defer_until(arq_redis: ArqRedis): 153 | j1 = await arq_redis.enqueue_job('foobar', _job_id='job_id', _defer_until=datetime(2032, 1, 1, tzinfo=timezone.utc)) 154 | assert isinstance(j1, Job) 155 | score = await arq_redis.zscore(default_queue_name, 'job_id') 156 | assert score == 1_956_528_000_000 157 | 158 | 159 | async def test_defer_by(arq_redis: ArqRedis): 160 | j1 = await arq_redis.enqueue_job('foobar', _job_id='job_id', _defer_by=20) 161 | assert isinstance(j1, Job) 162 | score = await arq_redis.zscore(default_queue_name, 'job_id') 163 | ts = timestamp_ms() 164 | assert score > ts + 19000 165 | assert ts + 21000 > score 166 | 167 | 168 | async def test_mung(arq_redis: ArqRedis, worker): 169 | """ 170 | check a job can't be enqueued multiple times with the same id 171 | """ 172 | counter = Counter() 173 | 174 | async def count(ctx, v): 175 | counter[v] += 1 176 | 177 | tasks = [] 178 | for i in range(50): 179 | tasks += [ 180 | arq_redis.enqueue_job('count', i, _job_id=f'v-{i}'), 181 | arq_redis.enqueue_job('count', i, _job_id=f'v-{i}'), 182 | ] 183 | shuffle(tasks) 184 | await asyncio.gather(*tasks) 185 | 186 | worker: Worker = worker(functions=[func(count, name='count')]) 187 | await worker.main() 188 | assert counter.most_common(1)[0][1] == 1 # no job go enqueued twice 189 | 190 | 191 | async def test_custom_try(arq_redis: ArqRedis, worker): 192 | async def foobar(ctx): 193 | return ctx['job_try'] 194 | 195 | j1 = await arq_redis.enqueue_job('foobar') 196 | w: Worker = worker(functions=[func(foobar, name='foobar')]) 197 | await w.main() 198 | r = await j1.result(poll_delay=0) 199 | assert r == 1 200 | 201 | j2 = await arq_redis.enqueue_job('foobar', _job_try=3) 202 | await w.main() 203 | r = await j2.result(poll_delay=0) 204 | assert r == 3 205 | 206 | 207 | async def test_custom_try2(arq_redis: ArqRedis, worker): 208 | async def foobar(ctx): 209 | if ctx['job_try'] == 3: 210 | raise Retry() 211 | return ctx['job_try'] 212 | 213 | j1 = await arq_redis.enqueue_job('foobar', _job_try=3) 214 | w: Worker = worker(functions=[func(foobar, name='foobar')]) 215 | await w.main() 216 | r = await j1.result(poll_delay=0) 217 | assert r == 4 218 | 219 | 220 | async def test_cant_pickle_arg(arq_redis: ArqRedis): 221 | class Foobar: 222 | def __getstate__(self): 223 | raise TypeError("this doesn't pickle") 224 | 225 | with pytest.raises(SerializationError, match='unable to serialize job "foobar"'): 226 | await arq_redis.enqueue_job('foobar', Foobar()) 227 | 228 | 229 | async def test_cant_pickle_result(arq_redis: ArqRedis, worker): 230 | class Foobar: 231 | def __getstate__(self): 232 | raise TypeError("this doesn't pickle") 233 | 234 | async def foobar(ctx): 235 | return Foobar() 236 | 237 | j1 = await arq_redis.enqueue_job('foobar') 238 | w: Worker = worker(functions=[func(foobar, name='foobar')]) 239 | await w.main() 240 | with pytest.raises(SerializationError, match='unable to serialize result'): 241 | await j1.result(poll_delay=0) 242 | 243 | 244 | async def test_get_jobs(arq_redis: ArqRedis): 245 | await arq_redis.enqueue_job('foobar', a=1, b=2, c=3, _job_id='1') 246 | await asyncio.sleep(0.01) 247 | await arq_redis.enqueue_job('second', 4, b=5, c=6, _job_id='2') 248 | await asyncio.sleep(0.01) 249 | await arq_redis.enqueue_job('third', 7, b=8, _job_id='3') 250 | jobs = await arq_redis.queued_jobs() 251 | assert [dataclasses.asdict(j) for j in jobs] == [ 252 | { 253 | 'function': 'foobar', 254 | 'args': (), 255 | 'kwargs': {'a': 1, 'b': 2, 'c': 3}, 256 | 'job_try': None, 257 | 'enqueue_time': IsNow(tz='utc'), 258 | 'score': IsInt(), 259 | 'job_id': '1', 260 | }, 261 | { 262 | 'function': 'second', 263 | 'args': (4,), 264 | 'kwargs': {'b': 5, 'c': 6}, 265 | 'job_try': None, 266 | 'enqueue_time': IsNow(tz='utc'), 267 | 'score': IsInt(), 268 | 'job_id': '2', 269 | }, 270 | { 271 | 'function': 'third', 272 | 'args': (7,), 273 | 'kwargs': {'b': 8}, 274 | 'job_try': None, 275 | 'enqueue_time': IsNow(tz='utc'), 276 | 'score': IsInt(), 277 | 'job_id': '3', 278 | }, 279 | ] 280 | assert jobs[0].score < jobs[1].score < jobs[2].score 281 | assert isinstance(jobs[0], JobDef) 282 | assert isinstance(jobs[1], JobDef) 283 | assert isinstance(jobs[2], JobDef) 284 | 285 | 286 | async def test_enqueue_multiple(arq_redis: ArqRedis, caplog): 287 | caplog.set_level(logging.DEBUG) 288 | results = await asyncio.gather(*[arq_redis.enqueue_job('foobar', i, _job_id='testing') for i in range(10)]) 289 | assert sum(r is not None for r in results) == 1 290 | assert sum(r is None for r in results) == 9 291 | assert 'WatchVariableError' not in caplog.text 292 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import re 3 | import sys 4 | from dataclasses import asdict 5 | from datetime import timedelta 6 | 7 | import pytest 8 | from pydantic import BaseModel, field_validator 9 | from redis.asyncio import ConnectionError, ResponseError 10 | 11 | import arq.typing 12 | import arq.utils 13 | from arq.connections import RedisSettings, log_redis_info 14 | 15 | from .conftest import SetEnv 16 | 17 | 18 | def test_settings_changed(): 19 | settings = RedisSettings(port=123) 20 | assert settings.port == 123 21 | assert ( 22 | "RedisSettings(host='localhost', port=123, unix_socket_path=None, database=0, username=None, password=None, " 23 | "ssl=False, ssl_keyfile=None, ssl_certfile=None, ssl_cert_reqs='required', ssl_ca_certs=None, " 24 | 'ssl_ca_data=None, ssl_check_hostname=False, conn_timeout=1, conn_retries=5, conn_retry_delay=1, ' 25 | "max_connections=None, sentinel=False, sentinel_master='mymaster', " 26 | 'retry_on_timeout=False, retry_on_error=None, retry=None)' 27 | ) == str(settings) 28 | 29 | 30 | async def test_redis_timeout(mocker, create_pool): 31 | mocker.spy(arq.utils.asyncio, 'sleep') 32 | with pytest.raises(ConnectionError): 33 | await create_pool(RedisSettings(port=0, conn_retry_delay=0)) 34 | assert arq.utils.asyncio.sleep.call_count == 5 35 | 36 | 37 | async def test_redis_timeout_and_retry_many_times(mocker, create_pool): 38 | mocker.spy(arq.utils.asyncio, 'sleep') 39 | default_recursion_limit = sys.getrecursionlimit() 40 | sys.setrecursionlimit(100) 41 | try: 42 | with pytest.raises(ConnectionError): 43 | await create_pool(RedisSettings(port=0, conn_retry_delay=0, conn_retries=150)) 44 | assert arq.utils.asyncio.sleep.call_count == 150 45 | finally: 46 | sys.setrecursionlimit(default_recursion_limit) 47 | 48 | 49 | @pytest.mark.skip(reason='this breaks many other tests as low level connections remain after failed connection') 50 | async def test_redis_sentinel_failure(create_pool, cancel_remaining_task, mocker): 51 | settings = RedisSettings() 52 | settings.host = [('localhost', 6379), ('localhost', 6379)] 53 | settings.sentinel = True 54 | with pytest.raises(ResponseError, match='unknown command `SENTINEL`'): 55 | await create_pool(settings) 56 | 57 | 58 | async def test_redis_success_log(test_redis_settings: RedisSettings, caplog, create_pool): 59 | caplog.set_level(logging.INFO) 60 | pool = await create_pool(test_redis_settings) 61 | assert 'redis connection successful' not in [r.message for r in caplog.records] 62 | await pool.close(close_connection_pool=True) 63 | 64 | pool = await create_pool(test_redis_settings, retry=1) 65 | assert 'redis connection successful' in [r.message for r in caplog.records] 66 | await pool.close(close_connection_pool=True) 67 | 68 | 69 | async def test_redis_log(test_redis_settings: RedisSettings, create_pool): 70 | redis = await create_pool(test_redis_settings) 71 | await redis.flushall() 72 | await redis.set(b'a', b'1') 73 | await redis.set(b'b', b'2') 74 | 75 | log_msgs = [] 76 | 77 | def _log(s): 78 | log_msgs.append(s) 79 | 80 | await log_redis_info(redis, _log) 81 | assert len(log_msgs) == 1 82 | assert re.search(r'redis_version=\d\.', log_msgs[0]), log_msgs 83 | assert log_msgs[0].endswith(' db_keys=2') 84 | 85 | 86 | def test_truncate(): 87 | assert arq.utils.truncate('123456', 4) == '123…' 88 | 89 | 90 | def test_args_to_string(): 91 | assert arq.utils.args_to_string((), {'d': 4}) == 'd=4' 92 | assert arq.utils.args_to_string((1, 2, 3), {}) == '1, 2, 3' 93 | assert arq.utils.args_to_string((1, 2, 3), {'d': 4}) == '1, 2, 3, d=4' 94 | 95 | 96 | @pytest.mark.parametrize( 97 | 'input,output', [(timedelta(days=1), 86_400_000), (42, 42000), (42.123, 42123), (42.123_987, 42124), (None, None)] 98 | ) 99 | def test_to_ms(input, output): 100 | assert arq.utils.to_ms(input) == output 101 | 102 | 103 | @pytest.mark.parametrize('input,output', [(timedelta(days=1), 86400), (42, 42), (42.123, 42.123), (None, None)]) 104 | def test_to_seconds(input, output): 105 | assert arq.utils.to_seconds(input) == output 106 | 107 | 108 | def test_typing(): 109 | assert 'OptionType' in arq.typing.__all__ 110 | 111 | 112 | def redis_settings_validation(): 113 | class Settings(BaseModel, arbitrary_types_allowed=True): 114 | redis_settings: RedisSettings 115 | 116 | @field_validator('redis_settings', mode='before') 117 | def parse_redis_settings(cls, v): 118 | if isinstance(v, str): 119 | return RedisSettings.from_dsn(v) 120 | else: 121 | return v 122 | 123 | s1 = Settings(redis_settings='redis://foobar:123/4') 124 | assert s1.redis_settings.host == 'foobar' 125 | assert s1.redis_settings.port == 123 126 | assert s1.redis_settings.database == 4 127 | assert s1.redis_settings.ssl is False 128 | 129 | s2 = Settings(redis_settings={'host': 'testing.com'}) 130 | assert s2.redis_settings.host == 'testing.com' 131 | assert s2.redis_settings.port == 6379 132 | 133 | with pytest.raises(ValueError, match='1 validation error for Settings\nredis_settings.ssl'): 134 | Settings(redis_settings={'ssl': 123}) 135 | 136 | s3 = Settings(redis_settings={'ssl': True}) 137 | assert s3.redis_settings.host == 'localhost' 138 | assert s3.redis_settings.ssl is True 139 | 140 | s4 = Settings(redis_settings='redis://user:pass@foobar') 141 | assert s4.redis_settings.host == 'foobar' 142 | assert s4.redis_settings.username == 'user' 143 | assert s4.redis_settings.password == 'pass' 144 | 145 | s5 = Settings(redis_settings={'unix_socket_path': '/tmp/redis.sock'}) 146 | assert s5.redis_settings.unix_socket_path == '/tmp/redis.sock' 147 | assert s5.redis_settings.database == 0 148 | 149 | s6 = Settings(redis_settings='unix:///tmp/redis.socket?db=6') 150 | assert s6.redis_settings.unix_socket_path == '/tmp/redis.socket' 151 | assert s6.redis_settings.database == 6 152 | 153 | 154 | def test_ms_to_datetime_tz(env: SetEnv): 155 | arq.utils.get_tz.cache_clear() 156 | env.set('ARQ_TIMEZONE', 'Asia/Shanghai') 157 | env.set('TIMEZONE', 'Europe/Berlin') # lower priority as per `timezone_keys` 158 | dt = arq.utils.ms_to_datetime(1_647_345_420_000) # 11.57 UTC 159 | assert dt.isoformat() == '2022-03-15T19:57:00+08:00' 160 | assert dt.tzinfo.zone == 'Asia/Shanghai' 161 | 162 | # should have no effect due to caching 163 | env.set('ARQ_TIMEZONE', 'Europe/Berlin') 164 | dt = arq.utils.ms_to_datetime(1_647_345_420_000) 165 | assert dt.isoformat() == '2022-03-15T19:57:00+08:00' 166 | 167 | 168 | def test_ms_to_datetime_no_tz(env: SetEnv): 169 | arq.utils.get_tz.cache_clear() 170 | dt = arq.utils.ms_to_datetime(1_647_345_420_000) # 11.57 UTC 171 | assert dt.isoformat() == '2022-03-15T11:57:00+00:00' 172 | 173 | # should have no effect due to caching 174 | env.set('ARQ_TIMEZONE', 'Europe/Berlin') 175 | dt = arq.utils.ms_to_datetime(1_647_345_420_000) 176 | assert dt.isoformat() == '2022-03-15T11:57:00+00:00' 177 | 178 | 179 | def test_ms_to_datetime_tz_invalid(env: SetEnv, caplog): 180 | arq.utils.get_tz.cache_clear() 181 | env.set('ARQ_TIMEZONE', 'foobar') 182 | caplog.set_level(logging.WARNING) 183 | dt = arq.utils.ms_to_datetime(1_647_345_420_000) 184 | assert dt.isoformat() == '2022-03-15T11:57:00+00:00' 185 | assert "unknown timezone: 'foobar'\n" in caplog.text 186 | 187 | 188 | def test_import_string_valid(): 189 | sqrt = arq.utils.import_string('math.sqrt') 190 | assert sqrt(4) == 2 191 | 192 | 193 | def test_import_string_invalid_short(): 194 | with pytest.raises(ImportError, match='"foobar" doesn\'t look like a module path'): 195 | arq.utils.import_string('foobar') 196 | 197 | 198 | def test_import_string_invalid_missing(): 199 | with pytest.raises(ImportError, match='Module "math" does not define a "foobar" attribute'): 200 | arq.utils.import_string('math.foobar') 201 | 202 | 203 | def test_settings_plain(): 204 | settings = RedisSettings() 205 | assert asdict(settings) == { 206 | 'host': 'localhost', 207 | 'port': 6379, 208 | 'unix_socket_path': None, 209 | 'database': 0, 210 | 'username': None, 211 | 'password': None, 212 | 'ssl': False, 213 | 'ssl_keyfile': None, 214 | 'ssl_certfile': None, 215 | 'ssl_cert_reqs': 'required', 216 | 'ssl_ca_certs': None, 217 | 'ssl_ca_data': None, 218 | 'ssl_check_hostname': False, 219 | 'conn_timeout': 1, 220 | 'conn_retries': 5, 221 | 'conn_retry_delay': 1, 222 | 'sentinel': False, 223 | 'sentinel_master': 'mymaster', 224 | 'retry_on_timeout': False, 225 | 'retry_on_error': None, 226 | 'retry': None, 227 | 'max_connections': None, 228 | } 229 | 230 | 231 | def test_settings_from_socket_dsn(): 232 | settings = RedisSettings.from_dsn('unix:///run/redis/redis.sock') 233 | # insert_assert(asdict(settings)) 234 | assert asdict(settings) == { 235 | 'host': 'localhost', 236 | 'port': 6379, 237 | 'unix_socket_path': '/run/redis/redis.sock', 238 | 'database': 0, 239 | 'username': None, 240 | 'password': None, 241 | 'ssl': False, 242 | 'ssl_keyfile': None, 243 | 'ssl_certfile': None, 244 | 'ssl_cert_reqs': 'required', 245 | 'ssl_ca_certs': None, 246 | 'ssl_ca_data': None, 247 | 'ssl_check_hostname': False, 248 | 'conn_timeout': 1, 249 | 'conn_retries': 5, 250 | 'conn_retry_delay': 1, 251 | 'sentinel': False, 252 | 'sentinel_master': 'mymaster', 253 | 'retry_on_timeout': False, 254 | 'retry_on_error': None, 255 | 'retry': None, 256 | 'max_connections': None, 257 | } 258 | --------------------------------------------------------------------------------