├── .bumpversion.cfg
├── .copier-answers.yml
├── .dockerignore
├── .drone.yml
├── .env
├── .github
    └── FUNDING.yml
├── .gitignore
├── .python-base-version
├── CHANGELOG.md
├── Dockerfile
├── Dockerfile-docs
├── LICENSE
├── README.md
├── README.pypi.md
├── config
    ├── mkdocs.yml
    ├── nginx.conf
    └── overrides
    │   ├── main.html
    │   └── partials
    │       └── comments.html
├── docker-compose.yml
├── docs
    ├── comments.md
    ├── page
    │   ├── examples.ipynb
    │   └── related.md
    ├── ppqueue.excalidraw
    └── reference
    │   ├── 01.md
    │   ├── 02.md
    │   ├── 03.md
    │   └── index.md
├── images.txt
├── pyproject.toml
├── src
    └── ppqueue
    │   ├── __init__.py
    │   ├── __version__.py
    │   ├── exceptions.py
    │   ├── job.py
    │   ├── plot.py
    │   ├── py.typed
    │   ├── queue.py
    │   └── utils.py
└── tests
    ├── __init__.py
    ├── common.py
    ├── test_plot.py
    ├── test_queue_processes.py
    ├── test_queue_threads.py
    └── test_utils.py


/.bumpversion.cfg:
--------------------------------------------------------------------------------
 1 | [bumpversion]
 2 | current_version = 0.4.0
 3 | commit = False
 4 | message = bump: {current_version} --> {new_version}
 5 | tag = False
 6 | tag_name = {current_version}
 7 | tag_message = 
 8 | parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?P<rc_kind>[A-Za-z]*)(?P<rc>\d*)
 9 | serialize = 
10 | 	{major}.{minor}.{patch}{rc_kind}{rc}
11 | 	{major}.{minor}.{patch}
12 | 
13 | [bumpversion:glob:src/*/__version__.py]
14 | 


--------------------------------------------------------------------------------
/.copier-answers.yml:
--------------------------------------------------------------------------------
 1 | # Changes here will be overwritten by Copier
 2 | _commit: 2c1aa3f
 3 | _src_path: https://codeberg.org/Fresh2dev/copier-f2dv-project.git
 4 | author_email: hello@f2dv.com
 5 | author_name: Donald Mellenbruch
 6 | ci_domain_name: lokalhost.net
 7 | docs_url: https://www.f2dv.com/r/ppqueue
 8 | funding_url: https://www.f2dv.com/fund
 9 | home_domain: f2dv.com
10 | home_page: https://www.f2dv.com
11 | is_minimal: false
12 | is_python: true
13 | license_type: MIT
14 | package_name: ppqueue
15 | project_description: A Parallel Process Queue for Python.
16 | project_name: ppqueue
17 | python_version: '3.8'
18 | repo_mirror: https://www.f2dv.com/r/ppqueue
19 | repo_name: fresh2dev/ppqueue
20 | repo_owner: fresh2dev
21 | repo_url: https://www.github.com/fresh2dev/ppqueue
22 | 
23 | 


--------------------------------------------------------------------------------
/.dockerignore:
--------------------------------------------------------------------------------
 1 | **/.git
 2 | 
 3 | **/secrets
 4 | !**/*.keep
 5 | **/*.tmp
 6 | **/.vscode
 7 | **/.DS_store
 8 | /build
 9 | **/.ipynb_checkpoints
10 | **/.coverage
11 | **/.hypothesis
12 | **/.tox
13 | **/.idea
14 | **/*.egg-info
15 | **/.mypy_cache
16 | **/__pycache__
17 | **/.pytest_cache
18 | 


--------------------------------------------------------------------------------
/.drone.yml:
--------------------------------------------------------------------------------
  1 | kind: template
  2 | load: hostbutter.jsonnet
  3 | data:
  4 |   domains: >-
  5 |     ["lokalhost.net", "fresh2.dev"]
  6 |   domainTriggers: >-
  7 |     {
  8 |       "lokalhost.net": {
  9 |         "event": {"exclude": ["promote"]}
 10 |       },
 11 |       "fresh2.dev": {
 12 |         "event": ["promote", "tag"]
 13 |       }
 14 |     }
 15 |   domainClusterMap: >-
 16 |     {}
 17 |   publishRegistries: >-
 18 |     {}
 19 |   secrets: >-
 20 |     []
 21 |   secretFiles: >-
 22 |     {
 23 |       "ENV_FILE": ".env"
 24 |     }
 25 |   volumes: >-
 26 |     []
 27 |   initSteps: >-
 28 |     []
 29 |   beforeSteps: >-
 30 |     [
 31 |       {
 32 |         "name": "py-test",
 33 |         "image": "registry.lokalhost.net/fresh2dev/mykefiles:a43f1d5",
 34 |         "environment": {
 35 |           "PIP_CONF": {"from_secret": "PIP_CONF"},
 36 |           "MYKE_MODULE": "mykefiles.python"
 37 |         },
 38 |         "commands": [
 39 |           "mkdir -p ~/.config/pip && echo \"$PIP_CONF\" > ~/.config/pip/pip.conf",
 40 |           "myke py-install",
 41 |           "myke py-reports",
 42 |           "myke py-build"
 43 |         ],
 44 |         "when": {}
 45 |       }
 46 |     ]
 47 |   afterSteps: >-
 48 |     []
 49 |   finalSteps: >-
 50 |     [
 51 |       {
 52 |         "name": "py-build-package",
 53 |         "image": "registry.lokalhost.net/fresh2dev/mykefiles:a43f1d5",
 54 |         "environment": {
 55 |           "PYPI_CREDS": {"from_secret": "PYPI_CREDS"},
 56 |           "PIP_CONF": {"from_secret": "PIP_CONF"},
 57 |           "MYKE_MODULE": "mykefiles.python"
 58 |         },
 59 |         "commands": [
 60 |           "echo \"$PYPI_CREDS\" > ~/.pypirc",
 61 |           "mkdir -p ~/.config/pip && echo \"$PIP_CONF\" > ~/.config/pip/pip.conf",
 62 |           "myke py-version-set --repository lokalhost",
 63 |           "myke py-build"
 64 |         ],
 65 |         "when": {}
 66 |       },
 67 |       {
 68 |         "name": "py-publish-sandbox",
 69 |         "image": "registry.lokalhost.net/fresh2dev/mykefiles:a43f1d5",
 70 |         "environment": {
 71 |           "PYPI_CREDS": {"from_secret": "PYPI_CREDS"},
 72 |           "PIP_CONF": {"from_secret": "PIP_CONF"},
 73 |           "TWINE_CERT": "/etc/ssl/certs/ca-certificates.crt",
 74 |           "MYKE_MODULE": "mykefiles.python"
 75 |         },
 76 |         "commands": [
 77 |           "echo \"$PYPI_CREDS\" > ~/.pypirc",
 78 |           "mkdir -p ~/.config/pip && echo \"$PIP_CONF\" > ~/.config/pip/pip.conf",
 79 |           "myke py-publish --repository lokalhost"
 80 |         ],
 81 |         "when": {}
 82 |       },
 83 |       {
 84 |         "name": "py-publish-dev",
 85 |         "image": "registry.lokalhost.net/fresh2dev/mykefiles:a43f1d5",
 86 |         "environment": {
 87 |           "PYPI_CREDS": {"from_secret": "PYPI_CREDS"},
 88 |           "PIP_CONF": {"from_secret": "PIP_CONF"},
 89 |           "TWINE_CERT": "/etc/ssl/certs/ca-certificates.crt",
 90 |           "MYKE_MODULE": "mykefiles.python"
 91 |         },
 92 |         "commands": [
 93 |           "echo \"$PYPI_CREDS\" > ~/.pypirc",
 94 |           "mkdir -p ~/.config/pip && echo \"$PIP_CONF\" > ~/.config/pip/pip.conf",
 95 |           "myke py-publish --repository codeberg"
 96 |         ],
 97 |         "when": {
 98 |           "ref": ["refs/heads/dev", "refs/heads/main", "refs/tags/*"]
 99 |         }
100 |       },
101 |       {
102 |         "name": "py-publish-test",
103 |         "image": "registry.lokalhost.net/fresh2dev/mykefiles:a43f1d5",
104 |         "environment": {
105 |           "PYPI_CREDS": {"from_secret": "PYPI_CREDS"},
106 |           "PIP_CONF": {"from_secret": "PIP_CONF"},
107 |           "TWINE_CERT": "/etc/ssl/certs/ca-certificates.crt",
108 |           "MYKE_MODULE": "mykefiles.python"
109 |         },
110 |         "commands": [
111 |           "echo \"$PYPI_CREDS\" > ~/.pypirc",
112 |           "mkdir -p ~/.config/pip && echo \"$PIP_CONF\" > ~/.config/pip/pip.conf",
113 |           "myke py-publish --repository testpypi"
114 |         ],
115 |         "when": {
116 |           "ref": ["refs/tags/*"]
117 |         }
118 |       },
119 |       {
120 |         "name": "py-publish-prod",
121 |         "image": "registry.lokalhost.net/fresh2dev/mykefiles:a43f1d5",
122 |         "environment": {
123 |           "PYPI_CREDS": {"from_secret": "PYPI_CREDS"},
124 |           "PIP_CONF": {"from_secret": "PIP_CONF"},
125 |           "TWINE_CERT": "/etc/ssl/certs/ca-certificates.crt",
126 |           "MYKE_MODULE": "mykefiles.python"
127 |         },
128 |         "commands": [
129 |           "echo \"$PYPI_CREDS\" > ~/.pypirc",
130 |           "mkdir -p ~/.config/pip && echo \"$PIP_CONF\" > ~/.config/pip/pip.conf",
131 |           "myke py-publish --repository pypi"
132 |         ],
133 |         "when": {
134 |           "ref": ["refs/tags/*"]
135 |         }
136 |       }
137 |     ]
138 |   extraObjects: >-
139 |     [
140 |       {
141 |         "kind": "secret",
142 |         "name": "PYPI_CREDS",
143 |         "get": {
144 |           "path": "secret/data/hostbutter/global",
145 |           "name": "PYPI_CREDS"
146 |         }
147 |       },
148 |       {
149 |         "kind": "secret",
150 |         "name": "PIP_CONF",
151 |         "get": {
152 |           "path": "secret/data/hostbutter/global",
153 |           "name": "PIP_CONF"
154 |         }
155 |       }
156 |     ]
157 | 


--------------------------------------------------------------------------------
/.env:
--------------------------------------------------------------------------------
 1 | ### STACK NAME
 2 | COMPOSE_PROJECT_NAME=ppqueue
 3 | 
 4 | ### GLOBAL VARIABLES
 5 | HB_DOMAIN=
 6 | HB_PROXY_NETWORK=
 7 | HB_NFS_OPTS=
 8 | HB_IMAGE_REGISTRY=
 9 | 
10 | ### PROJECT VARIABLES
11 | 
12 | ## APP
13 | APP_SVC_ID=app
14 | APP_SUBDOMAIN=ppqueue.
15 | APP_ROUTE_PREFIX=
16 | # APP_STRIP_PREFIX=
17 | APP_HTTP_PORT=80
18 | APP_PLACEMENT_LABEL=frontend
19 | 
20 | PYTHONPATH=src
21 | 


--------------------------------------------------------------------------------
/.github/FUNDING.yml:
--------------------------------------------------------------------------------
1 | custom: ["https://www.f2dv.com/fund", "https://www.f2dv.com/paypal"]
2 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | **/secrets
 2 | !**/*.keep
 3 | **/*.tmp
 4 | **/.vscode
 5 | **/.DS_store
 6 | .python-version
 7 | 
 8 | /public
 9 | 
10 | /requirements.txt
11 | /dist
12 | /build
13 | 
14 | *.pyc
15 | **/.ipynb_checkpoints
16 | **/.coverage
17 | **/.hypothesis
18 | **/.tox
19 | **/.idea
20 | **/*.egg-info
21 | **/.mypy_cache
22 | **/__pycache__
23 | **/.pytest_cache
24 | **/.ropeproject
25 | **/.ruff_cache
26 | 


--------------------------------------------------------------------------------
/.python-base-version:
--------------------------------------------------------------------------------
1 | 3.8.16
2 | 


--------------------------------------------------------------------------------
/CHANGELOG.md:
--------------------------------------------------------------------------------
 1 | # Changelog
 2 | 
 3 | ## 0.4.0 - 2023-08-04
 4 | 
 5 | ### :point_right: Changes
 6 | 
 7 | - *Breaking:* Default engine changed from `threading.Thread` to `mp.Process` [a3dc1af]
 8 | 
 9 | ### :metal: Other
10 | 
11 | - Correct annotations and docstrings [7a2470a]
12 | 
13 | ## 0.3.0 - 2023-06-22
14 | 
15 | - rename from `ezpq` to `ppqueue`
16 | - replaced `queue.get` with `queue.dequeue` / `queue.pop`
17 | - `queue.put` is an alias for `queue.enqueue`
18 | - renamed `queue.empty` to `queue.is_empty`
19 | - renamed `queue.full` to `queue.is_full`
20 | - renamed `queue.count_queues` to `queue.sizes`
21 | - replaced `ezpq.FinishedJob` with `ezpq.Job`
22 | - `job.output` is now `job.result`
23 | - `job.callback` is now `job.callback_result`
24 | - wrapped `Plot(...).build(...)` into just `plot_jobs(...)`
25 | - use of enums to constrain plot parameter choices
26 | - bump dependencies
27 | 
28 | ## v0.2.1
29 | 
30 | ### Added
31 | 
32 | - `clear_waiting()` function to clear the waiting queue.
33 | - Added `stop_on_lane_error` parameter to `ezpq.Job` to allow for short-circuiting a synchronous lane if a job in the lane fails. When set to `True` and the preceding job has a non-zero exit code, this job will not be run. Note that this is to be set per-job for flexibility.
34 | - Additional unit tests.
35 | 
36 | ### Changed
37 | 
38 | - `stop_all()` function now clears the waiting queue and terminate running jobs. This addresses a bug where a queue would fail to close when disposing with jobs still in the waiting queue.
39 | - The default `poll` for the queue itself is still `0.1`. Now, the default `poll` for `get` and `wait` is equal to the `poll` for the queue itself, as it makes no sense to check for changes more freqeuntly than changes could arise.
40 | 
41 | ### Removed
42 | 
43 | - Removed functions `has_waiting`, `has_work`, and `has_completed`. Use `size(...)` for this.
44 | - Renamed `Queue.is_started` to `Queue.is_running`.
45 | 
46 | ## v0.2.0
47 | 
48 | ### Added
49 | 
50 | - Added `map()` function to `ezpq.Queue`.
51 | - Integration with tqdm package for progress bars.
52 | - Added `lane` property to `ezpq.Job`. When set, jobs in the same lane are processing sequentially.
53 | - Added `exception` property to `ezpq.Job`. Stores the exception of a failed job.
54 | - Added `suppress_errors` parameter to `ezpq.Job`.
55 | - Added `facet_by`, `theme`, and `color_pal` parameters to `ezpq.Plot.build()`.
56 | - Added `qid` (queue id) to job data output. This can be used to facet an `ezpq.Plot`.
57 | - Unit tests.
58 | 
59 | ### Changed
60 | 
61 | - Removed parameter `n` from the `wait()` function of `ezpq.Queue`; `wait()` will halt execution until all jobs in the queue are processed.
62 | - Moved `ezpq.Plot()` plotting parameters to `ezpq.Plot.build()`.
63 | - Reordered `ezpq.Queue()` init parameters.
64 | 
65 | ## v0.1.0
66 | 
67 | - Initial release
68 | 


--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
 1 | ARG HB_IMAGE_REGISTRY=docker.io
 2 | FROM ${HB_IMAGE_REGISTRY}/python:3.10.10-slim-bullseye as build
 3 | LABEL org.opencontainers.image.source=https://www.github.com/fresh2dev/ppqueue
 4 | LABEL org.opencontainers.image.description="A Parallel Process Queue for Python."
 5 | LABEL org.opencontainers.image.licenses=MIT
 6 | RUN apt-get update && apt-get install --upgrade -y build-essential git
 7 | WORKDIR /app
 8 | ENV PYTHONUNBUFFERED=1
 9 | RUN python3 -m venv /app/venv
10 | ENV PATH="/app/venv/bin:$PATH"
11 | RUN python3 -m pip install --no-cache-dir --upgrade pip
12 | COPY ./dist /dist
13 | RUN find /dist -name "*.whl" -exec \
14 |     pip install --no-cache-dir \
15 |         --extra-index-url "https://codeberg.org/api/packages/Fresh2dev/pypi/simple" \
16 |         "{}" \; \
17 |     && pip show "ppqueue"
18 | 
19 | FROM ${HB_IMAGE_REGISTRY}/python:3.10.10-slim-bullseye
20 | COPY --from=build /app/venv /app/venv
21 | COPY --from=build /usr/local/bin /usr/local/bin
22 | ENV PATH="/app/venv/bin:$PATH"
23 | ENTRYPOINT ["ppqueue"]
24 | WORKDIR /workspace
25 | 


--------------------------------------------------------------------------------
/Dockerfile-docs:
--------------------------------------------------------------------------------
1 | ARG HB_IMAGE_REGISTRY=docker.io
2 | FROM ${HB_IMAGE_REGISTRY}/nginx:1
3 | LABEL org.opencontainers.image.source=https://www.github.com/fresh2dev/ppqueue
4 | LABEL org.opencontainers.image.description="A Parallel Process Queue for Python."
5 | LABEL org.opencontainers.image.licenses=MIT
6 | ARG CONTENT_PATH=public
7 | COPY $CONTENT_PATH /usr/share/nginx/html
8 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Copyright 2023 Donald Mellenbruch
2 | 
3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
4 | 
5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
6 | 
7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
8 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # ppqueue
 2 | 
 3 | > A Parallel Process Queue for Python.
 4 | 
 5 | | Links         |                                          |
 6 | |---------------|------------------------------------------|
 7 | | Code Repo     | https://www.github.com/fresh2dev/ppqueue |
 8 | | Documentation | https://www.f2dv.com/r/ppqueue           |
 9 | | Changelog     | https://www.f2dv.com/r/ppqueue/changelog |
10 | | License       | https://www.f2dv.com/r/ppqueue/license   |
11 | | Funding       | https://www.f2dv.com/fund                |
12 | 
13 | [![GitHub Repo stars](https://img.shields.io/github/stars/fresh2dev/ppqueue?color=blue&style=for-the-badge)](https://star-history.com/#fresh2dev/ppqueue&Date)
14 | [![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/fresh2dev/ppqueue?color=blue&style=for-the-badge)](https://www.f2dv.com/r/ppqueue/changelog)
15 | [![GitHub Release Date](https://img.shields.io/github/release-date/fresh2dev/ppqueue?color=blue&style=for-the-badge)](https://www.f2dv.com/r/ppqueue/changelog)
16 | [![License](https://img.shields.io/github/license/fresh2dev/ppqueue?color=blue&style=for-the-badge)](https://www.f2dv.com/r/ppqueue/license)
17 | <!-- [![GitHub issues](https://img.shields.io/github/issues-raw/fresh2dev/ppqueue?color=blue&style=for-the-badge)](https://www.github.com/fresh2dev/ppqueue/issues) -->
18 | <!-- [![GitHub pull requests](https://img.shields.io/github/issues-pr-raw/fresh2dev/ppqueue?color=blue&style=for-the-badge)](https://www.github.com/fresh2dev/ppqueue/pulls) -->
19 | <!-- [![PyPI - Downloads](https://img.shields.io/pypi/dm/ppqueue?color=blue&style=for-the-badge)](https://pypi.org/project/ppqueue) -->
20 | <!-- [![Docker Pulls](https://img.shields.io/docker/pulls/fresh2dev/ppqueue?color=blue&style=for-the-badge)](https://hub.docker.com/r/fresh2dev/ppqueue) -->
21 | <!-- [![Funding](https://img.shields.io/badge/funding-%24%24%24-blue?style=for-the-badge)](https://www.f2dv.com/fund) -->
22 | 
23 | ---
24 | 
25 | ## Overview
26 | 
27 | `ppqueue` is a Python module that serves as an abstraction layer to both `multiprocessing.Process` and `threading.Thread`. I built `ppqueue` because I too often notice that parallelizing code results in *ugly* code. With this simple Queue, you can parallelize code easily and attractively. ppqueue offers:
28 | 
29 | - a single API for parallel execution using processes or threads.
30 | - FIFO priority queueing.
31 | - Gantt charts of job execution (thanks `plotnine` + `pandas`)
32 | - progress bars (thanks to `tqdm`)
33 | 
34 | ![](https://img.fresh2.dev/1687407526_84b23a13b5f.svg)
35 | 
36 | ## Install
37 | 
38 | Install from PyPi:
39 | 
40 | ```python
41 | pip install ppqueue[plot]
42 | ```
43 | 
44 | ## Examples
45 | 
46 | An notebook of examples is available at:
47 | 
48 | https://www.f2dv.com/r/ppqueue/page/examples/
49 | 
50 | And more examples are provided in the reference docs:
51 | 
52 | https://www.f2dv.com/r/ppqueue/reference/
53 | 
54 | ## Support
55 | 
56 | If this project delivers value to you, please [provide feedback](https://www.github.com/fresh2dev/ppqueue/issues), code contributions, and/or [funding](https://www.f2dv.com/fund).
57 | 
58 | ---
59 | 
60 | *Brought to you by...*
61 | 
62 | <a href="https://www.f2dv.com"><img src="https://img.fresh2.dev/fresh2dev.svg" style="filter: invert(50%);"></img></a>
63 | 


--------------------------------------------------------------------------------
/README.pypi.md:
--------------------------------------------------------------------------------
1 | [https://www.f2dv.com/r/ppqueue](https://www.f2dv.com/r/ppqueue)
2 | 


--------------------------------------------------------------------------------
/config/mkdocs.yml:
--------------------------------------------------------------------------------
  1 | site_name: "ppqueue"
  2 | site_description: "ppqueue Docs"
  3 | site_url: !ENV [MKDOCS_SITE_URL, "https://www.f2dv.com/r/ppqueue"]
  4 | repo_url: "https://www.github.com/fresh2dev/ppqueue"
  5 | repo_name: "fresh2dev/ppqueue"
  6 | edit_uri: "edit/main/docs"
  7 | #edit_uri_template: "src/branch/main/docs/{path}"
  8 | docs_dir: "../docs"
  9 | site_dir: "../public"
 10 | 
 11 | copyright: '<a href="https://www.fresh2.dev"><img src="https://img.fresh2.dev/fresh2dev.svg" style="height: 3em; margin-right: 1em; filter: invert(100%);"></img></a> &copy; 2023 <a rel="license" target="_blank" href="http://creativecommons.org/licenses/by-sa/4.0/"><img alt="Creative Commons License" style="border-width:0; margin-left: 1em;" src="https://i.creativecommons.org/l/by-sa/4.0/88x31.png" /></a>'
 12 | extra:
 13 |   generator: false
 14 |   social: []
 15 |     # - name: Fresh2.dev
 16 |     #   icon: "material/home"
 17 |     #   link: 'https://www.fresh2.dev'
 18 | 
 19 | ## https://www.mkdocs.org/user-guide/writing-your-docs/#configure-pages-and-navigation
 20 | nav:
 21 |   - Home: index.md
 22 |   - Articles:
 23 |       - Examples: 'page/examples.ipynb'
 24 |       - 'page/related.md'
 25 |   - Reference: reference/
 26 |   - Changelog: changelog.md
 27 |   - License: license.md
 28 |   - Discussion: comments.md
 29 |   - '<- f2dv.com':
 30 |     - '/home': 'https://www.f2dv.com'
 31 |     - '/code': 'https://www.f2dv.com/projects'
 32 |     - '/fund': 'https://www.f2dv.com/fund'
 33 | 
 34 | watch:
 35 |   - ../src
 36 | 
 37 | theme:
 38 |   name: material
 39 |   language: en
 40 |   custom_dir: overrides
 41 |   palette:
 42 |     # https://squidfunk.github.io/mkdocs-material/setup/changing-the-colors/#primary-color
 43 |     - scheme: slate
 44 |       primary: green
 45 |       toggle:
 46 |         icon: material/toggle-switch-off-outline
 47 |         name: Switch to light mode
 48 |     - scheme: default
 49 |       primary: green
 50 |       toggle:
 51 |         icon: material/toggle-switch
 52 |         name: Switch to dark mode
 53 |   features:
 54 |     - content.code.copy
 55 |     - content.code.annotate
 56 |     - navigation.instant
 57 |     - navigation.indexes
 58 |     - navigation.expand
 59 |     - navigation.top
 60 |     - announce.dismiss
 61 |     # - navigation.footer
 62 | 
 63 | # extra_css:
 64 | #   - css/extra.css
 65 | 
 66 | markdown_extensions:
 67 |   - toc:
 68 |       permalink: true
 69 |       toc_depth: 3
 70 |   - attr_list
 71 |   - pymdownx.emoji:
 72 |       emoji_index: !!python/name:materialx.emoji.twemoji
 73 |       emoji_generator: !!python/name:materialx.emoji.to_svg
 74 |   - pymdownx.tasklist:
 75 |       custom_checkbox: true
 76 |   - pymdownx.magiclink:
 77 |       hide_protocol: true
 78 |       repo_url_shortener: false
 79 |       social_url_shortener: false
 80 |   - pymdownx.saneheaders
 81 |   - pymdownx.highlight
 82 |   - pymdownx.superfences
 83 |   - pymdownx.caret
 84 |   - pymdownx.keys
 85 |   - pymdownx.mark
 86 |   - pymdownx.tilde
 87 | 
 88 | plugins:
 89 |   - search
 90 |   - include_dir_to_nav
 91 |   - autorefs
 92 |   - mkdocs-jupyter:
 93 |       ignore_h1_titles: True
 94 |       include_source: True
 95 |       execute: false
 96 |       remove_tag_config:
 97 |         remove_input_tags:
 98 |           - hidden
 99 |   - mkdocstrings:
100 |       default_handler: python
101 |       handlers:
102 |         python:
103 |           paths: ["src"]
104 |           options:
105 |             show_bases: false
106 |             show_source: false
107 |             members_order: 'alphabetical'  # 'source'
108 |             docstring_style: google
109 |             show_submodules: false
110 |             show_signature: false
111 |             merge_init_into_class: true
112 |             show_signature_annotations: false
113 |             separate_signature: false
114 |             show_if_no_docstring: false
115 |             show_root_heading: false
116 |             show_root_toc_entry: false
117 |             show_root_full_path: false
118 |             show_root_members_full_path: false
119 |             show_object_full_path: false
120 |             show_category_heading: true
121 | 


--------------------------------------------------------------------------------
/config/nginx.conf:
--------------------------------------------------------------------------------
1 | server {
2 |   listen  80;
3 | 
4 |   location /{{ env "HOST_PREFIX" }} {
5 |     alias /usr/share/nginx/html/;
6 |   }
7 | }
8 | 


--------------------------------------------------------------------------------
/config/overrides/main.html:
--------------------------------------------------------------------------------
 1 | {% extends "base.html" %}
 2 | 
 3 | {% raw %}
 4 | <!-- {% block announce %} -->
 5 | <!-- <p>Hello world</p> -->
 6 | <!-- {% endblock %} -->
 7 | {% endraw %}
 8 | 
 9 | {% block fonts %}
10 | <style>
11 | @import url("https://www.fresh2.dev/css/fonts.css");
12 | 
13 | :root {
14 |   --md-text-font: iAWriterQuattroS, -apple-system, BlinkMacSystemFont, avenir next, avenir, segoe ui, helvetica neue, helvetica, Ubuntu, roboto, noto, arial, sans-serif;
15 |   --md-code-font: iAWriterMonoS, Menlo, Consolas, Monaco, Liberation Mono, Lucida Console, monospace;
16 | }
17 | 
18 | /* hide jupyter notebooks input/output numbers */
19 | .jp-InputPrompt {
20 |   display: none !important;
21 | }
22 | 
23 | .jp-OutputPrompt {
24 |   display: none !important;
25 | }
26 | 
27 | /* hide jupyter notebooks tags */
28 | .jupyter-wrapper .celltoolbar {
29 |   display: none !important;
30 | }
31 | 
32 | </style>
33 | {% endblock %}
34 | 
35 | {% block analytics %}
36 | <script>
37 |     window.goatcounter = {
38 |         path: function(p) {
39 |             if (window.location.host.toLowerCase().endsWith(".fresh2.dev"))
40 |                 return p
41 |             window.goatcounter = {no_onload: true}
42 |             return null
43 |         }
44 |     }
45 | </script>
46 | <script data-goatcounter="https://stats.fresh2.dev/count"
47 |         async src="//stats.fresh2.dev/count.js"></script>
48 | {% endblock %}
49 | 
50 | {% block content %}
51 | {% if page.nb_url %}
52 |     <a href="{{ page.nb_url }}" title="Download Notebook" class="md-content__button md-icon">
53 |         {% include ".icons/material/download.svg" %}
54 |     </a>
55 | {% endif %}
56 | 
57 | {{ super() }}
58 | {% endblock content %}
59 | 


--------------------------------------------------------------------------------
/config/overrides/partials/comments.html:
--------------------------------------------------------------------------------
1 | {% if page.meta.comments %}
2 | <script defer src="https://comment.fresh2.dev/js/commento.js"></script>
3 | <div id="commento"></div>
4 | {% endif %}
5 | 


--------------------------------------------------------------------------------
/docker-compose.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | version: "3.9"
 3 | 
 4 | networks:
 5 |   proxy:
 6 |     name: ${HB_PROXY_NETWORK}
 7 |     external: true
 8 | 
 9 | services:
10 |   app:
11 |     image: ${HB_IMAGE_REGISTRY:?}/${HB_IMAGE_OWNER:?}/${COMPOSE_PROJECT_NAME:?}-docs:${HB_IMAGE_TAG:?}
12 |     networks:
13 |       - proxy
14 |     healthcheck:
15 |       disable: true
16 |     deploy:
17 |       # mode: global
18 |       mode: replicated
19 |       replicas: ${APP_REPLICAS:-1}
20 |       placement:
21 |         preferences: []
22 |         max_replicas_per_node: ${APP_REPLICAS_PER_NODE:-${APP_REPLICAS:-1}}
23 |         constraints:
24 |           - "node.labels.${APP_PLACEMENT_LABEL:-frontend} == true"
25 |           # - "node.role == manager"
26 |           # - "node.platform.arch==x86_64"
27 |           # - "node.platform.arch==armv7l"
28 |       update_config:
29 |         parallelism: 1
30 |         delay: 5s
31 |         monitor: 5s
32 |         failure_action: pause
33 |         order: stop-first
34 |       resources:
35 |         reservations:
36 |           cpus: "0.01"
37 |           memory: "16M"
38 |       #   limits:
39 |       #     cpus: "0.50"
40 |       #     memory: "512M"
41 |       labels:
42 |         # enable traefik
43 |         - "traefik.enable=true"
44 |         - "traefik.docker.network=${HB_PROXY_NETWORK:?}"
45 |         # define load balancer.
46 |         - "traefik.http.services.${COMPOSE_PROJECT_NAME:?}_${APP_SVC_ID:?}.loadbalancer.server.port=${APP_HTTP_PORT:?}"
47 |         - "traefik.http.services.${COMPOSE_PROJECT_NAME:?}_${APP_SVC_ID:?}.loadbalancer.server.scheme=${APP_HTTP_PROTOCOL:-http}"
48 |         - "traefik.http.services.${COMPOSE_PROJECT_NAME:?}_${APP_SVC_ID:?}.loadbalancer.sticky=${APP_STICKY_LB:-false}"
49 |         - "traefik.http.services.${COMPOSE_PROJECT_NAME:?}_${APP_SVC_ID:?}.loadbalancer.sticky.cookie.secure=${APP_STICKY_LB:-false}"
50 |         - "traefik.http.services.${COMPOSE_PROJECT_NAME:?}_${APP_SVC_ID:?}.loadbalancer.sticky.cookie.name=StickySessionCookie_${COMPOSE_PROJECT_NAME:?}_${APP_SVC_ID:?}"
51 |         # define https entrypoint.
52 |         - "traefik.http.routers.${COMPOSE_PROJECT_NAME:?}_${APP_SVC_ID:?}.entrypoints=websecure"
53 |         - "traefik.http.routers.${COMPOSE_PROJECT_NAME:?}_${APP_SVC_ID:?}.rule=(Host(`${APP_SUBDOMAIN}${HB_DOMAIN:?}`) && PathPrefix(`/${APP_ROUTE_PREFIX}`))${APP_EXTRA_ROUTES:-}"
54 |         - "traefik.http.routers.${COMPOSE_PROJECT_NAME:?}_${APP_SVC_ID:?}.tls=true"
55 |         # - "traefik.http.routers.${COMPOSE_PROJECT_NAME:?}_${APP_SVC_ID:?}.tls.certresolver=letsencrypt"
56 |         # strip prefixes.
57 |         # - "traefik.http.middlewares.${COMPOSE_PROJECT_NAME:?}_${APP_SVC_ID:?}_stripprefix.stripprefix.prefixes=/${APP_STRIP_PREFIX}"
58 |         # set middlewares.
59 |         - "traefik.http.routers.${COMPOSE_PROJECT_NAME:?}_${APP_SVC_ID:?}.middlewares=${APP_MIDDLEWARE-secured-auth}"  #",${COMPOSE_PROJECT_NAME:?}_${APP_SVC_ID:?}_stripprefix"
60 |     volumes:
61 |       - /etc/ssl/certs/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt:ro
62 |     configs:
63 |       - source: app-config
64 |         target: /etc/nginx/conf.d/default.conf
65 |     environment:
66 |       HOST_PREFIX: $APP_ROUTE_PREFIX
67 | 
68 | configs:
69 |   app-config:
70 |     name: ${COMPOSE_PROJECT_NAME:?}_app-config_01
71 |     file: ./config/nginx.conf
72 |     template_driver: golang
73 | 


--------------------------------------------------------------------------------
/docs/comments.md:
--------------------------------------------------------------------------------
1 | ---
2 | comments: true
3 | ---
4 | 


--------------------------------------------------------------------------------
/docs/page/related.md:
--------------------------------------------------------------------------------
1 | # Related Projects
2 | 


--------------------------------------------------------------------------------
/docs/reference/01.md:
--------------------------------------------------------------------------------
1 | # ppqueue.Queue
2 | ::: ppqueue.Queue
3 | 


--------------------------------------------------------------------------------
/docs/reference/02.md:
--------------------------------------------------------------------------------
1 | # ppqueue.Job
2 | ::: ppqueue.Job
3 | 


--------------------------------------------------------------------------------
/docs/reference/03.md:
--------------------------------------------------------------------------------
1 | # ppqueue.plot
2 | ::: ppqueue.plot
3 | 


--------------------------------------------------------------------------------
/docs/reference/index.md:
--------------------------------------------------------------------------------
1 | ---
2 | title: null
3 | ---
4 | 
5 | ## [Go to Reference](./01)
6 | 


--------------------------------------------------------------------------------
/images.txt:
--------------------------------------------------------------------------------
1 | # public images to pull into local registry during ci; fully-qualified.
2 | # e.g., <registry>/<owner>/<image> -> docker.io/library/alpine:latest
3 | 
4 | docker.io/python:3.10.10-slim-bullseye
5 | docker.io/nginx:1
6 | 


--------------------------------------------------------------------------------
/pyproject.toml:
--------------------------------------------------------------------------------
  1 | [build-system]
  2 | requires = ["setuptools>=64", "build==0.*", "wheel==0.*", "twine==4.*"]
  3 | build-backend = "setuptools.build_meta"
  4 | 
  5 | [project]
  6 | name = "ppqueue"
  7 | authors = [
  8 |     {name = "Donald Mellenbruch", email = "hello@f2dv.com"},
  9 | ]
 10 | description = "A Parallel Process Queue for Python."
 11 | readme = "README.pypi.md"
 12 | license = {file = "LICENSE"}
 13 | requires-python = ">=3.8"
 14 | classifiers = [
 15 |     "Programming Language :: Python :: 3",
 16 | ]
 17 | dynamic = ["version"]
 18 | dependencies = [
 19 |     "typing-extensions; python_version<'3.10'",
 20 |     "tqdm==4.*",
 21 | ]
 22 | 
 23 | [project.optional-dependencies]
 24 | dev = [
 25 |     "python-lsp-server[rope]==1.*",
 26 |     "pylint==2.*",
 27 |     "pylint-pytest==1.*",
 28 |     "mypy[reports]==1.*",
 29 |     "ruff==0.*",
 30 |     "black[jupyter]==23.*",
 31 |     "isort==5.*",
 32 |     "bump2version==1.*",
 33 |     "pdbpp",
 34 |     # python -m ipykernel install --user --name ppqueue
 35 |     "ipykernel",
 36 |     "ipywidgets",
 37 |     "ipython",
 38 | ]
 39 | docs = [
 40 |     "mkdocs==1.*",
 41 |     "mkdocs-material==9.*",
 42 |     "mkdocs-jupyter==0.24.*",
 43 |     "mkdocstrings[python]==0.20.*",
 44 |     "mkdocs-autorefs==0.*",
 45 |     "mkdocs-include-dir-to-nav==1.*",
 46 | ]
 47 | tests = [
 48 |     "pytest==7.*",
 49 |     "pytest-cov==4.*",
 50 |     "pytest-html==3.*",
 51 |     "pytest-sugar==0.*",
 52 |     "pytest-custom-exit-code==0.3.*",
 53 |     "pylint==2.*",
 54 |     "pylint-pytest==1.*",
 55 |     "packaging==23.*",
 56 |     #
 57 |     "mockish==0.1.*",
 58 |     "tqdm==4.*",
 59 |     "pandas>=1,<=2",
 60 |     "plotnine==0.*",
 61 | ]
 62 | plot = [
 63 |     "pandas>=1,<=2",
 64 |     "plotnine==0.*",
 65 | ]
 66 | extras = [
 67 |     "pandas>=1,<=2",
 68 |     "plotnine==0.*",
 69 | ]
 70 | 
 71 | [project.urls]
 72 | Homepage = "https://www.f2dv.com/r/ppqueue"
 73 | Repository = "https://www.github.com/fresh2dev/ppqueue"
 74 | Funding = "https://www.f2dv.com/fund"
 75 | 
 76 | [project.scripts]
 77 | ppqueue = "ppqueue.__main__:main"
 78 | 
 79 | [tool.setuptools.package-data]
 80 | "*" = ["**"]
 81 | [tool.setuptools.packages.find]
 82 | where = ["src"]
 83 | include = ["*"]
 84 | exclude = []
 85 | namespaces = false
 86 | 
 87 | [tool.setuptools.dynamic]
 88 | version = {attr = "ppqueue.__version__.__version__"}
 89 | 
 90 | [tool.pytest.ini_options]
 91 | minversion = 7.0
 92 | testpaths = ["tests"]
 93 | 
 94 | [tool.pylint.MASTER]
 95 | ignore-paths = "^(?!src|tests).*$"
 96 | load-plugins = ["pylint_pytest"]
 97 | extension-pkg-whitelist = ["pydantic"]
 98 | [tool.pylint.messages_control]
 99 | max-line-length = 88
100 | disable = [
101 |     "fixme",
102 |     "invalid-name",
103 |     "unnecessary-pass",
104 |     "unnecessary-ellipsis",
105 |     "too-few-public-methods",
106 |     "import-outside-toplevel",
107 |     "missing-module-docstring",
108 | ]
109 | 
110 | [tool.mypy]
111 | mypy_path = "$MYPY_CONFIG_FILE_DIR/src"
112 | files = "src/**/*.py"
113 | plugins = ["pydantic.mypy"]
114 | namespace_packages = true
115 | explicit_package_bases = true
116 | strict = true
117 | [tool.pydantic-mypy]
118 | init_forbid_extra = true
119 | init_typed = true
120 | warn_required_dynamic_aliases = true
121 | warn_untyped_fields = true
122 | 
123 | [tool.black]
124 | line-length = 88
125 | include = 'src\/.*\.pyi?$|tests\/.*\.pyi?|docs\/.*\.ipynb$'
126 | 
127 | [tool.isort]
128 | profile = "black"
129 | line_length = 88
130 | src_paths = ["src", "tests"]
131 | float_to_top = true
132 | include_trailing_comma = true
133 | honor_noqa = true
134 | quiet = true
135 | 
136 | [tool.ruff]
137 | select = [
138 |     "E",  # pycodestyle errors
139 |     "W",  # pycodestyle warnings
140 |     # "D",  # pydocstyle
141 |     "F",  # pyflakes
142 |     "UP", # pyupgrade
143 |     # "I",  # isort (missing-required-import)
144 |     "C4",  # flake8-comprehensions
145 |     "B",  # flake8-bugbear
146 |     "BLE",  # flake8-blind-except
147 |     "DTZ",  # flake8-datetimez
148 |     "EM",  # flake8-errmsg
149 |     "ISC",  # flake8-implicit-str-concat
150 |     "G",  # flake8-logging-format
151 |     "PIE",  # flake8-pie
152 |     "RSE",  # flake8-raise
153 |     # "ANN",# flake8-annotations
154 |     "A",# flake8-builtins
155 |     "COM",# flake8-commas
156 |     "PT",# flake8-pytest-style
157 |     "Q",# flake8-quotes
158 |     "RET",# flake8-return
159 |     "SIM",# flake8-simplify
160 |     "ARG",# flake8-unused-arguments
161 |     "PTH",# flake8-use-pathlib
162 |     "ERA",# eradicate
163 |     "PLW",# pylint-specific warnings
164 |     "PLE",# pylint-specific errors
165 |     "PLR",# pylint-specific refactors
166 |     "PLC",# pylint-specific conventions
167 |     # "RUF",# ruff-specific
168 |     "TRY",# tryceratops
169 | ]
170 | ignore = [
171 |     # "E501",  # line too long, handled by black
172 |     "B008",  # do not perform function calls in argument defaults
173 | ]
174 | 
175 | # Avoid trying to fix flake8-bugbear (`B`) violations.
176 | # Allow autofix for all enabled rules (when `--fix`) is provided.
177 | # autofix F401=unused-imports
178 | fixable = ["W", "E", "COM", "F401"]
179 | unfixable = []
180 | 
181 | respect-gitignore = true
182 | 
183 | # Same as Black.
184 | line-length = 88
185 | 
186 | target-version = "py37"
187 | 
188 | # Allow unused variables when underscore-prefixed.
189 | dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
190 | 
191 | # Ignore `E402` (import violations) in all `__init__.py` files, and in `path/to/file.py`.
192 | [tool.ruff.per-file-ignores]
193 | "__init__.py" = ["E402", "F401"]
194 | 
195 | [tool.tox]
196 | legacy_tox_ini = """
197 | [tox]
198 | envlist = py3{7,8,9,10,11}
199 | skip_missing_interpreters = false
200 | toxworkdir = /tmp/.tox
201 | minversion = 4.0
202 | [testenv]
203 | recreate = true
204 | extras = tests
205 | commands =
206 |     python -m pytest {posargs}
207 |     python -m pylint --disable=C,R '**/*.py'
208 | """
209 | 


--------------------------------------------------------------------------------
/src/ppqueue/__init__.py:
--------------------------------------------------------------------------------
1 | from .job import Job
2 | from .queue import Queue
3 | 
4 | __all__ = ["Job", "Queue"]
5 | 


--------------------------------------------------------------------------------
/src/ppqueue/__version__.py:
--------------------------------------------------------------------------------
1 | __version__ = "0.4.0"
2 | 


--------------------------------------------------------------------------------
/src/ppqueue/exceptions.py:
--------------------------------------------------------------------------------
1 | class CustomException(Exception):
2 |     ...
3 | 


--------------------------------------------------------------------------------
/src/ppqueue/job.py:
--------------------------------------------------------------------------------
  1 | from __future__ import annotations
  2 | 
  3 | import logging as log
  4 | import time
  5 | from collections.abc import Iterable
  6 | from dataclasses import dataclass, field
  7 | from datetime import datetime
  8 | from enum import Enum, auto
  9 | from multiprocessing import Process
 10 | from threading import Thread
 11 | from typing import Any, Callable, Sequence
 12 | 
 13 | from .utils import compare_by
 14 | 
 15 | 
 16 | class JobState(Enum):
 17 |     UNSUBMITTED = auto()
 18 |     QUEUED = auto()
 19 |     RUNNING = auto()
 20 |     CANCELLED = auto()
 21 |     FINISHED = auto()
 22 |     UNKNOWN = auto()
 23 | 
 24 | 
 25 | @dataclass
 26 | class Job(object):
 27 |     """Represents a job returned by a Queue."""
 28 | 
 29 |     fun: Callable[..., Any]
 30 |     args: Sequence[Any] | None = field(default_factory=list)
 31 |     kwargs: dict[str, Any] | None = field(default_factory=dict)
 32 |     name: str | None = None
 33 |     priority: int = 100
 34 |     group: int | None = None
 35 |     timeout: float = 0
 36 |     suppress_errors: bool = False
 37 |     skip_on_group_error: bool = False
 38 |     # automatically assigned during processing.
 39 |     qid: str | None = field(default=None, init=False)
 40 |     """The ID of the queue that ran this job."""
 41 |     idx: int | None = field(default=None, init=False)
 42 |     """The ID of this job."""
 43 |     cancelled: bool = field(default=False, init=False)
 44 |     """Did this job timeout or was it cancelled?"""
 45 |     submit_timestamp: float | None = field(default=None, init=False)
 46 |     """Unix timestamp of when this job was submitted."""
 47 |     start_timestamp: float | None = field(default=None, init=False)
 48 |     """Unix timestamp of when this job started."""
 49 |     finish_timestamp: float | None = field(default=None, init=False)
 50 |     """Unix timestamp of when this job finished."""
 51 |     process_timestamp: float | None = field(default=None, init=False)
 52 |     """Unix timestamp of when this job was processed from the working queue."""
 53 |     result: Any = field(default=None, init=False)
 54 |     """Result of this job, if it exited gracefully."""
 55 |     callback_result: str | None = field(default=None, init=False)
 56 |     """Result of the callback for this job."""
 57 |     error: str | None = field(default=None, init=False)
 58 |     """Exception text for this job, if an error occurred."""
 59 |     exitcode: int | None = field(default=None, init=False)
 60 |     inner_job: Process | Thread | None = field(default=None, init=False)
 61 | 
 62 |     def __post_init__(self) -> None:
 63 |         if self.args is None:
 64 |             self.args = []
 65 |         elif isinstance(self.args, str) or not isinstance(self.args, Iterable):
 66 |             self.args = [self.args]
 67 | 
 68 |         if self.kwargs is None:
 69 |             self.kwargs = {}
 70 | 
 71 |     def _compare(self, job: Job) -> int:
 72 |         """compares two jobs by priority or index.
 73 | 
 74 |         Arguments:
 75 |             job: ...
 76 | 
 77 |         Returns:
 78 |             `1` if `self` is greater than comparison,
 79 |             `-1` if `self` is less than,
 80 |             `0` if equal.
 81 |         """
 82 |         return compare_by(self, job, by=["priority", "idx"])
 83 | 
 84 |     def __eq__(self, job: object) -> bool:
 85 |         return self._compare(job) == 0
 86 | 
 87 |     def __ne__(self, job: object) -> bool:
 88 |         return self._compare(job) != 0
 89 | 
 90 |     def __lt__(self, job: object) -> bool:
 91 |         return self._compare(job) < 0
 92 | 
 93 |     def __le__(self, job: object) -> bool:
 94 |         return self._compare(job) <= 0
 95 | 
 96 |     def __gt__(self, job: object) -> bool:
 97 |         return self._compare(job) > 0
 98 | 
 99 |     def __ge__(self, job: object) -> bool:
100 |         return self._compare(job) >= 0
101 | 
102 |     def is_running(self) -> bool:
103 |         return self.inner_job is not None and self.inner_job.is_alive()
104 | 
105 |     def is_expired(self) -> bool:
106 |         return (
107 |             self.is_running()
108 |             and self.timeout > 0
109 |             and self.start_timestamp is not None
110 |             and self.finish_timestamp is None
111 |             and self.start_timestamp + self.timeout < time.time()
112 |         )
113 | 
114 |     def join(self, *args, **kwargs) -> None:
115 |         # waits for the job to complete.
116 |         if self.inner_job is not None:
117 |             self.inner_job.join(*args, **kwargs)
118 | 
119 |     def get_exit_code(self) -> int | None:
120 |         """Exit code of the job.
121 | 
122 |         Returns:
123 |             ...
124 |         """
125 |         if (
126 |             not self.inner_job
127 |             or not hasattr(self.inner_job, "exitcode")
128 |             or self.inner_job.exitcode is None
129 |         ):
130 |             return None
131 |         return self.inner_job.exitcode
132 | 
133 |     def terminate(self) -> None:
134 |         if self.inner_job is not None:
135 |             self.inner_job.terminate()
136 | 
137 |     def stop(self) -> None:
138 |         # Terminates an existing process. Does not work for threads.
139 |         if not self.inner_job or not self.is_running():
140 |             return
141 |         elif not hasattr(self.inner_job, "terminate"):
142 |             log.error("Unable to terminate thread.")
143 |         else:
144 |             self.inner_job.terminate()
145 |             self.inner_job.join()
146 |             self.finish_timestamp = time.time()
147 |             self.cancelled = True
148 |             log.debug("Stopped data: '%d'", self.idx)
149 | 
150 |     def get_seconds_running(self) -> float | None:
151 |         """The number of seconds a job was running for.
152 | 
153 |         Returns:
154 |             ...
155 | 
156 |         Examples:
157 |             >>> from ppqueue import Queue
158 |             >>> from time import sleep
159 |             ...
160 |             >>> with Queue() as queue:
161 |             ...     for i in range(5):
162 |             ...         jobs = queue.map(sleep, [1, 2, 3])
163 |             ...
164 |             >>> [int(job.get_seconds_running()) for job in jobs]
165 |             [1, 2, 3]
166 |         """
167 |         if not self.start_timestamp or not self.finish_timestamp:
168 |             return None
169 |         return self.finish_timestamp - self.start_timestamp
170 | 
171 |     def get_seconds_waiting(self) -> float | None:
172 |         """The amount of time a data has spent in the waiting queue.
173 | 
174 |         Returns:
175 |             ...
176 | 
177 |         Examples:
178 |             >>> job.get_seconds_waiting()  # doctest: +SKIP
179 |         """
180 |         if not self.submit_timestamp:
181 |             return None
182 | 
183 |         if not self.start_timestamp:
184 |             return time.time() - self.submit_timestamp
185 | 
186 |         return self.start_timestamp - self.submit_timestamp
187 | 
188 |     def get_seconds_total(self) -> float | None:
189 |         """Returns the waiting + running duration of this job.
190 | 
191 |         Returns:
192 |             ...
193 | 
194 |         Examples:
195 |             >>> job.get_seconds_total()  # doctest: +SKIP
196 |         """
197 |         if not self.submit_timestamp:
198 |             return None
199 | 
200 |         if not self.finish_timestamp:
201 |             return time.time() - self.submit_timestamp
202 | 
203 |         return self.finish_timestamp - self.submit_timestamp
204 | 
205 |     def get_submit_timestamp(self) -> datetime | None:
206 |         """The time this job was submitted.
207 | 
208 |         Returns:
209 |             ...
210 | 
211 |         Examples:
212 |             >>> job.get_submit_timestamp()  # doctest: +SKIP
213 |         """
214 |         if self.submit_timestamp:
215 |             return datetime.utcfromtimestamp(self.submit_timestamp)
216 |         return None
217 | 
218 |     def get_start_timestamp(self) -> datetime | None:
219 |         """Returns a datetime object of the time this data was started.
220 | 
221 |         Returns:
222 |             ...
223 |         """
224 |         if self.start_timestamp:
225 |             return datetime.utcfromtimestamp(self.start_timestamp)
226 |         return None
227 | 
228 |     def get_finish_timestamp(self) -> datetime | None:
229 |         """Returns a datetime object of the time this data finished.
230 | 
231 |         Returns:
232 |             ...
233 |         """
234 |         if self.finish_timestamp:
235 |             return datetime.utcfromtimestamp(self.finish_timestamp)
236 |         return None
237 | 
238 |     def get_processed_timestamp(self) -> datetime | None:
239 |         """Returns a datetime object of the time this data was processed.
240 | 
241 |         Returns:
242 |             ...
243 |         """
244 |         if self.process_timestamp:
245 |             return datetime.utcfromtimestamp(self.process_timestamp)
246 |         return None
247 | 
248 |     def is_processed(self) -> bool:
249 |         # Returns true if this data has been processed; false otherwise.
250 |         # A processed data is one that has had its output gathered, callback called,
251 |         #   before being removed from the working dictionary.
252 | 
253 |         return self.process_timestamp is not None
254 | 
255 |     def get_state(self) -> JobState:
256 |         if not self.submit_timestamp:
257 |             return JobState.UNSUBMITTED
258 |         if self.is_running():
259 |             return JobState.RUNNING
260 |         if self.cancelled:
261 |             return JobState.CANCELLED
262 |         if self.finish_timestamp:
263 |             return JobState.FINISHED
264 |         return JobState.UNKNOWN
265 | 


--------------------------------------------------------------------------------
/src/ppqueue/plot.py:
--------------------------------------------------------------------------------
  1 | from __future__ import annotations
  2 | 
  3 | import logging as log
  4 | from enum import Enum, auto
  5 | 
  6 | import pandas as pd
  7 | import plotnine as gg
  8 | 
  9 | from .job import Job
 10 | from .utils import dedupe_list
 11 | 
 12 | __all__ = [
 13 |     "plot_jobs",
 14 |     "PlotColorBy",
 15 |     "PlotFacetBy",
 16 |     "PlotFacetScale",
 17 |     "PlotGridLines",
 18 |     "PlotTheme",
 19 | ]
 20 | 
 21 | 
 22 | class PlotColorBy(Enum):
 23 |     """Enum used to define color groups using this job property."""
 24 | 
 25 |     QID = auto()
 26 |     PRIORITY = auto()
 27 |     GROUP = auto()
 28 |     CANCELLED = auto()
 29 |     EXITCODE = auto()
 30 |     NAME = auto()
 31 |     RESULT = auto()
 32 |     CALLBACK_RESULT = auto()
 33 | 
 34 | 
 35 | class PlotFacetBy(Enum):
 36 |     """Enum used to facet the plot by this job property."""
 37 | 
 38 |     QID = auto()
 39 |     PRIORITY = auto()
 40 |     GROUP = auto()
 41 |     CANCELLED = auto()
 42 |     EXITCODE = auto()
 43 |     NAME = auto()
 44 |     RESULT = auto()
 45 |     CALLBACK_RESULT = auto()
 46 | 
 47 | 
 48 | class PlotTheme(Enum):
 49 |     """Enum used to expose supported plot themes."""
 50 | 
 51 |     BW = auto()
 52 |     CLASSIC = auto()
 53 |     GRAY = auto()
 54 |     GREY = auto()
 55 |     SEABORN = auto()
 56 |     DARK = auto()
 57 |     MATPLOTLIB = auto()
 58 |     MINIMAL = auto()
 59 |     XKCD = auto()
 60 |     LIGHT = auto()
 61 | 
 62 | 
 63 | class PlotFacetScale(Enum):
 64 |     FIXED = auto()
 65 |     FREE = auto()
 66 |     FREE_X = auto()
 67 |     FREE_Y = auto()
 68 | 
 69 | 
 70 | class PlotGridLines(Enum):
 71 |     NONE = auto()
 72 |     MAJOR = auto()
 73 |     MINOR = auto()
 74 |     BOTH = auto()
 75 | 
 76 | 
 77 | class Plot:
 78 |     def __init__(self, *args: list[Job]):
 79 |         df = pd.DataFrame([j.__dict__ for jobs in args for j in jobs])
 80 |         df.group.fillna(value="", inplace=True)
 81 |         min_time = df["submit_timestamp"].min()
 82 |         df["submitted_offset"] = df["submit_timestamp"] - min_time
 83 |         df["started_offset"] = df["start_timestamp"] - min_time
 84 |         df["ended_offset"] = df["finish_timestamp"] - min_time
 85 |         df["processed_offset"] = df["process_timestamp"] - min_time
 86 |         self.jobs_df = df
 87 | 
 88 |     @staticmethod
 89 |     def _plot_theme(
 90 |         theme: PlotTheme,
 91 |         grid_lines: PlotGridLines,
 92 |         grid_axis: str | None,
 93 |     ) -> gg.theme:
 94 |         # Used to provide a consistent theme across plots.
 95 | 
 96 |         drop_grid = set()
 97 | 
 98 |         if grid_axis is None or grid_lines == PlotGridLines.NONE:
 99 |             drop_grid.update(["panel_grid_major", "panel_grid_minor"])
100 |         elif grid_axis == "x":
101 |             drop_grid.update(["panel_grid_major_y", "panel_grid_minor_y"])
102 |             if grid_lines == PlotGridLines.MAJOR:
103 |                 drop_grid.add("panel_grid_minor_y")
104 |             elif grid_lines == PlotGridLines.MINOR:
105 |                 drop_grid.add("panel_grid_major_y")
106 |         elif grid_axis == "y":
107 |             drop_grid.update(["panel_grid_major_x", "panel_grid_minor_x"])
108 |             if grid_lines == PlotGridLines.MAJOR:
109 |                 drop_grid.add("panel_grid_minor_x")
110 |             elif grid_lines == PlotGridLines.MINOR:
111 |                 drop_grid.add("panel_grid_major_x")
112 | 
113 |         grid_opt = {}
114 |         for x in drop_grid:
115 |             grid_opt[x] = gg.element_blank()
116 | 
117 |         return getattr(gg, "theme_" + theme.name.lower())() + gg.theme(
118 |             panel_border=gg.element_blank(),
119 |             axis_line=gg.element_line(color="black"),
120 |             **grid_opt,
121 |         )
122 | 
123 |     def build(
124 |         self,
125 |         title: str | None,
126 |         color_by: PlotColorBy,
127 |         facet_by: PlotFacetBy,
128 |         facet_scale: PlotFacetScale,
129 |         theme: PlotTheme,
130 |         no_legend: bool,
131 |         bar_width: int,
132 |         color_pal: list[str] | None,
133 |     ) -> gg.ggplot:
134 |         df2 = self.jobs_df.loc[
135 |             :,
136 |             dedupe_list(
137 |                 [
138 |                     "qid",
139 |                     "idx",
140 |                     color_by.name.lower(),
141 |                     facet_by.name.lower(),
142 |                     "submitted_offset",
143 |                     "started_offset",
144 |                     "ended_offset",
145 |                     "processed_offset",
146 |                 ],
147 |             ),
148 |         ].melt(
149 |             id_vars=dedupe_list(
150 |                 ["qid", "idx", color_by.name.lower(), facet_by.name.lower()],
151 |             ),
152 |         )
153 | 
154 |         df2 = df2[df2["value"].notnull()]
155 | 
156 |         df_submit_start = df2[
157 |             (df2["variable"] == "submitted_offset")
158 |             | (df2["variable"] == "started_offset")
159 |         ]
160 |         df_start_end = df2[
161 |             (df2["variable"] == "started_offset") | (df2["variable"] == "ended_offset")
162 |         ]
163 |         df_end_processed = df2[
164 |             (df2["variable"] == "ended_offset")
165 |             | (df2["variable"] == "processed_offset")
166 |         ]
167 | 
168 |         labs = {"x": "duration", "y": "job index"}
169 |         if title is not None:
170 |             labs["title"] = title
171 | 
172 |         gg_obj = (
173 |             gg.ggplot(gg.aes(x="value", y="idx", group="factor(idx)"))
174 |             + gg.geom_line(df_submit_start, color="gray", size=bar_width, alpha=0.2)
175 |             + gg.geom_line(
176 |                 df_start_end,
177 |                 gg.aes(color=f"factor({color_by.name.lower()})"),
178 |                 size=bar_width,
179 |                 show_legend=not bool(no_legend),
180 |             )
181 |             + gg.geom_line(df_end_processed, color="gray", size=bar_width, alpha=0.2)
182 |             + gg.labs(**labs)
183 |             + gg.labs(color=color_by.name.lower())
184 |             + Plot._plot_theme(
185 |                 theme=theme,
186 |                 grid_lines=PlotGridLines.BOTH,
187 |                 grid_axis="x",
188 |             )
189 |             + gg.facet_grid(
190 |                 facets=facet_by.name.lower() + "~",
191 |                 labeller="label_both",
192 |                 scales=facet_scale.name.lower(),
193 |                 as_table=True,
194 |             )
195 |         )
196 | 
197 |         if not color_pal:
198 |             gg_obj += gg.scale_color_hue(h=0.65)
199 |         else:
200 |             n_colors = self.jobs_df[color_by.name.lower()].unique().size
201 | 
202 |             if len(color_pal) < n_colors:
203 |                 log.warning("Insufficient number of colors; need at least %d", n_colors)
204 |                 gg_obj += gg.scale_color_hue(h=0.65)
205 |             else:
206 |                 gg_obj += gg.scale_color_manual(color_pal[:n_colors])
207 | 
208 |         return gg_obj
209 | 
210 | 
211 | def plot_jobs(
212 |     *args: list[Job],
213 |     title: str | None = None,
214 |     color_by: PlotColorBy = PlotColorBy.QID,
215 |     facet_by: PlotFacetBy = PlotFacetBy.QID,
216 |     facet_scale: PlotFacetScale = PlotFacetScale.FIXED,
217 |     theme: PlotTheme = PlotTheme.BW,
218 |     no_legend: bool = False,
219 |     bar_width: int = 1,
220 |     color_pal: list[str] | None = None,
221 | ) -> gg.ggplot:
222 |     """
223 | 
224 |     Args:
225 |         *args: Sequences of `Job` instances to plot.
226 |         title: ...
227 |         color_by: ...
228 |         facet_by: ...
229 |         facet_scale: ...
230 |         theme: ...
231 |         no_legend: ...
232 |         bar_width: ...
233 |         color_pal: a sequence of colors used to color each group of `color_by`.
234 | 
235 |     Returns:
236 |         ...
237 | 
238 |     Examples:
239 |         >>> from ppqueue import Queue
240 |         >>> from ppqueue.plot import plot_jobs
241 |         >>> from time import sleep
242 |         ...
243 |         >>> with Queue() as queue:
244 |         ...     q1_jobs = queue.map(sleep, [1, 2, 3, 4, 5])
245 |         ...
246 |         >>> with Queue() as queue:
247 |         ...     q2_jobs = queue.map(sleep, [1, 2, 3, 4, 5])
248 |         ...
249 |         >>> plot = plot_jobs(
250 |         ...     q1_jobs, q2_jobs,
251 |         ...     title='Demo',
252 |         ...     color_by=PlotColorBy.QID,
253 |         ...     facet_by=PlotFacetBy.QID,
254 |         ...     theme=PlotTheme.DARK,
255 |         ...     bar_width=2,
256 |         ...     color_pal=['red', 'blue']
257 |         ... )
258 |         ...
259 |         >>> print(plot)  # doctest: +SKIP
260 |     """
261 | 
262 |     return Plot(*args).build(
263 |         color_by=color_by,
264 |         facet_by=facet_by,
265 |         facet_scale=facet_scale,
266 |         no_legend=no_legend,
267 |         bar_width=bar_width,
268 |         title=title,
269 |         color_pal=color_pal,
270 |         theme=theme,
271 |     )
272 | 


--------------------------------------------------------------------------------
/src/ppqueue/py.typed:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/fresh2dev/ppqueue/7cf96a7b2110b89df2ae13364943374a299c4e12/src/ppqueue/py.typed


--------------------------------------------------------------------------------
/src/ppqueue/queue.py:
--------------------------------------------------------------------------------
   1 | from __future__ import annotations
   2 | 
   3 | import logging
   4 | import multiprocessing as mp
   5 | import threading
   6 | import time
   7 | import traceback
   8 | from heapq import heappop, heappush
   9 | from typing import Any, Callable, Sequence
  10 | from uuid import uuid4
  11 | 
  12 | from tqdm.auto import tqdm
  13 | 
  14 | from .job import Job
  15 | from .utils import get_logger, is_windows_os
  16 | 
  17 | LOG = get_logger(__name__, level=logging.INFO)
  18 | 
  19 | 
  20 | class PulseTimer:
  21 |     # Periodically runs a function in a background thread.
  22 |     # src: https://stackoverflow.com/a/13151299
  23 | 
  24 |     def __init__(
  25 |         self,
  26 |         *args: Any,
  27 |         interval_ms: int,
  28 |         fun: Callable[..., Any],
  29 |         **kwargs: Any,
  30 |     ):
  31 |         self._timer: threading.Timer | None = None
  32 |         self._interval: int = interval_ms
  33 |         self._function: Callable[..., Any] = fun
  34 |         self._args: tuple[Any, ...] = args
  35 |         self._kwargs: dict[str, Any] = kwargs
  36 |         self._is_enabled: bool = False
  37 |         self._is_running: bool = False
  38 | 
  39 |     def _run(self) -> None:
  40 |         self._is_running = False
  41 |         self.start()
  42 |         self._function(*self._args, **self._kwargs)
  43 | 
  44 |     def start(self) -> None:
  45 |         if not self._is_running:
  46 |             self._timer = threading.Timer(self._interval / 1000, self._run)
  47 |             self._timer.start()
  48 |             self._is_running = True
  49 | 
  50 |         self._is_enabled = True
  51 | 
  52 |     def stop(self) -> None:
  53 |         if self._timer is not None:
  54 |             self._timer.cancel()
  55 | 
  56 |         self._is_enabled = False
  57 |         self._is_running = False
  58 | 
  59 |     @property
  60 |     def is_running(self) -> bool:
  61 |         return self._is_running
  62 | 
  63 |     @property
  64 |     def is_enabled(self) -> bool:
  65 |         return self._is_enabled
  66 | 
  67 | 
  68 | class Queue:
  69 |     DEFAULT_GROUP: int | None = None
  70 | 
  71 |     def __init__(
  72 |         self,
  73 |         max_concurrent: int = mp.cpu_count(),
  74 |         *,
  75 |         max_size: int = 0,
  76 |         engine: str | type[mp.Process] | type[threading.Thread] = mp.Process,
  77 |         name: str | None = None,
  78 |         callback: Callable[[Job], Any] | None = None,
  79 |         show_progress: bool = False,
  80 |         drop_finished: bool = False,
  81 |         stop_when_idle: bool = False,
  82 |         pulse_freq_ms: int = 100,
  83 |         no_start: bool = False,
  84 |     ):
  85 |         """A parallel processing job runner / data structure.
  86 | 
  87 |         Args:
  88 |             max_concurrent: max number of concurrently running jobs.
  89 |             max_size: max size of the queue (default=0, unlimited).
  90 |             engine: the engine used to run jobs.
  91 |             name: an identifier for this queue.
  92 |             callback: a callable that is called immediately after each job is finished.
  93 |             show_progress: global setting for showing progress bars.
  94 |             drop_finished: if True, the queue will not store finished jobs for retrieval.
  95 |             stop_when_idle: if True, the queue will stop the pulse when all jobs are finished.
  96 |             pulse_freq_ms: the interval at which jobs are transitioned between internal queues.
  97 |             no_start: if True, do not start the queue pulse on instantiation.
  98 | 
  99 |         Examples:
 100 |             >>> from ppqueue import Queue
 101 |             >>> from time import sleep
 102 |             ...
 103 |             >>> with Queue() as queue:
 104 |             ...     jobs = queue.map(sleep, [1, 2, 3, 4, 5])
 105 |             ...
 106 |             >>> len(jobs)
 107 |             5
 108 |         """
 109 | 
 110 |         self._qid: str = name if name else str(uuid4())[:8]
 111 | 
 112 |         self.show_progress: bool = show_progress
 113 |         self._max_size: int = max_size
 114 |         self._count_input: int = 0
 115 |         self._count_output: int = 0
 116 |         self._max_concurrent: int = max_concurrent
 117 |         self._callback: Callable[..., Any] | None = callback
 118 | 
 119 |         # https://opensource.com/article/17/4/grok-gil
 120 |         self._lock: threading.Lock = threading.Lock()
 121 | 
 122 |         if isinstance(engine, str):
 123 |             if engine.lower() in ["thread", "threads", "threading"]:
 124 |                 engine = threading.Thread
 125 |             elif engine.lower() in [
 126 |                 "process",
 127 |                 "processes",
 128 |                 "processing",
 129 |                 "multiprocessing",
 130 |             ]:
 131 |                 engine = mp.Process
 132 |             else:
 133 |                 err: str = f"Unrecognized engine: {engine}. Choose either 'process' or 'thread'."
 134 |                 raise ValueError(err)
 135 | 
 136 |         if engine is mp.Process and is_windows_os():
 137 |             LOG.warning(
 138 |                 (
 139 |                     "multiprocessing performance is degraded on Windows systems. see: "
 140 |                     "https://docs.python.org/3/library/multiprocessing.html?highlight=process#the-spawn-and-forkserver-start-methods"
 141 |                 ),
 142 |             )
 143 | 
 144 |         self._engine: type[mp.Process] | type[threading.Thread] = engine
 145 | 
 146 |         self._waiting_groups: dict[int | None, list[Job]] = {self.DEFAULT_GROUP: []}
 147 |         self._working_queue: dict[int | None, Job] = {}
 148 |         self._finished_queue: list[Job] = []
 149 | 
 150 |         self._drop_finished: bool = drop_finished
 151 |         self._mp_manager: mp.Manager | None = None
 152 | 
 153 |         self._output: dict[int, Job]
 154 | 
 155 |         if self._engine is mp.Process:
 156 |             self._mp_manager = mp.Manager()
 157 |             self._output = self._mp_manager.dict()
 158 |         else:
 159 |             self._output = {}
 160 | 
 161 |         LOG.debug("Initialized queue with %d max_concurrent.", self._max_concurrent)
 162 | 
 163 |         self._pulse_freq_ms: int = pulse_freq_ms
 164 |         self._timer: PulseTimer = PulseTimer(
 165 |             interval_ms=self._pulse_freq_ms,
 166 |             fun=self._pulse,
 167 |         )
 168 | 
 169 |         self._stop_when_idle: bool = stop_when_idle
 170 | 
 171 |         if not no_start:
 172 |             self.start()
 173 | 
 174 |         LOG.debug("Initialized pulse.")
 175 | 
 176 |     # decorator
 177 |     def __call__(self, fun: Callable[..., Any], *args: Any, **kwargs: Any):
 178 |         self.start()
 179 | 
 180 |         def wrapped_f(iterable: Sequence[Any], *args: Any, **kwargs: Any) -> list[Job]:
 181 |             for x in iterable:
 182 |                 self.enqueue(fun, args=[x, *args], kwargs=kwargs)
 183 |             self.wait()
 184 |             job_data = self.collect()
 185 |             self.dispose()
 186 |             return job_data
 187 | 
 188 |         return wrapped_f
 189 | 
 190 |     def __del__(self) -> None:
 191 |         self.dispose()
 192 | 
 193 |     def start(self) -> None:
 194 |         """Start the queue pulse."""
 195 |         self._timer.start()
 196 | 
 197 |     def stop(self) -> None:
 198 |         """Stop the queue pulse."""
 199 |         self._timer.stop()
 200 | 
 201 |     def dispose(self) -> None:
 202 |         """Stop running jobs, then clear the queue, then stop the queue pulse."""
 203 |         LOG.debug("Disposing")
 204 | 
 205 |         if self.is_running:
 206 |             self._stop_all(wait=True)
 207 |             self.stop()
 208 | 
 209 |         LOG.debug("Removed jobs.")
 210 |         self._output = None
 211 |         LOG.debug("Removed output.")
 212 |         self._finished_queue.clear()  # [:] = []
 213 |         LOG.debug("Removed finished.")
 214 |         self._count_input = 0
 215 |         self._count_output = 0
 216 |         LOG.debug("Reset counters.")
 217 | 
 218 |         if self._mp_manager is not None:
 219 |             self._mp_manager.shutdown()
 220 | 
 221 |     def _clear_waiting(self) -> None:
 222 |         LOG.debug("Clearing waiting queue")
 223 |         with self._lock:
 224 |             keys = list(self._waiting_groups.keys())
 225 |             for k in keys:
 226 |                 group_jobs = self._waiting_groups.get(k)
 227 |                 for _ in range(len(group_jobs)):
 228 |                     job = heappop(group_jobs)
 229 |                     job.cancelled = True
 230 |                     job.process_timestamp = time.time()
 231 |                     if not self._drop_finished:
 232 |                         heappush(self._finished_queue, job)
 233 |                     else:
 234 |                         self._count_output += 1
 235 |                 del self._waiting_groups[k]
 236 | 
 237 |     def _stop_all(self, *, wait: bool = True) -> None:
 238 |         self._clear_waiting()
 239 | 
 240 |         keys = list(self._working_queue.keys())
 241 |         for k in keys:
 242 |             job = self._working_queue.get(k)
 243 |             if job is not None:
 244 |                 job.stop()
 245 | 
 246 |         if wait:
 247 |             self.wait()
 248 | 
 249 |     def __enter__(self) -> Queue:
 250 |         return self
 251 | 
 252 |     def __exit__(self, *args: Any) -> None:
 253 |         self.dispose()
 254 | 
 255 |     def __iter__(self) -> Queue:
 256 |         return self
 257 | 
 258 |     def __next__(self) -> Job:
 259 |         job: Job | None = self.dequeue()
 260 |         if job is None:
 261 |             raise StopIteration
 262 |         return job
 263 | 
 264 |     def _pulse(self) -> None:
 265 |         # Used internally; manages the queue system operations.
 266 | 
 267 |         with self._lock:
 268 |             try:
 269 |                 if self._stop_when_idle and self.is_idle():
 270 |                     self.stop()
 271 |                 else:
 272 |                     # stop expired jobs.
 273 |                     for job_id, job in self._working_queue.items():
 274 |                         if job.is_expired():
 275 |                             job.stop()
 276 | 
 277 |                     for job_id in list(self._working_queue.keys()):
 278 |                         job = self._working_queue[job_id]
 279 | 
 280 |                         if not job.is_running() and not job.is_processed():
 281 |                             job.join()
 282 |                             if not job.cancelled:
 283 |                                 try:
 284 |                                     (
 285 |                                         stdout,
 286 |                                         stderr,
 287 |                                         exitcode,
 288 |                                         timestamp,
 289 |                                     ) = self._output.pop(job.idx)
 290 |                                     job.result = stdout
 291 |                                     job.error = stderr
 292 |                                     job.exitcode = exitcode
 293 |                                     job.finish_timestamp = timestamp
 294 |                                 except KeyError:
 295 |                                     job.exception_txt = str(
 296 |                                         Exception(
 297 |                                             "{}\n\nNo data for data; it may have exited"
 298 |                                             " unexpectedly.".format(job.idx),
 299 |                                         ),
 300 |                                     )
 301 | 
 302 |                             if self._callback is not None:
 303 |                                 try:
 304 |                                     job.callback_result = self._callback(job)
 305 |                                 except (
 306 |                                     Exception  # pylint: disable=broad-exception-caught
 307 |                                 ) as ex:
 308 |                                     job.callback_result = str(ex)
 309 | 
 310 |                             job.process_timestamp = time.time()
 311 | 
 312 |                             LOG.debug("Completed job: %d", job.idx)
 313 | 
 314 |                             if not self._drop_finished:
 315 |                                 heappush(self._finished_queue, job)
 316 |                             else:
 317 |                                 self._count_output += 1
 318 | 
 319 |                             del self._working_queue[job_id]
 320 | 
 321 |                             if (
 322 |                                 job.group is not None
 323 |                                 and job.group != self.DEFAULT_GROUP
 324 |                             ):
 325 |                                 group_jobs: list[Job] = self._waiting_groups.get(
 326 |                                     job.group,
 327 |                                 )
 328 |                                 if group_jobs is not None:
 329 |                                     next_job = None
 330 |                                     parent_exitcode = job.exitcode
 331 |                                     while len(group_jobs) > 0:
 332 |                                         next_job = heappop(group_jobs)
 333 | 
 334 |                                         if (
 335 |                                             parent_exitcode == 0
 336 |                                             or not next_job.skip_on_group_error
 337 |                                         ):
 338 |                                             break
 339 |                                         else:
 340 |                                             next_job.cancelled = True
 341 |                                             next_job.exitcode = parent_exitcode
 342 |                                             next_job.error = (
 343 |                                                 "skip_on_group_error = True and"
 344 |                                                 " preceding data ({}) exit code is {}".format(
 345 |                                                     job.idx,
 346 |                                                     parent_exitcode,
 347 |                                                 )
 348 |                                             )
 349 |                                             next_job.process_timestamp = time.time()
 350 |                                             if not self._drop_finished:
 351 |                                                 heappush(
 352 |                                                     self._finished_queue,
 353 |                                                     next_job,
 354 |                                                 )
 355 |                                             else:
 356 |                                                 self._count_output += 1
 357 |                                             next_job = None
 358 |                                             continue
 359 | 
 360 |                                     if len(group_jobs) == 0:
 361 |                                         del self._waiting_groups[job.group]
 362 | 
 363 |                                     if next_job is not None:
 364 |                                         self._working_queue[next_job.idx] = next_job
 365 |                                         self._start_job(next_job)
 366 | 
 367 |                     while (
 368 |                         self._waiting_groups.get(self.DEFAULT_GROUP)
 369 |                         and self.max_concurrent - self._count_working() > 0
 370 |                     ):
 371 |                         job = heappop(self._waiting_groups[self.DEFAULT_GROUP])
 372 |                         self._working_queue[job.idx] = job
 373 |                         self._start_job(job)
 374 |             except:  # pylint: disable=bare-except
 375 |                 self.stop()
 376 |                 LOG.error(traceback.format_exc())
 377 |             finally:
 378 |                 LOG.debug(
 379 |                     "waiting=%d; working=%d; finished=%d.",
 380 |                     self._count_waiting(),
 381 |                     self._count_working(),
 382 |                     self._count_finished(),
 383 |                 )
 384 | 
 385 |     def _count_waiting(self) -> int:
 386 |         """
 387 |         Returns:
 388 |             The number of pending jobs.
 389 |         """
 390 |         return sum(len(v) for _, v in self._waiting_groups.items())
 391 | 
 392 |     def _count_working(self) -> int:
 393 |         """
 394 |         Returns:
 395 |             The number of running jobs.
 396 |         """
 397 |         return len(self._working_queue)
 398 | 
 399 |     def _count_finished(self) -> int:
 400 |         """
 401 |         Returns:
 402 |             The number of completed jobs.
 403 |         """
 404 |         return len(self._finished_queue)
 405 | 
 406 |     def _count_remaining(self) -> int:
 407 |         """
 408 |         Returns:
 409 |             The number of unfinished jobs (i.e., waiting + working).
 410 |         """
 411 |         return self.size(waiting=True, working=True)
 412 | 
 413 |     def _sizes(
 414 |         self,
 415 |         *,
 416 |         waiting: bool = False,
 417 |         working: bool = False,
 418 |         finished: bool = False,
 419 |     ) -> list[int]:
 420 |         """Returns the number of jobs in the corresponding queue(s).
 421 | 
 422 |         Args:
 423 |             waiting: include jobs in the waiting queue?
 424 |                 - Accepts: bool
 425 |                 - Default: False
 426 |             working: include jobs in the working table?
 427 |                 - Accepts: bool
 428 |                 - Default: False
 429 |             finished: include jobs in the completed queue?
 430 |                 - Accepts: bool
 431 |                 - Default: False
 432 | 
 433 |         Note: when all are False, all jobs are counted (default).
 434 | 
 435 |         Returns:
 436 |             int
 437 |         """
 438 | 
 439 |         counts = {}
 440 | 
 441 |         is_locked = False
 442 | 
 443 |         try:
 444 |             to_tally = sum([waiting, working, finished])
 445 |             if to_tally != 1:
 446 |                 if to_tally == 0:
 447 |                     waiting, working, finished = True, True, True
 448 |                 # must lock when more than 1 component included.
 449 |                 is_locked = self._lock.acquire()
 450 | 
 451 |             if waiting:
 452 |                 counts["waiting"] = self._count_waiting()
 453 |             if working:
 454 |                 counts["working"] = self._count_working()
 455 |             if finished:
 456 |                 counts["finished"] = self._count_finished()
 457 |         finally:
 458 |             if is_locked:
 459 |                 self._lock.release()
 460 | 
 461 |         return list(counts.values())
 462 | 
 463 |     def __len__(self) -> int:
 464 |         return self.size()
 465 | 
 466 |     def size(
 467 |         self,
 468 |         *,
 469 |         waiting: bool = False,
 470 |         working: bool = False,
 471 |         finished: bool = False,
 472 |     ) -> int:
 473 |         """Get the number of jobs in the queue in state: waiting, working, and/or finished.
 474 | 
 475 |         If no options are given, the total number of jobs in the queue is returned.
 476 | 
 477 |         Args:
 478 |             waiting: include waiting jobs.
 479 |             working: include working jobs.
 480 |             finished: include finished jobs.
 481 | 
 482 |         Returns:
 483 |             ...
 484 | 
 485 |         Examples:
 486 |             >>> from ppqueue import Queue
 487 |             ...
 488 |             >>> def add_nums(x: int, y: int) -> int:
 489 |             ...     return x + y
 490 |             ...
 491 |             >>> with Queue() as queue:
 492 |             ...     for i in range(5):
 493 |             ...         _ = queue.enqueue(add_nums, args=[i, 100])
 494 |             ...         print(queue.size())
 495 |             1
 496 |             2
 497 |             3
 498 |             4
 499 |             5
 500 |         """
 501 |         return sum(self._sizes(waiting=waiting, working=working, finished=finished))
 502 | 
 503 |     def is_idle(self) -> bool:
 504 |         return self._count_waiting() == 0 and self._count_working() == 0
 505 | 
 506 |     def is_busy(self) -> bool:
 507 |         """True if max concurrent limit is reached or if there are waiting jobs.
 508 | 
 509 |         Returns:
 510 |             ...
 511 |         """
 512 |         return self._count_waiting() > 0 or self.max_concurrent <= self._count_working()
 513 | 
 514 |     def is_empty(self) -> bool:
 515 |         """True if there are no jobs in the queue system.
 516 | 
 517 |         Returns:
 518 |             ...
 519 |         """
 520 |         return self.size() == 0
 521 | 
 522 |     def is_full(self) -> bool:
 523 |         """True if the number of jobs in the queue system is equal to max_size.
 524 | 
 525 |         Returns:
 526 |             ...
 527 |         """
 528 |         return 0 < self._max_size <= self.size()
 529 | 
 530 |     def join(self, *args, **kwargs) -> int:
 531 |         return self.wait(*args, **kwargs)
 532 | 
 533 |     def wait(
 534 |         self,
 535 |         *,
 536 |         n: int = 0,
 537 |         timeout: float = 0,
 538 |         poll_ms: int = 0,
 539 |         show_progress: bool | None = None,
 540 |     ) -> int:
 541 |         """Wait for jobs to finish.
 542 | 
 543 |         Args:
 544 |             n: the number of jobs to wait for (default=0, all).
 545 |             timeout: seconds to wait before raising `TimeoutError` (default=0, indefinitely).
 546 |             poll_ms: milliseconds to pause between checks (default=100).
 547 |             show_progress: if True, present a progress bar.
 548 | 
 549 |         Returns:
 550 |             If `n <= 0`, returns the count of unfinished jobs.
 551 |             Else, returns the count of finished jobs.
 552 |         """
 553 |         if poll_ms is None or poll_ms <= 0:
 554 |             poll_ms = self._pulse_freq_ms
 555 |         elif poll_ms < self._pulse_freq_ms:
 556 |             err: str = "`poll_ms` must be >= `ppqueue.Queue.pulse_freq_ms`"
 557 |             raise ValueError(err)
 558 | 
 559 |         if show_progress is None:
 560 |             show_progress = self.show_progress
 561 | 
 562 |         _fun: Callable[..., Any] | None = None
 563 |         _target_value: int | None = None
 564 |         _comparator: Callable[..., bool] | None = None
 565 | 
 566 |         if n <= 0:
 567 |             _fun = self._count_remaining
 568 |             _target_value = 0
 569 |             _comparator = int.__le__
 570 |         else:
 571 |             _fun = self._count_finished
 572 |             _target_value = n
 573 |             _comparator = int.__ge__
 574 | 
 575 |         current_value: int = _fun()
 576 | 
 577 |         if not _comparator(current_value, _target_value):
 578 |             start = time.time()
 579 | 
 580 |             pb = None
 581 | 
 582 |             try:
 583 |                 if show_progress:
 584 |                     pb = tqdm(total=current_value, unit="op")
 585 | 
 586 |                 while not _comparator(current_value, _target_value) and (
 587 |                     timeout == 0 or time.time() - start < timeout
 588 |                 ):
 589 |                     time.sleep(poll_ms / 1000)
 590 | 
 591 |                     tmp_value = _fun()
 592 |                     if pb is None:
 593 |                         current_value = tmp_value
 594 |                     else:
 595 |                         diff_value = current_value - tmp_value
 596 |                         if diff_value > 0:
 597 |                             current_value = tmp_value
 598 |                             pb.update(diff_value)
 599 |             finally:
 600 |                 if pb:
 601 |                     pb.close()
 602 | 
 603 |         return current_value
 604 | 
 605 |     @staticmethod
 606 |     def _job_wrapper(
 607 |         _job: Job,
 608 |         _output: dict[int, tuple[str | None, str | None, int, float]],
 609 |         /,
 610 |         *args: Any,
 611 |         **kwargs: Any,
 612 |     ) -> tuple[str | None, str | None, int, float]:
 613 |         # Used internally to wrap data, capture output and any exception.
 614 |         stdout: str | None = None
 615 |         stderr: str | None = None
 616 |         exitcode: int
 617 |         timestamp: float
 618 | 
 619 |         try:
 620 |             stdout = _job.fun(*args, **kwargs)
 621 |             exitcode = 0
 622 |         except Exception as ex:  # pylint: disable=broad-exception-caught
 623 |             stderr = traceback.format_exc()
 624 |             exitcode = -1
 625 |             if not _job.suppress_errors:
 626 |                 raise ex
 627 |         finally:
 628 |             timestamp = time.time()
 629 |             _output[_job.idx] = (stdout, stderr, exitcode, timestamp)
 630 | 
 631 |     def _start_job(self, job: Job, /) -> None:
 632 |         inner_job: mp.Process | threading.Thread = self._engine(
 633 |             name=str(job.idx),
 634 |             target=self._job_wrapper,
 635 |             args=[job, self._output, *job.args],
 636 |             kwargs=job.kwargs,
 637 |         )
 638 | 
 639 |         LOG.debug("starting job %d (name=%s)", job.idx, job.name)
 640 |         inner_job.start()
 641 | 
 642 |         job.start_timestamp = time.time()
 643 |         job.inner_job = inner_job
 644 | 
 645 |     def _submit(self, job: Job, /) -> int:
 646 |         """Submits a job into the ppqueue.Queue system.
 647 | 
 648 |         Args:
 649 |             job: ...
 650 | 
 651 |         Raises:
 652 |             OverflowError: if max_size will be exceeded after adding this job.
 653 | 
 654 |         Returns:
 655 |             The number of jobs submitted to the queue.
 656 |         """
 657 | 
 658 |         assert not (self._engine is threading.Thread and job.timeout > 0)
 659 | 
 660 |         if self.is_full():
 661 |             raise OverflowError("Max queue size exceeded.")
 662 | 
 663 |         job.submit_timestamp = time.time()
 664 | 
 665 |         job.qid = self._qid
 666 |         self._count_input += 1
 667 |         job.idx = self._count_input
 668 | 
 669 |         if job.name is None:
 670 |             job.name = job.idx
 671 | 
 672 |         LOG.debug("queuing job %d (name=%s)", job.idx, job.name)
 673 | 
 674 |         group_jobs = self._waiting_groups.get(job.group)
 675 |         if group_jobs is not None:
 676 |             heappush(group_jobs, job)
 677 |         else:
 678 |             group_jobs = []
 679 |             self._waiting_groups[job.group] = group_jobs
 680 | 
 681 |             # if this is the first job in this group,
 682 |             # it should be pushed to the default group.
 683 |             heappush(self._waiting_groups[self.DEFAULT_GROUP], job)
 684 | 
 685 |         return job.idx
 686 | 
 687 |     def enqueue(
 688 |         self,
 689 |         fun: Callable[..., Any],
 690 |         /,
 691 |         args: Sequence[Any] | None = None,
 692 |         kwargs: dict[str, Any] | None = None,
 693 |         name: str | None = None,
 694 |         priority: int = 100,
 695 |         group: int | None = None,
 696 |         timeout: float = 0,
 697 |         suppress_errors: bool = False,
 698 |         skip_on_group_error: bool = False,
 699 |     ) -> int:
 700 |         """Adds a job to the queue.
 701 | 
 702 |         Args:
 703 |             fun: ...
 704 |             args: ...
 705 |             kwargs: ...
 706 |             name: ...
 707 |             priority: ...
 708 |             group: ...
 709 |             timeout: ...
 710 |             suppress_errors: ...
 711 |             skip_on_group_error: ...
 712 | 
 713 |         Returns:
 714 |             ...
 715 | 
 716 |         Examples:
 717 |             >>> from ppqueue import Queue
 718 |             ...
 719 |             >>> def add_nums(x: int, y: int) -> int:
 720 |             ...     return x + y
 721 |             ...
 722 |             >>> with Queue() as queue:
 723 |             ...     for i in range(5):
 724 |             ...         _ = queue.enqueue(add_nums, args=[i, 100])
 725 |             ...
 726 |             ...     jobs = queue.collect(wait=True)
 727 |             ...
 728 |             >>> [job.result for job in jobs]
 729 |             [100, 101, 102, 103, 104]
 730 |         """
 731 |         job = Job(
 732 |             fun=fun,
 733 |             args=args,
 734 |             kwargs=kwargs,
 735 |             name=name,
 736 |             priority=priority,
 737 |             group=group,
 738 |             timeout=timeout,
 739 |             suppress_errors=suppress_errors,
 740 |             skip_on_group_error=skip_on_group_error,
 741 |         )
 742 | 
 743 |         return self._submit(job)
 744 | 
 745 |     def put(self, *args, **kwargs) -> int:
 746 |         """Alias for `enqueue`.
 747 | 
 748 |         Adds a job to the queue.
 749 | 
 750 |         Args:
 751 |             *args: ...
 752 |             **kwargs: ...
 753 | 
 754 |         Returns:
 755 |             ...
 756 | 
 757 |         Examples:
 758 |             >>> from ppqueue import Queue
 759 |             ...
 760 |             >>> def add_nums(x: int, y: int) -> int:
 761 |             ...     return x + y
 762 |             ...
 763 |             >>> with Queue() as queue:
 764 |             ...     for i in range(5):
 765 |             ...         _ = queue.put(add_nums, args=[i, 100])
 766 |             ...     jobs = queue.collect(wait=True)
 767 |             ...
 768 |             >>> [job.result for job in jobs]
 769 |             [100, 101, 102, 103, 104]
 770 |         """
 771 | 
 772 |         return self.enqueue(*args, **kwargs)
 773 | 
 774 |     def map(
 775 |         self,
 776 |         fun: Callable[..., Any],
 777 |         iterable: Sequence[Any],
 778 |         /,
 779 |         *args: Any,
 780 |         timeout: float = 0,
 781 |         show_progress: bool | None = None,
 782 |         **kwargs: Any,
 783 |     ) -> list[Job]:
 784 |         """Submits many jobs to the queue -- one for each item in the iterable. Waits for all to finish, then returns the results.
 785 | 
 786 |         Args:
 787 |             fun: ...
 788 |             iterable: ...
 789 |             *args: ...
 790 |             timeout: ...
 791 |             show_progress: ...
 792 |             **kwargs: ...
 793 | 
 794 |         Returns:
 795 |             ...
 796 | 
 797 |         Examples:
 798 |             >>> from ppqueue import Queue
 799 |             >>> from time import sleep
 800 |             ...
 801 |             >>> with Queue() as queue:
 802 |             ...     jobs = queue.map(sleep, [1, 2, 3, 4, 5])
 803 |             ...
 804 |             >>> len(jobs)
 805 |             5
 806 |         """
 807 |         for x in iterable:
 808 |             job = Job(
 809 |                 fun=fun,
 810 |                 args=[x, *args],
 811 |                 kwargs=dict(kwargs),
 812 |                 timeout=timeout,
 813 |             )
 814 |             self._submit(job)
 815 | 
 816 |         return self.collect(wait=True, show_progress=show_progress)
 817 | 
 818 |     def starmap(
 819 |         self,
 820 |         fun: Callable[..., Any],
 821 |         iterable: Sequence[Sequence[Any]],
 822 |         /,
 823 |         *args: Any,
 824 |         timeout: float = 0,
 825 |         show_progress: bool | None = None,
 826 |         **kwargs: Any,
 827 |     ) -> list[Job]:
 828 |         """Submits many jobs to the queue -- one for each sequence in the iterable. Waits for all to finish, then returns the results.
 829 | 
 830 |         Args:
 831 |             fun: ...
 832 |             iterable: ...
 833 |             *args: static arguments passed to the function.
 834 |             timeout: ...
 835 |             show_progress: ...
 836 |             **kwargs: static keyword-arguments passed to the function.
 837 | 
 838 |         Returns:
 839 |             ...
 840 | 
 841 |         Examples:
 842 |             >>> from ppqueue import Queue
 843 |             ...
 844 |             >>> def add_nums(x: int, y: int) -> int:
 845 |             ...     return x + y
 846 |             ...
 847 |             >>> with Queue() as queue:
 848 |             ...     jobs = queue.starmap(
 849 |             ...         add_nums, [(1, 2), (3, 4)]
 850 |             ...     )
 851 |             ...
 852 |             >>> [job.result for job in jobs]
 853 |             [3, 7]
 854 |         """
 855 |         for x in iterable:
 856 |             job = Job(
 857 |                 fun=fun,
 858 |                 args=[*x, *args],
 859 |                 kwargs=dict(kwargs),
 860 |                 timeout=timeout,
 861 |             )
 862 |             self._submit(job)
 863 | 
 864 |         return self.collect(wait=True, show_progress=show_progress)
 865 | 
 866 |     def starmapkw(
 867 |         self,
 868 |         fun: Callable[..., Any],
 869 |         iterable: Sequence[dict[str, Any]],
 870 |         /,
 871 |         *args: Any,
 872 |         timeout: float = 0,
 873 |         show_progress: bool | None = None,
 874 |         **kwargs: Any,
 875 |     ) -> list[Job]:
 876 |         """Submits many jobs to the queue -- one for each dictionary in the iterable. Waits for all to finish, then returns the results.
 877 | 
 878 |         Args:
 879 |             fun: ...
 880 |             iterable: ...
 881 |             *args: static arguments passed to the function.
 882 |             timeout: ...
 883 |             show_progress: ...
 884 |             **kwargs: static keyword-arguments passed to the function.
 885 | 
 886 |         Returns:
 887 |             ...
 888 | 
 889 |         Examples:
 890 |             >>> from ppqueue import Queue
 891 |             >>> from time import sleep
 892 |             ...
 893 |             >>> def add_nums(x: int, y: int) -> int:
 894 |             ...     return x + y
 895 |             ...
 896 |             >>> with Queue() as queue:
 897 |             ...     jobs = queue.starmapkw(
 898 |             ...         add_nums, [{'x': 1, 'y': 2}, {'x': 3, 'y': 4}]
 899 |             ...     )
 900 |             ...
 901 |             >>> [job.result for job in jobs]
 902 |             [3, 7]
 903 |         """
 904 |         for x in iterable:
 905 |             job = Job(
 906 |                 fun=fun,
 907 |                 args=list(args),
 908 |                 kwargs={**x, **kwargs},
 909 |                 timeout=timeout,
 910 |             )
 911 |             self._submit(job)
 912 | 
 913 |         return self.collect(wait=True, show_progress=show_progress)
 914 | 
 915 |     def dequeue(
 916 |         self,
 917 |         *,
 918 |         wait: bool = False,
 919 |         _peek: bool = False,
 920 |         **kwargs: Any,
 921 |     ) -> Job | None:
 922 |         """Removes and returns the finished job with the highest priority from the queue.
 923 | 
 924 |         Args:
 925 |             wait: if no jobs are finished, wait for one.
 926 |             **kwargs: passed to `Queue.wait`
 927 | 
 928 |         Returns:
 929 |             ...
 930 | 
 931 |         Examples:
 932 |             >>> from ppqueue import Queue
 933 |             ...
 934 |             >>> def add_nums(x: int, y: int) -> int:
 935 |             ...     return x + y
 936 |             ...
 937 |             >>> with Queue() as queue:
 938 |             ...     for i in range(5):
 939 |             ...         _ = queue.enqueue(add_nums, args=[i, 100])
 940 |             ...
 941 |             ...     jobs = [queue.dequeue(wait=True) for _ in range(queue.size())]
 942 |             ...
 943 |             >>> [job.result for job in jobs]
 944 |             [100, 101, 102, 103, 104]
 945 |         """
 946 | 
 947 |         if self._count_finished() == 0:
 948 |             if not wait:
 949 |                 return None
 950 | 
 951 |             kwargs["show_progress"] = False
 952 |             self.wait(n=1, **kwargs)
 953 | 
 954 |         job: Job
 955 |         if _peek:
 956 |             job = self._finished_queue[0]
 957 |         else:
 958 |             job = heappop(self._finished_queue)
 959 |             self._count_output += 1
 960 | 
 961 |         return job
 962 | 
 963 |     def pop(self, *args: Any, **kwargs: Any) -> Job | None:
 964 |         """Alias for `dequeue`.
 965 | 
 966 |         Args:
 967 |             *args: ...
 968 |             **kwargs: ...
 969 | 
 970 |         Returns:
 971 |             ...
 972 | 
 973 |         Examples:
 974 |             >>> from ppqueue import Queue
 975 |             ...
 976 |             >>> def add_nums(x: int, y: int) -> int:
 977 |             ...     return x + y
 978 |             ...
 979 |             >>> with Queue() as queue:
 980 |             ...     for i in range(5):
 981 |             ...         _ = queue.put(add_nums, args=[i, 100])
 982 |             ...
 983 |             ...     jobs = [queue.pop(wait=True) for _ in range(queue.size())]
 984 |             ...
 985 |             >>> [job.result for job in jobs]
 986 |             [100, 101, 102, 103, 104]
 987 |         """
 988 |         return self.dequeue(*args, **kwargs)
 989 | 
 990 |     def peek(self, *args: Any, **kwargs: Any) -> Job | None:
 991 |         """Returns the job with the highest priority from the queue.
 992 | 
 993 |         Similar to `dequeue` / `pop`, but the job remains in the queue.
 994 | 
 995 |         Args:
 996 |             *args: ...
 997 |             **kwargs: ...
 998 | 
 999 |         Returns:
1000 |             ...
1001 | 
1002 |         Examples:
1003 |             >>> from ppqueue import Queue
1004 |             ...
1005 |             >>> def add_nums(x: int, y: int) -> int:
1006 |             ...     return x + y
1007 |             ...
1008 |             >>> with Queue() as queue:
1009 |             ...     for i in range(5):
1010 |             ...         _ = queue.put(add_nums, args=[i, 100])
1011 |             ...
1012 |             ...     print('Before:', queue.size())
1013 |             ...
1014 |             ...     job = queue.peek(wait=True)
1015 |             ...
1016 |             ...     print('After:', queue.size())
1017 |             ...
1018 |             Before: 5
1019 |             After: 5
1020 |             >>> job.result
1021 |             100
1022 |         """
1023 |         return self.dequeue(*args, **kwargs, _peek=True)
1024 | 
1025 |     def collect(self, n: int = 0, wait: bool = False, **kwargs: Any) -> list[Job]:
1026 |         """Removes and returns all finished jobs from the queue.
1027 | 
1028 |         Args:
1029 |             n: collect this many jobs (default=0, all)
1030 |             wait: If True, block until this many jobs are finished. Else, immediately return all finished.
1031 |             **kwargs: kwargs given to `Queue.wait`.
1032 | 
1033 |         Returns:
1034 |             a list of `Job` instances.
1035 | 
1036 |         Examples:
1037 |             >>> from ppqueue import Queue
1038 |             ...
1039 |             >>> def add_nums(x: int, y: int) -> int:
1040 |             ...     return x + y
1041 |             ...
1042 |             >>> with Queue() as queue:
1043 |             ...     for i in range(5):
1044 |             ...         _ = queue.enqueue(add_nums, args=[i, 100])
1045 |             ...
1046 |             ...     jobs = queue.collect(wait=True)
1047 |             ...
1048 |             >>> type(jobs)
1049 |             <class 'list'>
1050 |             >>> len(jobs)
1051 |             5
1052 |         """
1053 | 
1054 |         if wait:
1055 |             self.wait(n=n, **kwargs)
1056 | 
1057 |         n_finished: int = self._count_finished()
1058 | 
1059 |         n_to_collect: int = min(n, n_finished) if n > 0 else n_finished
1060 | 
1061 |         return list(next(self) for _ in range(n_to_collect))
1062 | 
1063 |     @property
1064 |     def max_concurrent(self) -> int:
1065 |         return self._max_concurrent
1066 | 
1067 |     @max_concurrent.setter
1068 |     def max_concurrent(self, value: int):
1069 |         self._max_concurrent = value
1070 | 
1071 |     @property
1072 |     def max_size(self) -> int:
1073 |         return self._max_size
1074 | 
1075 |     @max_size.setter
1076 |     def max_size(self, value: int) -> None:
1077 |         self._max_size = value
1078 | 
1079 |     @property
1080 |     def is_running(self) -> bool:
1081 |         return self._timer.is_enabled
1082 | 


--------------------------------------------------------------------------------
/src/ppqueue/utils.py:
--------------------------------------------------------------------------------
 1 | from __future__ import annotations
 2 | 
 3 | import logging
 4 | import os
 5 | from typing import List, Optional, TypeVar
 6 | 
 7 | 
 8 | def get_logger(name: str, level: Optional[int] = None) -> logging.Logger:
 9 |     logger = logging.getLogger(name)
10 | 
11 |     logger.setLevel(logging.INFO if level is None else level)
12 |     logger.propagate = False
13 | 
14 |     log_handler = logging.StreamHandler()
15 |     log_handler.setFormatter(
16 |         logging.Formatter("%(asctime)s %(levelname)s %(message)s", "%Y-%m-%d %H:%M:%S"),
17 |     )
18 |     logger.addHandler(log_handler)
19 | 
20 |     return logger
21 | 
22 | 
23 | def is_windows_os() -> bool:
24 |     return os.name == "nt"
25 | 
26 | 
27 | def compare(num1: int | float, num2: int | float) -> int:
28 |     """Compare two numbers.
29 | 
30 |     Args:
31 |         num1: the reference number.
32 |         num2: the comparison number.
33 | 
34 |     Returns:
35 |         0  if num1 == num2
36 |         1  if num1 > num2
37 |         -1 if num1 < num2
38 |     """
39 |     if num1 == num2:
40 |         return 0
41 | 
42 |     diff = (num1 if num1 else 0) - (num2 if num2 else 0)
43 | 
44 |     if diff == 0:
45 |         return int(diff)
46 | 
47 |     if diff > 0:
48 |         return max(1, int(diff))
49 | 
50 |     # if diff < 0:
51 |     return min(-1, int(diff))
52 | 
53 | 
54 | def compare_by(object1: object, object2: object, by: List[str], _state: int = 0) -> int:
55 |     """Compare two objects by a list of attributes.
56 | 
57 |     Attributes are compared iteratively and the comparison will
58 |     short-circuit when/if the objects are determined unequal.
59 | 
60 |     Args:
61 |         object1: the reference object.
62 |         object2: the comparison object.
63 |         by: list of attributes to compare.
64 | 
65 |     Returns:
66 |         0  if object1 == object2
67 |         1  if object1 > object2
68 |         -1 if object1 < object2
69 |     """
70 |     if _state and _state >= len(by):
71 |         # if we've compared all attributes,
72 |         # these objects are considered equal.
73 |         return 0
74 | 
75 |     diff: int = compare(getattr(object1, by[_state]), getattr(object2, by[_state]))
76 | 
77 |     if diff == 0:
78 |         # if equal, compare next attribute.
79 |         return compare_by(object1, object2, by=by, _state=_state + 1)
80 | 
81 |     return diff
82 | 
83 | 
84 | T = TypeVar("T")
85 | 
86 | 
87 | def dedupe_list(l: list[T]) -> list[T]:
88 |     return list(dict.fromkeys(l).keys())
89 | 


--------------------------------------------------------------------------------
/tests/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/fresh2dev/ppqueue/7cf96a7b2110b89df2ae13364943374a299c4e12/tests/__init__.py


--------------------------------------------------------------------------------
/tests/common.py:
--------------------------------------------------------------------------------
  1 | import random
  2 | from typing import List, Tuple
  3 | from unittest import TestCase
  4 | 
  5 | import ppqueue
  6 | from ppqueue.utils import get_logger
  7 | 
  8 | LOG = get_logger(__name__)
  9 | 
 10 | 
 11 | # pylint: disable=protected-access
 12 | 
 13 | 
 14 | def return_me(x):
 15 |     return x
 16 | 
 17 | 
 18 | def reciprocal(x):
 19 |     return 1 / x
 20 | 
 21 | 
 22 | def get_sample_data(n: int = 31) -> Tuple[int, ...]:
 23 |     sample: List[int] = list(range(n))
 24 |     random.shuffle(sample)
 25 |     return tuple(sample)
 26 | 
 27 | 
 28 | class PPQueueTestCases(object):
 29 |     def setUp(self):
 30 |         self.queue: ppqueue.Queue
 31 |         self.input: Tuple[int, ...]
 32 |         raise NotImplementedError()
 33 | 
 34 |     def test_priority(self):
 35 |         TestCase().assertTrue(self.queue.is_running)
 36 | 
 37 |         self.queue.stop()
 38 | 
 39 |         TestCase().assertFalse(self.queue.is_running)
 40 | 
 41 |         for i, x in enumerate(self.input):
 42 |             self.queue.enqueue(
 43 |                 return_me,
 44 |                 args=x,
 45 |                 priority=-i,
 46 |             )  # should result in reversed inputs.
 47 | 
 48 |         self.queue.start()
 49 | 
 50 |         TestCase().assertTrue(self.queue.is_running)
 51 | 
 52 |         self.queue.wait()
 53 | 
 54 |         output = tuple(job.result for job in self.queue.collect())
 55 | 
 56 |         TestCase().assertTupleEqual(tuple(reversed(self.input)), output)
 57 | 
 58 |         TestCase().assertEqual(len(self.input), self.queue._count_input)
 59 |         TestCase().assertEqual(self.queue._count_input, self.queue._count_output)
 60 | 
 61 |     def test_map(self):
 62 |         job_data = self.queue.map(return_me, self.input)
 63 | 
 64 |         output = tuple(job.result for job in job_data)
 65 | 
 66 |         TestCase().assertTupleEqual(self.input, output)
 67 | 
 68 |         TestCase().assertEqual(len(self.input), self.queue._count_input)
 69 |         TestCase().assertEqual(self.queue._count_input, self.queue._count_output)
 70 | 
 71 |     def test_starmap(self):
 72 |         job_data = self.queue.starmap(return_me, [[x] for x in self.input])
 73 | 
 74 |         output = tuple(job.result for job in job_data)
 75 | 
 76 |         TestCase().assertTupleEqual(self.input, output)
 77 | 
 78 |         TestCase().assertEqual(len(self.input), self.queue._count_input)
 79 |         TestCase().assertEqual(self.queue._count_input, self.queue._count_output)
 80 | 
 81 |     def test_starmapkw(self):
 82 |         job_data = self.queue.starmapkw(
 83 |             return_me,
 84 |             [{"x": x} for x in self.input],
 85 |         )
 86 | 
 87 |         output = tuple(job.result for job in job_data)
 88 | 
 89 |         TestCase().assertTupleEqual(self.input, output)
 90 | 
 91 |         TestCase().assertEqual(len(self.input), self.queue._count_input)
 92 |         TestCase().assertEqual(self.queue._count_input, self.queue._count_output)
 93 | 
 94 |     def test_groups(self):
 95 |         for i, x in enumerate(self.input):
 96 |             self.queue.enqueue(
 97 |                 return_me,
 98 |                 args=x,
 99 |                 group=i % self.queue.max_concurrent,
100 |             )  # returns in order
101 | 
102 |         self.queue.wait()
103 |         output = tuple(x.result for x in self.queue.collect())
104 | 
105 |         TestCase().assertTupleEqual(self.input, output)
106 | 
107 |         TestCase().assertEqual(len(self.input), self.queue._count_input)
108 |         TestCase().assertEqual(self.queue._count_input, self.queue._count_output)
109 | 
110 |     def test_group_error(self):
111 |         this_input = list(self.input)
112 |         this_input[int(len(this_input) / 2)] = 0
113 |         for i, x in enumerate(this_input):
114 |             self.queue.enqueue(
115 |                 reciprocal,
116 |                 x,
117 |                 group=i % 3,
118 |                 suppress_errors=True,
119 |                 skip_on_group_error=True,
120 |             )
121 |         self.queue.wait()
122 |         output = self.queue.collect()
123 | 
124 |         TestCase().assertGreater(
125 |             len(self.input),
126 |             len([x for x in output if x.start_timestamp is not None]),
127 |         )
128 | 
129 |     def test_size(self):
130 |         for x in self.input:
131 |             self.queue.enqueue(return_me, args=x)
132 | 
133 |         sizes = [
134 |             self.queue.size()
135 |             for _ in range(len(self.input))
136 |             for _ in [self.queue.dequeue(wait=True)]
137 |         ]
138 | 
139 |         # numbers in `sizes` should decrement by 1 until reaching 0.
140 |         TestCase().assertListEqual(sizes, list(reversed(range(len(self.input)))))
141 | 
142 |         TestCase().assertEqual(len(self.input), self.queue._count_input)
143 |         TestCase().assertEqual(self.queue._count_input, self.queue._count_output)
144 | 
145 |     def test_terminate(self):
146 |         for x in self.input:
147 |             self.queue.enqueue(return_me, args=x, priority=x)
148 |         self.queue._stop_all()
149 |         output = self.queue.collect()
150 | 
151 |         TestCase().assertListEqual(
152 |             list(sorted([x.idx for x in output])),
153 |             list(range(1, len(self.input) + 1)),
154 |         )
155 | 
156 |     def test_empty(self):
157 |         TestCase().assertEqual(self.queue.size(), 0)
158 |         TestCase().assertTrue(self.queue.is_empty())
159 | 


--------------------------------------------------------------------------------
/tests/test_plot.py:
--------------------------------------------------------------------------------
 1 | import threading
 2 | import unittest
 3 | from typing import Tuple
 4 | from unittest.case import TestCase
 5 | 
 6 | import ppqueue
 7 | from ppqueue.plot import plot_jobs
 8 | from ppqueue.utils import get_logger
 9 | 
10 | from .common import get_sample_data
11 | 
12 | LOG = get_logger(__name__)
13 | 
14 | 
15 | def return_me(x):
16 |     return x
17 | 
18 | 
19 | class TestPlot(unittest.TestCase):
20 |     def setUp(self):
21 |         LOG.info(self.id())
22 |         self.input: Tuple[int] = get_sample_data()
23 | 
24 |     def test_plot(self):
25 |         with ppqueue.Queue(engine=threading.Thread, max_concurrent=3) as queue:
26 |             job_data = queue.map(return_me, self.input)
27 | 
28 |         TestCase().assertEqual(len(self.input), len(job_data))
29 | 
30 |         plot = plot_jobs(job_data)
31 | 
32 |         TestCase().assertIsNotNone(plot)
33 | 


--------------------------------------------------------------------------------
/tests/test_queue_processes.py:
--------------------------------------------------------------------------------
 1 | import unittest
 2 | from multiprocessing import Process
 3 | from typing import Tuple
 4 | 
 5 | from ppqueue import Queue
 6 | from ppqueue.utils import get_logger, is_windows_os
 7 | 
 8 | from .common import PPQueueTestCases, get_sample_data
 9 | 
10 | LOG = get_logger(__name__)
11 | 
12 | 
13 | @unittest.skipIf(
14 |     is_windows_os(),
15 |     "multiprocessing tests are reliable on a windows os; skipping.",
16 | )
17 | class TestProcessing(unittest.TestCase, PPQueueTestCases):
18 |     def setUp(self):
19 |         LOG.info(self.id())
20 |         self.input: Tuple[int, ...] = get_sample_data()
21 |         self.queue = Queue(engine=Process, max_concurrent=3)
22 | 
23 |     def tearDown(self):
24 |         self.queue.dispose()
25 |         self.assertFalse(self.queue.is_running)
26 |         del self.queue
27 | 
28 | 
29 | if __name__ == "__main__":
30 |     unittest.main()
31 | 


--------------------------------------------------------------------------------
/tests/test_queue_threads.py:
--------------------------------------------------------------------------------
 1 | import unittest
 2 | from threading import Thread
 3 | from typing import Tuple
 4 | 
 5 | from ppqueue import Queue
 6 | from ppqueue.utils import get_logger
 7 | 
 8 | from .common import PPQueueTestCases, get_sample_data
 9 | 
10 | LOG = get_logger(__name__)
11 | 
12 | 
13 | class TestThreading(unittest.TestCase, PPQueueTestCases):
14 |     def setUp(self):
15 |         LOG.info(self.id())
16 |         self.input: Tuple[int, ...] = get_sample_data()
17 |         self.queue = Queue(engine=Thread, max_concurrent=3)
18 | 
19 |     def tearDown(self):
20 |         self.queue.dispose()
21 |         self.assertFalse(self.queue.is_running)
22 |         del self.queue
23 | 
24 | 
25 | if __name__ == "__main__":
26 |     unittest.main()
27 | 


--------------------------------------------------------------------------------
/tests/test_utils.py:
--------------------------------------------------------------------------------
 1 | import logging
 2 | import unittest
 3 | from unittest.case import TestCase
 4 | 
 5 | from ppqueue import Job
 6 | from ppqueue.utils import compare, compare_by, dedupe_list, get_logger
 7 | 
 8 | LOG = get_logger(__name__)
 9 | 
10 | 
11 | class TestUtils(unittest.TestCase):
12 |     def setUp(self) -> None:
13 |         LOG.info(self.id())
14 | 
15 |     def tearDown(self) -> None:
16 |         pass
17 | 
18 |     def test_get_logger(self):
19 |         name: str = "test"
20 |         level: int = logging.DEBUG
21 |         logger: logging.Logger = get_logger(name=name, level=level)
22 |         TestCase().assertEqual(name, logger.name)
23 |         TestCase().assertEqual(level, logger.level)
24 | 
25 |     def test_compare(self):
26 |         TestCase().assertEqual(0, compare(None, None))
27 |         TestCase().assertEqual(0, compare(1, 1))
28 |         TestCase().assertEqual(-1, compare(1, 2))
29 |         TestCase().assertEqual(1, compare(3, 2))
30 | 
31 |     def test_compare_by(self):
32 |         obj1: Job = Job(fun=None)
33 |         obj2: Job = Job(fun=None)
34 | 
35 |         obj1.priority = 1
36 |         obj2.priority = 1
37 |         obj1.idx = 1
38 |         obj2.idx = 1
39 |         TestCase().assertEqual(0, compare_by(obj1, obj2, by=["priority", "idx"]))
40 | 
41 |         obj2.idx = 2
42 |         TestCase().assertEqual(-1, compare_by(obj1, obj2, by=["priority", "idx"]))
43 | 
44 |     def test_dedupe_list(self):
45 |         TestCase().assertEqual([1, 2, 3], dedupe_list([1, 2, 3]))
46 |         TestCase().assertEqual([1, 2, 3], dedupe_list([1, 2, 3, 2, 1]))
47 | 


--------------------------------------------------------------------------------