├── .coveragerc ├── .github ├── dependabot.yml └── workflows │ └── ci.yaml ├── .gitignore ├── .mailmap ├── CONTRIBUTING.rst ├── HACKING.rst ├── LICENSE ├── README.rst ├── babel.cfg ├── cotyledon ├── __init__.py ├── _service.py ├── _service_manager.py ├── _utils.py ├── oslo_config_glue.py └── tests │ ├── __init__.py │ ├── base.py │ ├── examples.py │ ├── test_functional.py │ └── test_unit.py ├── doc └── source │ ├── api.rst │ ├── conf.py │ ├── contributing.rst │ ├── examples.rst │ ├── index.rst │ ├── installation.rst │ ├── non-posix-support.rst │ └── oslo-service-migration.rst ├── pyproject.toml ├── release.sh └── tox.ini /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | source = cotyledon 4 | 5 | [report] 6 | ignore_errors = True 7 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: pip 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | time: "09:00" 8 | timezone: Europe/Paris 9 | open-pull-requests-limit: 10 10 | - package-ecosystem: github-actions 11 | directory: / 12 | schedule: 13 | interval: daily 14 | time: "09:00" 15 | timezone: Europe/Paris 16 | -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: CI 2 | permissions: read-all 3 | 4 | on: 5 | workflow_dispatch: 6 | push: 7 | branches: 8 | - main 9 | pull_request: 10 | branches: 11 | - main 12 | 13 | jobs: 14 | ci: 15 | timeout-minutes: 7 16 | runs-on: ubuntu-22.04 17 | strategy: 18 | matrix: 19 | include: 20 | - version: "3.9" 21 | target: py39 22 | - version: "3.10" 23 | target: py310 24 | - version: "3.11" 25 | target: py311 26 | - version: "3.12" 27 | target: py312 28 | - version: "3.12" 29 | target: pep8 30 | steps: 31 | - name: Checkout 🛎️ 32 | uses: actions/checkout@v4 33 | with: 34 | fetch-depth: 0 35 | 36 | - name: Setup Python 🔧 37 | uses: actions/setup-python@v5 38 | with: 39 | python-version: ${{ matrix.version }} 40 | 41 | - name: Build 🔧 & Test 🔍 42 | run: | 43 | pip install tox 44 | tox -e ${{ matrix.target }} 45 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | 3 | # C extensions 4 | *.so 5 | 6 | # Packages 7 | *.egg* 8 | *.egg-info 9 | dist 10 | build 11 | eggs 12 | parts 13 | bin 14 | var 15 | sdist 16 | develop-eggs 17 | .installed.cfg 18 | lib 19 | lib64 20 | 21 | # Installer logs 22 | pip-log.txt 23 | 24 | # Unit test / coverage reports 25 | cover/ 26 | .coverage* 27 | !.coveragerc 28 | .tox 29 | nosetests.xml 30 | .testrepository 31 | .venv 32 | 33 | # Translations 34 | *.mo 35 | 36 | # Mr Developer 37 | .mr.developer.cfg 38 | .project 39 | .pydevproject 40 | 41 | # Complexity 42 | output/*.html 43 | output/*/index.html 44 | 45 | # Sphinx 46 | doc/build 47 | 48 | # pbr generates these 49 | AUTHORS 50 | ChangeLog 51 | 52 | # Editors 53 | *~ 54 | .*.swp 55 | .*sw? 56 | 57 | .pytest_cache 58 | -------------------------------------------------------------------------------- /.mailmap: -------------------------------------------------------------------------------- 1 | # Format is: 2 | # 3 | # 4 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | Bugs should be filed on Github: https://github.com/sileht/cotyledon/issues 2 | 3 | Contribution can be via Github pull requests: https://github.com/sileht/cotyledon/pulls 4 | 5 | -------------------------------------------------------------------------------- /HACKING.rst: -------------------------------------------------------------------------------- 1 | cotyledon Style Commandments 2 | =============================================== 3 | 4 | Read the OpenStack Style Commandments http://docs.openstack.org/developer/hacking/ 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | =============================== 2 | Cotyledon 3 | =============================== 4 | 5 | .. image:: https://img.shields.io/pypi/v/cotyledon.svg 6 | :target: https://pypi.python.org/pypi/cotyledon/ 7 | :alt: Latest Version 8 | 9 | .. image:: https://img.shields.io/pypi/dm/cotyledon.svg 10 | :target: https://pypi.python.org/pypi/cotyledon/ 11 | :alt: Downloads 12 | 13 | Cotyledon provides a framework for defining long-running services. 14 | 15 | It provides handling of Unix signals, spawning of workers, supervision of 16 | children processes, daemon reloading, sd-notify, rate limiting for worker 17 | spawning, and more. 18 | 19 | * Free software: Apache license 20 | * Documentation: http://cotyledon.readthedocs.org/ 21 | * Source: https://github.com/sileht/cotyledon 22 | * Bugs: https://github.com/sileht/cotyledon/issues 23 | 24 | Why Cotyledon 25 | ------------- 26 | 27 | This library is mainly used in OpenStack Telemetry projects, in replacement of 28 | *oslo.service*. However, as *oslo.service* depends on *eventlet*, a different 29 | library was needed for project that do not need it. When an application do not 30 | monkeypatch the Python standard library anymore, greenlets do not in timely 31 | fashion. That made other libraries such as `Tooz 32 | `_ or `oslo.messaging 33 | `_ to fail with e.g. their 34 | heartbeat systems. Also, processes would not exist as expected due to 35 | greenpipes never being processed. 36 | 37 | *oslo.service* is actually written on top of eventlet to provide two main 38 | features: 39 | 40 | * periodic tasks 41 | * workers processes management 42 | 43 | The first feature was replaced by another library called `futurist 44 | `_ and the second feature is 45 | superseded by *Cotyledon*. 46 | 47 | Unlike *oslo.service*, **Cotyledon** have: 48 | 49 | * The same code path when workers=1 and workers>=2 50 | * Reload API (on SIGHUP) hooks work in case of you don't want to restarting children 51 | * A separated API for children process termination and for master process termination 52 | * Seatbelt to ensure only one service workers manager run at a time. 53 | * Is signal concurrency safe. 54 | * Support non posix platform, because it's built on top of multiprocessing module 55 | instead of os.fork 56 | * Provide functional testing 57 | 58 | And doesn't: 59 | 60 | * facilitate the creation of wsgi application (sockets sharing between parent 61 | and children process). Because too many wsgi webserver already exists. 62 | 63 | *oslo.service* being impossible to fix and bringing an heavy dependency on 64 | eventlet, **Cotyledon** appeared. 65 | -------------------------------------------------------------------------------- /babel.cfg: -------------------------------------------------------------------------------- 1 | [python: **.py] 2 | 3 | -------------------------------------------------------------------------------- /cotyledon/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | 14 | # Public API 15 | from cotyledon._service import Service 16 | from cotyledon._service_manager import ServiceManager 17 | 18 | 19 | __all__ = ["Service", "ServiceManager"] 20 | -------------------------------------------------------------------------------- /cotyledon/_service.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | import contextlib 14 | import logging 15 | import os 16 | import random 17 | import signal 18 | import sys 19 | import threading 20 | 21 | from cotyledon import _utils 22 | 23 | 24 | LOG = logging.getLogger(__name__) 25 | 26 | 27 | class Service: 28 | """Base class for a service 29 | 30 | This class will be executed in a new child process/worker 31 | :py:class:`ServiceWorker` of a :py:class:`ServiceManager`. It registers 32 | signals to manager the reloading and the ending of the process. 33 | 34 | Methods :py:meth:`run`, :py:meth:`terminate` and :py:meth:`reload` are 35 | optional. 36 | """ 37 | 38 | name = None 39 | """Service name used in the process title and the log messages in 40 | additional of the worker_id.""" 41 | 42 | graceful_shutdown_timeout = None 43 | """Timeout after which a gracefully shutdown service will exit. zero means 44 | endless wait. None means same as ServiceManager that launch the service""" 45 | 46 | def __init__(self, worker_id) -> None: 47 | """Create a new Service 48 | 49 | :param worker_id: the identifier of this service instance 50 | :type worker_id: int 51 | 52 | The identifier of the worker can be used for workload repartition 53 | because it's consistent and always the same. 54 | 55 | For example, if the number of workers for this service is 3, 56 | one will got 0, the second got 1 and the last got 2. 57 | if worker_id 1 died, the new spawned process will got 1 again. 58 | """ 59 | super().__init__() 60 | self._initialize(worker_id) 61 | 62 | def _initialize(self, worker_id) -> None: 63 | if getattr(self, "_initialized", False): 64 | return 65 | self._initialized = True 66 | 67 | if self.name is None: 68 | self.name = self.__class__.__name__ 69 | self.worker_id = worker_id 70 | self.pid = os.getpid() 71 | 72 | self._signal_lock = threading.Lock() 73 | 74 | # Only used by oslo_config_glue for now, so we don't need 75 | # to have a list of hook 76 | self._on_reload_internal_hook = self._noop_hook 77 | 78 | def _noop_hook(self, service) -> None: 79 | pass 80 | 81 | def terminate(self) -> None: 82 | """Gracefully shutdown the service 83 | 84 | This method will be executed when the Service has to shutdown cleanly. 85 | 86 | If not implemented the process will just end with status 0. 87 | 88 | To customize the exit code, the :py:class:`SystemExit` exception can be 89 | used. 90 | 91 | Any exceptions raised by this method will be logged and the worker will 92 | exit with status 1. 93 | """ 94 | 95 | def reload(self) -> None: # noqa: PLR6301 96 | """Reloading of the service 97 | 98 | This method will be executed when the Service receives a SIGHUP. 99 | 100 | If not implemented the process will just end with status 0 and 101 | :py:class:`ServiceRunner` will start a new fresh process for this 102 | service with the same worker_id. 103 | 104 | Any exceptions raised by this method will be logged and the worker will 105 | exit with status 1. 106 | """ 107 | os.kill(os.getpid(), signal.SIGTERM) 108 | 109 | def run(self) -> None: 110 | """Method representing the service activity 111 | 112 | If not implemented the process will just wait to receive an ending 113 | signal. 114 | 115 | This method is ran into the thread and can block or return as needed 116 | 117 | Any exceptions raised by this method will be logged and the worker will 118 | exit with status 1. 119 | """ 120 | 121 | # Helper to run application methods in a safety way when signal are 122 | # received 123 | 124 | def _reload(self) -> None: 125 | with _utils.exit_on_exception(): 126 | if self._signal_lock.acquire(blocking=False): 127 | try: 128 | self._on_reload_internal_hook(self) 129 | self.reload() 130 | finally: 131 | self._signal_lock.release() 132 | 133 | def _terminate(self) -> None: 134 | with _utils.exit_on_exception(), self._signal_lock: 135 | self.terminate() 136 | sys.exit(0) 137 | 138 | def _run(self) -> None: 139 | with _utils.exit_on_exception(): 140 | self.run() 141 | 142 | 143 | class ServiceConfig: 144 | def __init__(self, service_id, service, workers, args, kwargs) -> None: 145 | self.service = service 146 | self.workers = workers 147 | self.args = args 148 | self.kwargs = kwargs 149 | self.service_id = service_id 150 | 151 | 152 | class ServiceWorker(_utils.SignalManager): 153 | """Service Worker Wrapper 154 | 155 | This represents the child process spawned by ServiceManager 156 | 157 | All methods implemented here, must run in the main threads 158 | """ 159 | 160 | @classmethod 161 | def create_and_wait(cls, *args, **kwargs) -> None: 162 | sw = cls(*args, **kwargs) 163 | sw.wait_forever() 164 | 165 | def __init__( # noqa: PLR0917, PLR0913 166 | self, 167 | config, 168 | service_id, 169 | worker_id, 170 | parent_pipe, 171 | started_hooks, 172 | graceful_shutdown_timeout, 173 | ) -> None: 174 | super().__init__() 175 | self._ready = threading.Event() 176 | _utils.spawn(self._watch_parent_process, parent_pipe) 177 | 178 | # Reseed random number generator 179 | random.seed() 180 | 181 | args = () if config.args is None else config.args 182 | kwargs = {} if config.kwargs is None else config.kwargs 183 | self.service = config.service(worker_id, *args, **kwargs) 184 | self.service._initialize(worker_id) # noqa: SLF001 185 | if self.service.graceful_shutdown_timeout is None: 186 | self.service.graceful_shutdown_timeout = graceful_shutdown_timeout 187 | 188 | self.title = f"{self.service.name}({worker_id}) [{os.getpid()}]" 189 | 190 | # Set process title 191 | _utils.setproctitle( 192 | f"{_utils.get_process_name()}: {self.service.name} worker({worker_id})", 193 | ) 194 | 195 | # We are ready tell them 196 | self._ready.set() 197 | _utils.run_hooks( 198 | "new_worker", 199 | started_hooks, 200 | service_id, 201 | worker_id, 202 | self.service, 203 | ) 204 | 205 | def _watch_parent_process(self, parent_pipe) -> None: 206 | # This will block until the write end is closed when the parent 207 | # dies unexpectedly 208 | parent_pipe[1].close() 209 | with contextlib.suppress(EOFError): 210 | parent_pipe[0].recv() 211 | 212 | if self._ready.is_set(): 213 | LOG.info("Parent process has died unexpectedly, %s exiting", self.title) 214 | if os.name == "posix": 215 | os.kill(os.getpid(), signal.SIGTERM) 216 | else: 217 | # Fallback to process signal later 218 | self._signals_received.appendleft(signal.SIGTERM) 219 | else: 220 | os._exit(0) 221 | 222 | def _alarm(self) -> None: 223 | LOG.info( 224 | "Graceful shutdown timeout (%d) exceeded, exiting %s now.", 225 | self.service.graceful_shutdown_timeout, 226 | self.title, 227 | ) 228 | os._exit(1) 229 | 230 | def _fast_exit(self) -> None: 231 | LOG.info( 232 | "Caught SIGINT signal, instantaneous exiting of service %s", 233 | self.title, 234 | ) 235 | os._exit(1) 236 | 237 | def _on_signal_received(self, sig) -> None: 238 | # Code below must not block to return to select.select() and catch 239 | # next signals 240 | if sig == _utils.SIGALRM: 241 | self._alarm() 242 | if sig == signal.SIGINT: 243 | self._fast_exit() 244 | elif sig == signal.SIGTERM: 245 | LOG.info( 246 | "Caught SIGTERM signal, graceful exiting of service %s", 247 | self.title, 248 | ) 249 | 250 | if self.service.graceful_shutdown_timeout > 0: 251 | if os.name == "posix": 252 | signal.alarm(self.service.graceful_shutdown_timeout) 253 | else: 254 | threading.Timer( 255 | self.service.graceful_shutdown_timeout, 256 | self._alarm, 257 | ).start() 258 | _utils.spawn(self.service._terminate) # noqa: SLF001 259 | elif sig == _utils.SIGHUP: 260 | _utils.spawn(self.service._reload) # noqa: SLF001 261 | 262 | def wait_forever(self) -> None: 263 | LOG.debug("Run service %s", self.title) 264 | _utils.spawn(self.service._run) # noqa: SLF001 265 | super()._wait_forever() 266 | -------------------------------------------------------------------------------- /cotyledon/_service_manager.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | import collections 14 | import contextlib 15 | import logging 16 | import multiprocessing 17 | import os 18 | import signal 19 | import socket 20 | import sys 21 | import threading 22 | import time 23 | import uuid 24 | 25 | from cotyledon import _service 26 | from cotyledon import _utils 27 | 28 | 29 | LOG = logging.getLogger(__name__) 30 | 31 | 32 | class ServiceManager(_utils.SignalManager): 33 | """Manage lifetimes of services 34 | 35 | :py:class:`ServiceManager` acts as a master process that controls the 36 | lifetime of children processes and restart them if they die unexpectedly. 37 | It also propagate some signals (SIGTERM, SIGALRM, SIGINT and SIGHUP) to 38 | them. 39 | 40 | Each child process (:py:class:`ServiceWorker`) runs an instance of 41 | a :py:class:`Service`. 42 | 43 | An application must create only one :py:class:`ServiceManager` class and 44 | use :py:meth:`ServiceManager.run()` as main loop of the application. 45 | 46 | 47 | 48 | Usage:: 49 | 50 | class MyService(Service): 51 | def __init__(self, worker_id, myconf): 52 | super(MyService, self).__init__(worker_id) 53 | preparing_my_job(myconf) 54 | self.running = True 55 | 56 | def run(self): 57 | while self.running: 58 | do_my_job() 59 | 60 | def terminate(self): 61 | self.running = False 62 | gracefully_stop_my_jobs() 63 | 64 | def reload(self): 65 | restart_my_job() 66 | 67 | 68 | class MyManager(ServiceManager): 69 | def __init__(self): 70 | super(MyManager, self).__init__() 71 | self.register_hooks(on_reload=self.reload) 72 | 73 | conf = {'foobar': 2} 74 | self.service_id = self.add(MyService, 5, conf) 75 | 76 | def reload(self): 77 | self.reconfigure(self.service_id, 10) 78 | 79 | MyManager().run() 80 | 81 | This will create 5 children processes running the service MyService. 82 | 83 | """ 84 | 85 | _process_runner_already_created = False 86 | 87 | def __init__(self, wait_interval=0.01, graceful_shutdown_timeout=60) -> None: 88 | """Creates the ServiceManager object 89 | 90 | :param wait_interval: time between each new process spawn 91 | :type wait_interval: float 92 | 93 | """ 94 | 95 | if self._process_runner_already_created: 96 | msg = "Only one instance of ServiceManager per application is allowed" 97 | raise RuntimeError(msg) 98 | ServiceManager._process_runner_already_created = True 99 | super().__init__() 100 | 101 | # We use OrderedDict to start services in adding order 102 | self._services = collections.OrderedDict() 103 | self._running_services = collections.defaultdict(dict) 104 | self._forktimes = [] 105 | self._graceful_shutdown_timeout = graceful_shutdown_timeout 106 | self._wait_interval = wait_interval 107 | 108 | self._dead = threading.Event() 109 | # NOTE(sileht): Set it on startup, so first iteration 110 | # will spawn initial workers 111 | self._got_sig_chld = threading.Event() 112 | self._got_sig_chld.set() 113 | 114 | self._child_supervisor = None 115 | 116 | self._hooks = { 117 | "terminate": [], 118 | "reload": [], 119 | "new_worker": [], 120 | "dead_worker": [], 121 | } 122 | 123 | _utils.setproctitle( 124 | "{}: master process [{}]".format( 125 | _utils.get_process_name(), 126 | " ".join(sys.argv), 127 | ), 128 | ) 129 | 130 | # Try to create a session id if possible 131 | with contextlib.suppress(OSError, AttributeError): 132 | os.setsid() 133 | 134 | self._death_detection_pipe = multiprocessing.Pipe(duplex=False) 135 | 136 | if os.name == "posix": 137 | signal.signal(signal.SIGCHLD, self._signal_catcher) 138 | 139 | def register_hooks( 140 | self, 141 | on_terminate=None, 142 | on_reload=None, 143 | on_new_worker=None, 144 | on_dead_worker=None, 145 | ) -> None: 146 | """Register hook methods 147 | 148 | This can be callable multiple times to add more hooks, hooks are 149 | executed in added order. If a hook raised an exception, next hooks 150 | will be not executed. 151 | 152 | :param on_terminate: method called on SIGTERM 153 | :type on_terminate: callable() 154 | :param on_reload: method called on SIGHUP 155 | :type on_reload: callable() 156 | :param on_new_worker: method called in the child process when this one 157 | is ready 158 | :type on_new_worker: callable(service_id, worker_id, service_obj) 159 | :param on_new_worker: method called when a child died 160 | :type on_new_worker: callable(service_id, worker_id, exit_code) 161 | 162 | If window support is planned, hooks callable must support 163 | to be pickle.pickle(). See CPython multiprocessing module documentation 164 | for more detail. 165 | """ 166 | 167 | if on_terminate is not None: 168 | _utils.check_callable(on_terminate, "on_terminate") 169 | self._hooks["terminate"].append(on_terminate) 170 | if on_reload is not None: 171 | _utils.check_callable(on_reload, "on_reload") 172 | self._hooks["reload"].append(on_reload) 173 | if on_new_worker is not None: 174 | _utils.check_callable(on_new_worker, "on_new_worker") 175 | self._hooks["new_worker"].append(on_new_worker) 176 | if on_dead_worker is not None: 177 | _utils.check_callable(on_dead_worker, "on_dead_worker") 178 | self._hooks["dead_worker"].append(on_dead_worker) 179 | 180 | def _run_hooks(self, name, *args, **kwargs) -> None: 181 | _utils.run_hooks(name, self._hooks[name], *args, **kwargs) 182 | 183 | def add(self, service, workers=1, args=None, kwargs=None): 184 | """Add a new service to the ServiceManager 185 | 186 | :param service: callable that return an instance of :py:class:`Service` 187 | :type service: callable 188 | :param workers: number of processes/workers for this service 189 | :type workers: int 190 | :param args: additional positional arguments for this service 191 | :type args: tuple 192 | :param kwargs: additional keywoard arguments for this service 193 | :type kwargs: dict 194 | 195 | :return: a service id 196 | :rtype: uuid.uuid4 197 | """ 198 | _utils.check_callable(service, "service") 199 | _utils.check_workers(workers, 1) 200 | service_id = uuid.uuid4() 201 | self._services[service_id] = _service.ServiceConfig( 202 | service_id, 203 | service, 204 | workers, 205 | args, 206 | kwargs, 207 | ) 208 | return service_id 209 | 210 | def reconfigure(self, service_id, workers) -> None: 211 | """Reconfigure a service registered in ServiceManager 212 | 213 | :param service_id: the service id 214 | :type service_id: uuid.uuid4 215 | :param workers: number of processes/workers for this service 216 | :type workers: int 217 | :raises: ValueError 218 | """ 219 | try: 220 | sc = self._services[service_id] 221 | except KeyError: 222 | msg = f"{service_id} service id doesn't exists" 223 | raise ValueError(msg) from None 224 | else: 225 | _utils.check_workers(workers, minimum=(1 - sc.workers)) 226 | sc.workers = workers 227 | # Reset forktimes to respawn services quickly 228 | self._forktimes = [] 229 | 230 | def run(self) -> None: 231 | """Start and supervise services workers 232 | 233 | This method will start and supervise all children processes 234 | until the master process asked to shutdown by a SIGTERM. 235 | 236 | All spawned processes are part of the same unix process group. 237 | """ 238 | 239 | self._systemd_notify_once() 240 | self._child_supervisor = _utils.spawn(self._child_supervisor_thread) 241 | self._wait_forever() 242 | 243 | def _child_supervisor_thread(self) -> None: 244 | while not self._dead.is_set(): 245 | self._got_sig_chld.wait() 246 | self._got_sig_chld.clear() 247 | 248 | info = self._get_last_worker_died() 249 | while info is not None: 250 | if self._dead.is_set(): 251 | return 252 | service_id, worker_id = info 253 | self._start_worker(service_id, worker_id) 254 | info = self._get_last_worker_died() 255 | 256 | self._adjust_workers() 257 | 258 | def _on_signal_received(self, sig) -> None: 259 | if sig == _utils.SIGALRM: 260 | self._alarm() 261 | elif sig == signal.SIGINT: 262 | self._fast_exit() 263 | elif sig == signal.SIGTERM: 264 | self._shutdown() 265 | elif sig == _utils.SIGHUP: 266 | self._reload() 267 | elif sig == _utils.SIGCHLD: 268 | self._got_sig_chld.set() 269 | else: 270 | LOG.debug("unhandled signal %s", sig) 271 | 272 | def _alarm(self) -> None: 273 | self._fast_exit( 274 | reason="Graceful shutdown timeout exceeded, " 275 | "instantaneous exiting of master process", 276 | ) 277 | 278 | def _reload(self) -> None: 279 | """reload all children 280 | 281 | posix only 282 | """ 283 | self._run_hooks("reload") 284 | 285 | # Reset forktimes to respawn services quickly 286 | self._forktimes = [] 287 | signal.signal(signal.SIGHUP, signal.SIG_IGN) 288 | os.killpg(0, signal.SIGHUP) 289 | signal.signal(signal.SIGHUP, self._signal_catcher) 290 | 291 | def shutdown(self) -> None: 292 | LOG.info("Manager shutdown requested") 293 | os.kill(os.getpid(), signal.SIGTERM) 294 | self._dead.wait() 295 | 296 | def _shutdown(self) -> None: 297 | LOG.info("Caught SIGTERM signal, graceful exiting of master process") 298 | signal.signal(signal.SIGTERM, signal.SIG_IGN) 299 | 300 | if self._graceful_shutdown_timeout > 0: 301 | if os.name == "posix": 302 | signal.alarm(self._graceful_shutdown_timeout) 303 | else: 304 | threading.Timer(self._graceful_shutdown_timeout, self._alarm).start() 305 | 306 | # NOTE(sileht): Stop the child supervisor 307 | self._dead.set() 308 | self._got_sig_chld.set() 309 | self._child_supervisor.join() 310 | 311 | # NOTE(sileht): During startup if we receive SIGTERM, python 312 | # multiprocess may fork the process after we send the killpg(0) 313 | # To workaround the issue we sleep a bit, so multiprocess can finish 314 | # its work. 315 | time.sleep(0.1) 316 | 317 | self._run_hooks("terminate") 318 | 319 | LOG.debug("Killing services with signal SIGTERM") 320 | if os.name == "posix": 321 | os.killpg(0, signal.SIGTERM) 322 | 323 | LOG.debug("Waiting services to terminate") 324 | for processes in self._running_services.values(): 325 | for process in processes: 326 | if os.name != "posix": 327 | # NOTE(sileht): we don't have killpg so we 328 | # kill all known processes instead 329 | # NOTE(sileht): We should use CTRL_BREAK_EVENT on windows 330 | # when CREATE_NEW_PROCESS_GROUP will be set on child 331 | # process 332 | process.terminate() 333 | process.join() 334 | 335 | LOG.debug("Shutdown finish") 336 | sys.exit(0) 337 | 338 | def _adjust_workers(self) -> None: 339 | for service_id, conf in self._services.items(): 340 | running_workers = len(self._running_services[service_id]) 341 | if running_workers < conf.workers: 342 | for worker_id in range(running_workers, conf.workers): 343 | self._start_worker(service_id, worker_id) 344 | elif running_workers > conf.workers: 345 | for worker_id in range(conf.workers, running_workers): 346 | self._stop_worker(service_id, worker_id) 347 | 348 | def _get_last_worker_died(self): 349 | """Return the last died worker information or None""" 350 | for service_id in list(self._running_services.keys()): 351 | # We copy the list to clean the orignal one 352 | processes = list(self._running_services[service_id].items()) 353 | for process, worker_id in processes: 354 | if not process.is_alive(): 355 | self._run_hooks( 356 | "dead_worker", 357 | service_id, 358 | worker_id, 359 | process.exitcode, 360 | ) 361 | if process.exitcode < 0: 362 | sig = _utils.signal_to_name(process.exitcode) 363 | LOG.info( 364 | "Child %(pid)d killed by signal %(sig)s", 365 | {"pid": process.pid, "sig": sig}, 366 | ) 367 | else: 368 | LOG.info( 369 | "Child %(pid)d exited with status %(code)d", 370 | {"pid": process.pid, "code": process.exitcode}, 371 | ) 372 | del self._running_services[service_id][process] 373 | return service_id, worker_id 374 | return None 375 | 376 | @staticmethod 377 | def _fast_exit( 378 | signo=None, 379 | frame=None, 380 | reason="Caught SIGINT signal, instantaneous exiting", 381 | ) -> None: 382 | if os.name == "posix": 383 | signal.signal(signal.SIGINT, signal.SIG_IGN) 384 | signal.signal(signal.SIGALRM, signal.SIG_IGN) 385 | LOG.info(reason) 386 | os.killpg(0, signal.SIGINT) 387 | else: 388 | # NOTE(sileht): On windows killing the master process 389 | # with SIGINT kill automatically children 390 | LOG.info(reason) 391 | os._exit(1) 392 | 393 | def _slowdown_respawn_if_needed(self) -> None: 394 | # Limit ourselves to one process a second (over the period of 395 | # number of workers * 1 second). This will allow workers to 396 | # start up quickly but ensure we don't fork off children that 397 | # die instantly too quickly. 398 | expected_children = sum(s.workers for s in self._services.values()) 399 | if len(self._forktimes) > expected_children: 400 | if time.time() - self._forktimes[0] < expected_children: 401 | LOG.info("Forking too fast, sleeping") 402 | time.sleep(5) 403 | self._forktimes.pop(0) 404 | else: 405 | time.sleep(self._wait_interval) 406 | self._forktimes.append(time.time()) 407 | 408 | def _start_worker(self, service_id, worker_id) -> None: 409 | self._slowdown_respawn_if_needed() 410 | 411 | # Create and run a new service 412 | p = _utils.spawn_process( 413 | _service.ServiceWorker.create_and_wait, 414 | self._services[service_id], 415 | service_id, 416 | worker_id, 417 | self._death_detection_pipe, 418 | self._hooks["new_worker"], 419 | self._graceful_shutdown_timeout, 420 | ) 421 | 422 | self._running_services[service_id][p] = worker_id 423 | 424 | def _stop_worker(self, service_id, worker_id) -> None: 425 | for process, _id in self._running_services[service_id].items(): 426 | if _id == worker_id: 427 | # NOTE(sileht): We should use CTRL_BREAK_EVENT on windows 428 | # when CREATE_NEW_PROCESS_GROUP will be set on child process 429 | process.terminate() 430 | 431 | @staticmethod 432 | def _systemd_notify_once() -> None: 433 | """Send notification once to Systemd that service is ready. 434 | 435 | Systemd sets NOTIFY_SOCKET environment variable with the name of the 436 | socket listening for notifications from services. 437 | This method removes the NOTIFY_SOCKET environment variable to ensure 438 | notification is sent only once. 439 | """ 440 | 441 | notify_socket = os.getenv("NOTIFY_SOCKET") 442 | if notify_socket: 443 | if notify_socket.startswith("@"): 444 | # abstract namespace socket 445 | notify_socket = f"\0{notify_socket[1:]}" 446 | sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) 447 | with contextlib.closing(sock): 448 | try: 449 | sock.connect(notify_socket) 450 | sock.sendall(b"READY=1") 451 | del os.environ["NOTIFY_SOCKET"] 452 | except OSError: 453 | LOG.debug("Systemd notification failed", exc_info=True) 454 | -------------------------------------------------------------------------------- /cotyledon/_utils.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | import collections 14 | import contextlib 15 | import errno 16 | import logging 17 | import multiprocessing 18 | import os 19 | import select 20 | import signal 21 | import sys 22 | import threading 23 | import time 24 | 25 | 26 | if os.name == "posix": 27 | import fcntl 28 | 29 | LOG = logging.getLogger(__name__) 30 | 31 | 32 | _SIGNAL_TO_NAME = { 33 | getattr(signal, name): name 34 | for name in dir(signal) 35 | if name.startswith("SIG") and name not in {"SIG_DFL", "SIG_IGN"} 36 | } 37 | 38 | 39 | def signal_to_name(sig): 40 | return _SIGNAL_TO_NAME.get(sig, sig) 41 | 42 | 43 | def spawn(target, *args, **kwargs): 44 | t = threading.Thread(target=target, args=args, kwargs=kwargs) 45 | t.daemon = True 46 | t.start() 47 | return t 48 | 49 | 50 | def check_workers(workers, minimum) -> None: 51 | if not isinstance(workers, int) or workers < minimum: 52 | msg = f"'workers' must be an int >= {minimum}, not: {workers} ({type(workers).__name__})" 53 | raise ValueError(msg) 54 | 55 | 56 | def check_callable(thing, name) -> None: 57 | if not callable(thing): 58 | msg = f"'{name}' must be a callable" 59 | raise TypeError(msg) 60 | 61 | 62 | def spawn_process(target, *args, **kwargs): 63 | p = multiprocessing.Process( 64 | target=target, 65 | args=args, 66 | kwargs=kwargs, 67 | ) 68 | p.start() 69 | return p 70 | 71 | 72 | try: 73 | from setproctitle import setproctitle 74 | except ImportError: 75 | 76 | def setproctitle(*args, **kwargs) -> None: 77 | pass 78 | 79 | 80 | def get_process_name(): 81 | return os.path.basename(sys.argv[0]) 82 | 83 | 84 | def run_hooks(name, hooks, *args, **kwargs) -> None: 85 | try: 86 | for hook in hooks: 87 | hook(*args, **kwargs) 88 | except Exception: 89 | LOG.exception("Exception raised during %s hooks", name) 90 | 91 | 92 | @contextlib.contextmanager 93 | def exit_on_exception(): 94 | try: 95 | yield 96 | except SystemExit as exc: 97 | os._exit(exc.code) 98 | except BaseException: 99 | LOG.exception("Unhandled exception") 100 | os._exit(2) 101 | 102 | 103 | if os.name == "posix": 104 | SIGALRM = signal.SIGALRM 105 | SIGHUP = signal.SIGHUP 106 | SIGCHLD = signal.SIGCHLD 107 | SIBREAK = None 108 | else: 109 | SIGALRM = SIGHUP = None 110 | SIGCHLD = "fake sigchld" 111 | SIGBREAK = signal.SIGBREAK 112 | 113 | 114 | class SignalManager: 115 | def __init__(self) -> None: 116 | # Setup signal fd, this allows signal to behave correctly 117 | if os.name == "posix": 118 | self.signal_pipe_r, self.signal_pipe_w = os.pipe() 119 | self._set_nonblock(self.signal_pipe_r) 120 | self._set_nonblock(self.signal_pipe_w) 121 | self._set_autoclose(self.signal_pipe_r) 122 | self._set_autoclose(self.signal_pipe_w) 123 | signal.set_wakeup_fd(self.signal_pipe_w) 124 | 125 | self._signals_received = collections.deque() 126 | 127 | if os.name == "posix": 128 | signal.signal(signal.SIGCHLD, signal.SIG_DFL) 129 | signal.signal(signal.SIGINT, self._signal_catcher) 130 | signal.signal(signal.SIGTERM, self._signal_catcher) 131 | signal.signal(signal.SIGALRM, self._signal_catcher) 132 | signal.signal(signal.SIGHUP, self._signal_catcher) 133 | else: 134 | signal.signal(signal.SIGINT, self._signal_catcher) 135 | # currently a noop on window... 136 | signal.signal(signal.SIGTERM, self._signal_catcher) 137 | # TODO(sileht): should allow to catch signal CTRL_BREAK_EVENT, 138 | # but we to create the child process with CREATE_NEW_PROCESS_GROUP 139 | # to make this work, so current this is a noop for later fix 140 | signal.signal(signal.SIGBREAK, self._signal_catcher) 141 | 142 | @staticmethod 143 | def _set_nonblock(fd) -> None: 144 | flags = fcntl.fcntl(fd, fcntl.F_GETFL, 0) 145 | flags |= os.O_NONBLOCK 146 | fcntl.fcntl(fd, fcntl.F_SETFL, flags) 147 | 148 | @staticmethod 149 | def _set_autoclose(fd) -> None: 150 | flags = fcntl.fcntl(fd, fcntl.F_GETFD) 151 | fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC) 152 | 153 | def _signal_catcher(self, sig, frame) -> None: 154 | # NOTE(sileht): This is useful only for python < 3.5 155 | # in python >= 3.5 we could read the signal number 156 | # from the wakeup_fd pipe 157 | if sig in {SIGALRM, signal.SIGTERM, signal.SIGINT}: 158 | self._signals_received.appendleft(sig) 159 | else: 160 | self._signals_received.append(sig) 161 | 162 | def _wait_forever(self) -> None: 163 | # Wait forever 164 | while True: 165 | # Check if signals have been received 166 | if os.name == "posix": 167 | self._empty_signal_pipe() 168 | self._run_signal_handlers() 169 | 170 | if os.name == "posix": 171 | # NOTE(sileht): we cannot use threading.Event().wait(), 172 | # threading.Thread().join(), or time.sleep() because signals 173 | # can be missed when received by non-main threads 174 | # (https://bugs.python.org/issue5315) 175 | # So we use select.select() alone, we will receive EINTR or 176 | # will read data from signal_r when signal is emitted and 177 | # cpython calls PyErr_CheckSignals() to run signals handlers 178 | # That looks perfect to ensure handlers are run and run in the 179 | # main thread 180 | try: 181 | select.select([self.signal_pipe_r], [], []) 182 | except OSError as e: 183 | if e.args[0] != errno.EINTR: 184 | raise 185 | else: 186 | # NOTE(sileht): here we do only best effort 187 | # and wake the loop periodically, set_wakeup_fd 188 | # doesn't work on non posix platform so 189 | # 1 seconds have been picked with the advice of a dice. 190 | time.sleep(1) 191 | # NOTE(sileht): We emulate SIGCHLD, _service_manager 192 | # will just check often for dead child 193 | self._signals_received.append(SIGCHLD) 194 | 195 | def _empty_signal_pipe(self) -> None: 196 | try: 197 | while os.read(self.signal_pipe_r, 4096) == 4096: 198 | pass 199 | except OSError: 200 | pass 201 | 202 | def _run_signal_handlers(self) -> None: 203 | while True: 204 | try: 205 | sig = self._signals_received.popleft() 206 | except IndexError: 207 | return 208 | self._on_signal_received(sig) 209 | 210 | def _on_signal_received(self, sig) -> None: 211 | pass 212 | -------------------------------------------------------------------------------- /cotyledon/oslo_config_glue.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | import copy 14 | import functools 15 | import logging 16 | import os 17 | 18 | from oslo_config import cfg 19 | 20 | 21 | LOG = logging.getLogger(__name__) 22 | 23 | service_opts = [ 24 | cfg.BoolOpt( 25 | "log_options", 26 | default=True, 27 | mutable=True, 28 | help="Enables or disables logging values of all " 29 | "registered options when starting a service (at DEBUG " 30 | "level).", 31 | ), 32 | cfg.IntOpt( 33 | "graceful_shutdown_timeout", 34 | mutable=True, 35 | default=60, 36 | help="Specify a timeout after which a gracefully shutdown " 37 | "server will exit. Zero value means endless wait.", 38 | ), 39 | ] 40 | 41 | 42 | def _load_service_manager_options(service_manager, conf) -> None: 43 | service_manager.graceful_shutdown_timeout = conf.graceful_shutdown_timeout 44 | if conf.log_options: 45 | LOG.debug("Full set of CONF:") 46 | conf.log_opt_values(LOG, logging.DEBUG) 47 | 48 | 49 | def _load_service_options(service, conf) -> None: 50 | service.graceful_shutdown_timeout = conf.graceful_shutdown_timeout 51 | 52 | if conf.log_options: 53 | LOG.debug("Full set of CONF:") 54 | conf.log_opt_values(LOG, logging.DEBUG) 55 | 56 | 57 | def _configfile_reload(conf, reload_method) -> None: 58 | if reload_method == "reload": 59 | conf.reload_config_files() 60 | elif reload_method == "mutate": 61 | conf.mutate_config_files() 62 | 63 | 64 | def _new_worker_hook(conf, reload_method, service_id, worker_id, service) -> None: 65 | def _service_reload(service) -> None: 66 | _configfile_reload(conf, reload_method) 67 | _load_service_options(service, conf) 68 | 69 | service._on_reload_internal_hook = _service_reload # noqa: SLF001 70 | _load_service_options(service, conf) 71 | 72 | 73 | def setup(service_manager, conf, reload_method="reload") -> None: 74 | """Load services configuration from oslo config object. 75 | 76 | It reads ServiceManager and Service configuration options from an 77 | oslo_config.ConfigOpts() object. Also It registers a ServiceManager hook to 78 | reload the configuration file on reload in the master process and in all 79 | children. And then when each child start or reload, the configuration 80 | options are logged if the oslo config option 'log_options' is True. 81 | 82 | On children, the configuration file is reloaded before the running the 83 | application reload method. 84 | 85 | Options currently supported on ServiceManager and Service: 86 | * graceful_shutdown_timeout 87 | 88 | :param service_manager: ServiceManager instance 89 | :type service_manager: cotyledon.ServiceManager 90 | :param conf: Oslo Config object 91 | :type conf: oslo_config.ConfigOpts() 92 | :param reload_method: reload or mutate the config files 93 | :type reload_method: str "reload/mutate" 94 | """ 95 | conf.register_opts(service_opts) 96 | 97 | # Set cotyledon options from oslo config options 98 | _load_service_manager_options(service_manager, conf) 99 | 100 | def _service_manager_reload() -> None: 101 | _configfile_reload(conf, reload_method) 102 | _load_service_manager_options(service_manager, conf) 103 | 104 | if os.name != "posix": 105 | # NOTE(sileht): reloading can't be supported oslo.config is not pickle 106 | # But we don't care SIGHUP is not support on window 107 | return 108 | 109 | service_manager.register_hooks( 110 | on_new_worker=functools.partial(_new_worker_hook, conf, reload_method), 111 | on_reload=_service_manager_reload, 112 | ) 113 | 114 | 115 | def list_opts(): 116 | """Entry point for oslo-config-generator.""" 117 | return [(None, copy.deepcopy(service_opts))] 118 | -------------------------------------------------------------------------------- /cotyledon/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sileht/cotyledon/be444189de32a8c29c7107a9b02da44248a7e64a/cotyledon/tests/__init__.py -------------------------------------------------------------------------------- /cotyledon/tests/base.py: -------------------------------------------------------------------------------- 1 | # Copyright 2010-2011 OpenStack Foundation 2 | # Copyright (c) 2013 Hewlett-Packard Development Company, L.P. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | # not use this file except in compliance with the License. You may obtain 6 | # a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations 14 | # under the License. 15 | 16 | from unittest import TestCase 17 | 18 | 19 | class TestCase(TestCase): 20 | """Test case base class for all unit tests.""" 21 | -------------------------------------------------------------------------------- /cotyledon/tests/examples.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | import logging 14 | import os 15 | import signal 16 | import sys 17 | import threading 18 | import time 19 | 20 | from oslo_config import cfg 21 | 22 | import cotyledon 23 | from cotyledon import _utils 24 | from cotyledon import oslo_config_glue 25 | 26 | 27 | logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) 28 | 29 | LOG = logging.getLogger("cotyledon.tests.examples") 30 | 31 | # We don't want functional tests to wait for this: 32 | cotyledon.ServiceManager._slowdown_respawn_if_needed = lambda *args: True 33 | 34 | 35 | class FullService(cotyledon.Service): 36 | name = "heavy" 37 | 38 | def __init__(self, worker_id) -> None: 39 | super().__init__(worker_id) 40 | self._shutdown = threading.Event() 41 | LOG.error("%s init", self.name) 42 | 43 | def run(self) -> None: 44 | LOG.error("%s run", self.name) 45 | self._shutdown.wait() 46 | 47 | def terminate(self) -> None: 48 | LOG.error("%s terminate", self.name) 49 | self._shutdown.set() 50 | sys.exit(42) 51 | 52 | def reload(self) -> None: 53 | LOG.error("%s reload", self.name) 54 | 55 | 56 | class LigthService(cotyledon.Service): 57 | name = "light" 58 | 59 | 60 | class BuggyService(cotyledon.Service): 61 | name = "buggy" 62 | graceful_shutdown_timeout = 1 63 | 64 | def terminate(self) -> None: # noqa: PLR6301 65 | time.sleep(60) 66 | LOG.error("time.sleep done") 67 | 68 | 69 | class BoomError(Exception): 70 | pass 71 | 72 | 73 | class BadlyCodedService(cotyledon.Service): 74 | def run(self): # noqa: PLR6301 75 | msg = "so badly coded service" 76 | raise BoomError(msg) 77 | 78 | 79 | class OsloService(cotyledon.Service): 80 | name = "oslo" 81 | 82 | 83 | class WindowService(cotyledon.Service): 84 | name = "window" 85 | 86 | 87 | def on_terminate() -> None: 88 | LOG.error("master terminate hook") 89 | 90 | 91 | def on_terminate2() -> None: 92 | LOG.error("master terminate2 hook") 93 | 94 | 95 | def on_reload() -> None: 96 | LOG.error("master reload hook") 97 | 98 | 99 | def example_app() -> None: 100 | p = cotyledon.ServiceManager() 101 | p.add(FullService, 2) 102 | service_id = p.add(LigthService, 5) 103 | p.reconfigure(service_id, 1) 104 | p.register_hooks(on_terminate, on_reload) 105 | p.register_hooks(on_terminate2) 106 | p.run() 107 | 108 | 109 | def buggy_app() -> None: 110 | p = cotyledon.ServiceManager() 111 | p.add(BuggyService) 112 | p.run() 113 | 114 | 115 | def oslo_app() -> None: 116 | conf = cfg.ConfigOpts() 117 | conf([], project="openstack-app", validate_default_values=True, version="0.1") 118 | 119 | p = cotyledon.ServiceManager() 120 | oslo_config_glue.setup(p, conf) 121 | p.add(OsloService) 122 | p.run() 123 | 124 | 125 | def window_sanity_check() -> None: 126 | p = cotyledon.ServiceManager() 127 | p.add(LigthService) 128 | t = _utils.spawn(p.run) 129 | time.sleep(10) 130 | os.kill(os.getpid(), signal.SIGTERM) 131 | t.join() 132 | 133 | 134 | def badly_coded_app() -> None: 135 | p = cotyledon.ServiceManager() 136 | p.add(BadlyCodedService) 137 | p.run() 138 | 139 | 140 | def exit_on_special_child_app() -> None: 141 | p = cotyledon.ServiceManager() 142 | sid = p.add(LigthService, 1) 143 | p.add(FullService, 2) 144 | 145 | def on_dead_worker(service_id, worker_id, exit_code) -> None: 146 | # Shutdown everybody if LigthService died 147 | if service_id == sid: 148 | p.shutdown() 149 | 150 | p.register_hooks(on_dead_worker=on_dead_worker) 151 | p.run() 152 | 153 | 154 | def sigterm_during_init() -> None: 155 | def kill() -> None: 156 | os.kill(os.getpid(), signal.SIGTERM) 157 | 158 | # Kill in 0.01 sec 159 | threading.Timer(0.01, kill).start() 160 | p = cotyledon.ServiceManager() 161 | p.add(LigthService, 10) 162 | p.run() 163 | 164 | 165 | if __name__ == "__main__": 166 | globals()[sys.argv[1]]() 167 | -------------------------------------------------------------------------------- /cotyledon/tests/test_functional.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | import os 14 | import re 15 | import signal 16 | import subprocess 17 | import threading 18 | import time 19 | import unittest 20 | 21 | from cotyledon import oslo_config_glue 22 | from cotyledon.tests import base 23 | 24 | 25 | if os.name == "posix": 26 | 27 | def pid_exists(pid): 28 | """Check whether pid exists in the current process table.""" 29 | import errno # noqa: PLC0415 30 | 31 | if pid < 0: 32 | return False 33 | try: 34 | os.kill(pid, 0) 35 | except OSError as e: 36 | return e.errno == errno.EPERM 37 | else: 38 | return True 39 | else: 40 | 41 | def pid_exists(pid) -> bool: 42 | import ctypes # noqa: PLC0415 43 | 44 | kernel32 = ctypes.windll.kernel32 45 | synchronize = 0x100000 46 | 47 | process = kernel32.OpenProcess(synchronize, 0, pid) 48 | if process != 0: 49 | kernel32.CloseHandle(process) 50 | return True 51 | return False 52 | 53 | 54 | class Base(base.TestCase): 55 | def setUp(self) -> None: 56 | super().setUp() 57 | 58 | self.lines = [] 59 | 60 | examplepy = os.path.join(os.path.dirname(__file__), "examples.py") 61 | if os.name == "posix": 62 | kwargs = { 63 | "preexec_fn": os.setsid, 64 | } 65 | else: 66 | kwargs = { 67 | "creationflags": subprocess.CREATE_NEW_PROCESS_GROUP, 68 | } 69 | 70 | self.subp = subprocess.Popen( 71 | ["python", examplepy, self.name], 72 | stdout=subprocess.PIPE, 73 | **kwargs, 74 | ) 75 | 76 | self.t = threading.Thread(target=self.readlog) 77 | self.t.daemon = True 78 | self.t.start() 79 | 80 | def readlog(self) -> None: 81 | while True: 82 | try: 83 | line = self.subp.stdout.readline() 84 | except OSError: 85 | return 86 | if not line: 87 | continue 88 | self.lines.append(line.strip()) 89 | 90 | def tearDown(self) -> None: 91 | if self.subp.poll() is None: 92 | self.subp.kill() 93 | super().tearDown() 94 | 95 | def get_lines(self, number=None): 96 | if number is not None: 97 | while len(self.lines) < number: 98 | time.sleep(0.1) 99 | lines = self.lines[:number] 100 | del self.lines[:number] 101 | return lines 102 | self.subp.wait() 103 | # Wait children to terminate 104 | return self.lines 105 | 106 | @staticmethod 107 | def hide_pids(lines): 108 | return [ 109 | re.sub( 110 | rb"Child \d+", 111 | b"Child XXXX", 112 | re.sub(rb" \[[^\]]*\]", b" [XXXX]", line), 113 | ) 114 | for line in lines 115 | ] 116 | 117 | @staticmethod 118 | def get_pid(line): 119 | try: 120 | return int(line.split()[-1][1:-1]) 121 | except Exception as exc: 122 | msg = f"Fail to find pid in {line.split()}" 123 | raise RuntimeError(msg) from exc 124 | 125 | 126 | class TestCotyledon(Base): 127 | name = "example_app" 128 | 129 | def assert_everything_has_started(self) -> None: 130 | lines = sorted(self.get_lines(7)) 131 | self.pid_heavy_1 = self.get_pid(lines[0]) 132 | self.pid_heavy_2 = self.get_pid(lines[1]) 133 | self.pid_light_1 = self.get_pid(lines[2]) 134 | lines = self.hide_pids(lines) 135 | assert lines == [ 136 | b"DEBUG:cotyledon._service:Run service heavy(0) [XXXX]", 137 | b"DEBUG:cotyledon._service:Run service heavy(1) [XXXX]", 138 | b"DEBUG:cotyledon._service:Run service light(0) [XXXX]", 139 | b"ERROR:cotyledon.tests.examples:heavy init", 140 | b"ERROR:cotyledon.tests.examples:heavy init", 141 | b"ERROR:cotyledon.tests.examples:heavy run", 142 | b"ERROR:cotyledon.tests.examples:heavy run", 143 | ] 144 | 145 | self.assert_everything_is_alive() 146 | 147 | def assert_everything_is_alive(self) -> None: 148 | assert pid_exists(self.subp.pid) 149 | assert pid_exists(self.pid_light_1) 150 | assert pid_exists(self.pid_heavy_1) 151 | assert pid_exists(self.pid_heavy_2) 152 | 153 | def assert_everything_is_dead(self, status=0) -> None: 154 | assert status == self.subp.poll() 155 | assert not pid_exists(self.subp.pid) 156 | assert not pid_exists(self.pid_light_1) 157 | assert not pid_exists(self.pid_heavy_1) 158 | assert not pid_exists(self.pid_heavy_2) 159 | 160 | @unittest.skipIf(os.name == "posix", "no window support") 161 | def test_workflow_window(self) -> None: 162 | # NOTE(sileht): The window workflow is a bit different because 163 | # SIGTERM doesn't really exists and processes are killed with SIGINT 164 | # TODO(sileht): Implements SIGBREAK to have graceful exists 165 | 166 | self.assert_everything_has_started() 167 | # Ensure we restart with terminate method exit code 168 | os.kill(self.pid_heavy_1, signal.SIGTERM) 169 | lines = self.get_lines(4) 170 | lines = self.hide_pids(lines) 171 | assert lines == [ 172 | b"INFO:cotyledon._service_manager:Child XXXX exited with status 15", 173 | b"ERROR:cotyledon.tests.examples:heavy init", 174 | b"DEBUG:cotyledon._service:Run service heavy(0) [XXXX]", 175 | b"ERROR:cotyledon.tests.examples:heavy run", 176 | ] 177 | 178 | # Kill master process 179 | os.kill(self.subp.pid, signal.SIGTERM) 180 | time.sleep(1) 181 | lines = self.get_lines() 182 | lines = sorted(self.hide_pids(lines)) 183 | assert lines == [ 184 | b"ERROR:cotyledon.tests.examples:heavy terminate", 185 | b"ERROR:cotyledon.tests.examples:heavy terminate", 186 | b"INFO:cotyledon._service:Caught SIGTERM signal, graceful exiting of service heavy(0) [XXXX]", 187 | b"INFO:cotyledon._service:Caught SIGTERM signal, graceful exiting of service heavy(1) [XXXX]", 188 | b"INFO:cotyledon._service:Caught SIGTERM signal, graceful exiting of service light(0) [XXXX]", 189 | b"INFO:cotyledon._service:Parent process has died unexpectedly, heavy(0) [XXXX] exiting", 190 | b"INFO:cotyledon._service:Parent process has died unexpectedly, heavy(1) [XXXX] exiting", 191 | b"INFO:cotyledon._service:Parent process has died unexpectedly, light(0) [XXXX] exiting", 192 | ] 193 | 194 | assert self.subp.poll() == 15 195 | 196 | @unittest.skipIf(os.name != "posix", "no posix support") 197 | def test_workflow(self) -> None: 198 | self.assert_everything_has_started() 199 | 200 | # Ensure we just call reload method 201 | os.kill(self.pid_heavy_1, signal.SIGHUP) 202 | assert self.get_lines(1) == [b"ERROR:cotyledon.tests.examples:heavy reload"] 203 | 204 | # Ensure we restart because reload method is missing 205 | os.kill(self.pid_light_1, signal.SIGHUP) 206 | lines = self.get_lines(3) 207 | self.pid_light_1 = self.get_pid(lines[-1]) 208 | lines = self.hide_pids(lines) 209 | assert lines == [ 210 | b"INFO:cotyledon._service:Caught SIGTERM signal, graceful exiting of service light(0) [XXXX]", 211 | b"INFO:cotyledon._service_manager:Child XXXX exited with status 0", 212 | b"DEBUG:cotyledon._service:Run service light(0) [XXXX]", 213 | ] 214 | 215 | # Ensure we restart with terminate method exit code 216 | os.kill(self.pid_heavy_1, signal.SIGTERM) 217 | lines = self.get_lines(6) 218 | self.pid_heavy_1 = self.get_pid(lines[-2]) 219 | lines = self.hide_pids(lines) 220 | assert lines == [ 221 | b"INFO:cotyledon._service:Caught SIGTERM signal, graceful exiting of service heavy(0) [XXXX]", 222 | b"ERROR:cotyledon.tests.examples:heavy terminate", 223 | b"INFO:cotyledon._service_manager:Child XXXX exited with status 42", 224 | b"ERROR:cotyledon.tests.examples:heavy init", 225 | b"DEBUG:cotyledon._service:Run service heavy(0) [XXXX]", 226 | b"ERROR:cotyledon.tests.examples:heavy run", 227 | ] 228 | 229 | # Ensure we restart when no terminate method 230 | os.kill(self.pid_light_1, signal.SIGTERM) 231 | lines = self.get_lines(3) 232 | self.pid_light_1 = self.get_pid(lines[-1]) 233 | lines = self.hide_pids(lines) 234 | assert lines == [ 235 | b"INFO:cotyledon._service:Caught SIGTERM signal, graceful exiting of service light(0) [XXXX]", 236 | b"INFO:cotyledon._service_manager:Child XXXX exited with status 0", 237 | b"DEBUG:cotyledon._service:Run service light(0) [XXXX]", 238 | ] 239 | 240 | # Ensure everything is still alive 241 | self.assert_everything_is_alive() 242 | 243 | # Kill master process 244 | os.kill(self.subp.pid, signal.SIGTERM) 245 | 246 | lines = self.get_lines() 247 | assert lines[-1] == b"DEBUG:cotyledon._service_manager:Shutdown finish" 248 | time.sleep(1) 249 | lines = sorted(self.hide_pids(lines)) 250 | assert lines == [ 251 | b"DEBUG:cotyledon._service_manager:Killing services with signal SIGTERM", 252 | b"DEBUG:cotyledon._service_manager:Shutdown finish", 253 | b"DEBUG:cotyledon._service_manager:Waiting services to terminate", 254 | b"ERROR:cotyledon.tests.examples:heavy terminate", 255 | b"ERROR:cotyledon.tests.examples:heavy terminate", 256 | b"ERROR:cotyledon.tests.examples:master terminate hook", 257 | b"ERROR:cotyledon.tests.examples:master terminate2 hook", 258 | b"INFO:cotyledon._service:Caught SIGTERM signal, graceful exiting of service heavy(0) [XXXX]", 259 | b"INFO:cotyledon._service:Caught SIGTERM signal, graceful exiting of service heavy(1) [XXXX]", 260 | b"INFO:cotyledon._service:Caught SIGTERM signal, graceful exiting of service light(0) [XXXX]", 261 | b"INFO:cotyledon._service_manager:Caught SIGTERM signal, graceful exiting of master process", 262 | ] 263 | 264 | self.assert_everything_is_dead() 265 | 266 | @unittest.skipIf(os.name != "posix", "http://bugs.python.org/issue18040") 267 | def test_sigint(self) -> None: 268 | self.assert_everything_has_started() 269 | os.kill(self.subp.pid, signal.SIGINT) 270 | time.sleep(1) 271 | lines = sorted(self.get_lines()) 272 | lines = self.hide_pids(lines) 273 | assert lines == [ 274 | b"INFO:cotyledon._service:Caught SIGINT signal, instantaneous exiting of service heavy(0) [XXXX]", 275 | b"INFO:cotyledon._service:Caught SIGINT signal, instantaneous exiting of service heavy(1) [XXXX]", 276 | b"INFO:cotyledon._service:Caught SIGINT signal, instantaneous exiting of service light(0) [XXXX]", 277 | b"INFO:cotyledon._service_manager:Caught SIGINT signal, instantaneous exiting", 278 | ] 279 | self.assert_everything_is_dead(1) 280 | 281 | @unittest.skipIf(os.name != "posix", "no posix support") 282 | def test_sighup(self) -> None: 283 | self.assert_everything_has_started() 284 | os.kill(self.subp.pid, signal.SIGHUP) 285 | time.sleep(0.5) 286 | lines = sorted(self.get_lines(6)) 287 | lines = self.hide_pids(lines) 288 | assert lines == [ 289 | b"DEBUG:cotyledon._service:Run service light(0) [XXXX]", 290 | b"ERROR:cotyledon.tests.examples:heavy reload", 291 | b"ERROR:cotyledon.tests.examples:heavy reload", 292 | b"ERROR:cotyledon.tests.examples:master reload hook", 293 | b"INFO:cotyledon._service:Caught SIGTERM signal, graceful exiting of service light(0) [XXXX]", 294 | b"INFO:cotyledon._service_manager:Child XXXX exited with status 0", 295 | ] 296 | 297 | os.kill(self.subp.pid, signal.SIGINT) 298 | time.sleep(0.5) 299 | self.assert_everything_is_dead(1) 300 | 301 | @unittest.skipIf(os.name != "posix", "no posix support") 302 | def test_sigkill(self) -> None: 303 | self.assert_everything_has_started() 304 | self.subp.kill() 305 | time.sleep(1) 306 | lines = sorted(self.get_lines()) 307 | lines = self.hide_pids(lines) 308 | assert lines == [ 309 | b"ERROR:cotyledon.tests.examples:heavy terminate", 310 | b"ERROR:cotyledon.tests.examples:heavy terminate", 311 | b"INFO:cotyledon._service:Caught SIGTERM signal, graceful exiting of service heavy(0) [XXXX]", 312 | b"INFO:cotyledon._service:Caught SIGTERM signal, graceful exiting of service heavy(1) [XXXX]", 313 | b"INFO:cotyledon._service:Caught SIGTERM signal, graceful exiting of service light(0) [XXXX]", 314 | b"INFO:cotyledon._service:Parent process has died unexpectedly, heavy(0) [XXXX] exiting", 315 | b"INFO:cotyledon._service:Parent process has died unexpectedly, heavy(1) [XXXX] exiting", 316 | b"INFO:cotyledon._service:Parent process has died unexpectedly, light(0) [XXXX] exiting", 317 | ] 318 | self.assert_everything_is_dead(-9) 319 | 320 | 321 | class TestBadlyCodedCotyledon(Base): 322 | name = "badly_coded_app" 323 | 324 | @unittest.skipIf(os.name != "posix", "no posix support") 325 | def test_badly_coded(self) -> None: 326 | time.sleep(2) 327 | self.subp.terminate() 328 | time.sleep(2) 329 | assert self.subp.poll() == 0, self.get_lines() 330 | assert not pid_exists(self.subp.pid) 331 | 332 | 333 | class TestBuggyCotyledon(Base): 334 | name = "buggy_app" 335 | 336 | @unittest.skipIf(os.name != "posix", "no posix support") 337 | def test_graceful_timeout_term(self) -> None: 338 | lines = self.get_lines(1) 339 | childpid = self.get_pid(lines[0]) 340 | self.subp.terminate() 341 | time.sleep(2) 342 | assert self.subp.poll() == 0 343 | assert not pid_exists(self.subp.pid) 344 | assert not pid_exists(childpid) 345 | lines = self.hide_pids(self.get_lines()) 346 | assert "ERROR:cotyledon.tests.examples:time.sleep done" not in lines 347 | assert lines[-2:] == [ 348 | b"INFO:cotyledon._service:Graceful shutdown timeout (1) exceeded, exiting buggy(0) [XXXX] now.", 349 | b"DEBUG:cotyledon._service_manager:Shutdown finish", 350 | ] 351 | 352 | @unittest.skipIf(os.name != "posix", "no posix support") 353 | def test_graceful_timeout_kill(self) -> None: 354 | lines = self.get_lines(1) 355 | childpid = self.get_pid(lines[0]) 356 | self.subp.kill() 357 | time.sleep(2) 358 | assert self.subp.poll() == -9 359 | assert not pid_exists(self.subp.pid) 360 | assert not pid_exists(childpid) 361 | lines = self.hide_pids(self.get_lines()) 362 | assert "ERROR:cotyledon.tests.examples:time.sleep done" not in lines 363 | assert lines[-3:] == [ 364 | b"INFO:cotyledon._service:Parent process has died unexpectedly, buggy(0) [XXXX] exiting", 365 | b"INFO:cotyledon._service:Caught SIGTERM signal, graceful exiting of service buggy(0) [XXXX]", 366 | b"INFO:cotyledon._service:Graceful shutdown timeout (1) exceeded, exiting buggy(0) [XXXX] now.", 367 | ] 368 | 369 | 370 | class TestOsloCotyledon(Base): 371 | name = "oslo_app" 372 | 373 | def test_options(self) -> None: 374 | options = oslo_config_glue.list_opts() 375 | assert len(options) == 1 376 | assert None is options[0][0] 377 | assert len(options[0][1]) == 2 378 | 379 | lines = self.get_lines(1) 380 | assert b"DEBUG:cotyledon.oslo_config_glue:Full set of CONF:" in lines 381 | self.subp.terminate() 382 | 383 | 384 | class TestTermDuringStartupCotyledon(Base): 385 | name = "sigterm_during_init" 386 | 387 | def test_sigterm(self) -> None: 388 | lines = self.hide_pids(self.get_lines()) 389 | assert b"DEBUG:cotyledon._service_manager:Shutdown finish" in lines 390 | -------------------------------------------------------------------------------- /cotyledon/tests/test_unit.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | from unittest import mock 14 | 15 | import pytest 16 | 17 | import cotyledon 18 | from cotyledon.tests import base 19 | 20 | 21 | class FakeService(cotyledon.Service): 22 | pass 23 | 24 | 25 | class SomeTest(base.TestCase): 26 | def setUp(self) -> None: 27 | super().setUp() 28 | cotyledon.ServiceManager._process_runner_already_created = False 29 | 30 | def test_forking_slowdown(self) -> None: # noqa: PLR6301 31 | sm = cotyledon.ServiceManager() 32 | sm.add(FakeService, workers=3) 33 | with mock.patch("time.sleep") as sleep: 34 | sm._slowdown_respawn_if_needed() 35 | sm._slowdown_respawn_if_needed() 36 | sm._slowdown_respawn_if_needed() 37 | # We simulatge 3 more spawn 38 | sm._slowdown_respawn_if_needed() 39 | sm._slowdown_respawn_if_needed() 40 | sm._slowdown_respawn_if_needed() 41 | assert len(sleep.mock_calls) == 6 42 | 43 | def test_invalid_service(self) -> None: 44 | sm = cotyledon.ServiceManager() 45 | 46 | self.assert_raises_msg( 47 | TypeError, 48 | "'service' must be a callable", 49 | sm.add, 50 | "foo", 51 | ) 52 | self.assert_raises_msg( 53 | ValueError, 54 | "'workers' must be an int >= 1, not: None (NoneType)", 55 | sm.add, 56 | FakeService, 57 | workers=None, 58 | ) 59 | self.assert_raises_msg( 60 | ValueError, 61 | "'workers' must be an int >= 1, not: -2 (int)", 62 | sm.add, 63 | FakeService, 64 | workers=-2, 65 | ) 66 | 67 | oid = sm.add(FakeService, workers=3) 68 | self.assert_raises_msg( 69 | ValueError, 70 | "'workers' must be an int >= -2, not: -5 (int)", 71 | sm.reconfigure, 72 | oid, 73 | workers=-5, 74 | ) 75 | self.assert_raises_msg( 76 | ValueError, 77 | "notexists service id doesn't exists", 78 | sm.reconfigure, 79 | "notexists", 80 | workers=-1, 81 | ) 82 | 83 | @staticmethod 84 | def assert_raises_msg(exc, msg, func, *args, **kwargs) -> None: 85 | with pytest.raises(exc) as exc_info: 86 | func(*args, **kwargs) 87 | assert msg == str(exc_info.value) 88 | -------------------------------------------------------------------------------- /doc/source/api.rst: -------------------------------------------------------------------------------- 1 | ======== 2 | API 3 | ======== 4 | 5 | .. autoclass:: cotyledon.Service 6 | :members: 7 | :special-members: __init__ 8 | 9 | .. autoclass:: cotyledon.ServiceManager 10 | :members: 11 | :special-members: __init__ 12 | 13 | .. autofunction:: cotyledon.oslo_config_glue.setup 14 | 15 | -------------------------------------------------------------------------------- /doc/source/conf.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 10 | # implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | import os 15 | import sys 16 | 17 | 18 | sys.path.insert(0, os.path.abspath("../..")) 19 | # -- General configuration ---------------------------------------------------- 20 | 21 | # Add any Sphinx extension module names here, as strings. They can be 22 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 23 | extensions = [ 24 | "sphinx.ext.autodoc", 25 | ] 26 | 27 | # autodoc generation is a bit aggressive and a nuisance when doing heavy 28 | # text edit cycles. 29 | # execute "export SPHINX_DEBUG=1" in your terminal to disable 30 | 31 | # The suffix of source filenames. 32 | source_suffix = ".rst" 33 | 34 | # The master toctree document. 35 | master_doc = "index" 36 | 37 | # General information about the project. 38 | project = "cotyledon" 39 | copyright = "2016, Mehdi Abaakouk" 40 | 41 | # If true, '()' will be appended to :func: etc. cross-reference text. 42 | add_function_parentheses = True 43 | 44 | # If true, the current module name will be prepended to all description 45 | # unit titles (such as .. function::). 46 | add_module_names = True 47 | 48 | # The name of the Pygments (syntax highlighting) style to use. 49 | pygments_style = "sphinx" 50 | 51 | # -- Options for HTML output -------------------------------------------------- 52 | 53 | # The theme to use for HTML and HTML Help pages. Major themes that come with 54 | # Sphinx are currently 'default' and 'sphinxdoc'. 55 | # html_theme_path = ["."] 56 | # html_theme = '_theme' 57 | # html_static_path = ['static'] 58 | 59 | try: 60 | import sphinx_rtd_theme 61 | except ImportError: 62 | pass 63 | else: 64 | html_theme = "sphinx_rtd_theme" 65 | 66 | # Output file base name for HTML help builder. 67 | htmlhelp_basename = "%sdoc" % project 68 | 69 | # Grouping the document tree into LaTeX files. List of tuples 70 | # (source start file, target name, title, author, documentclass 71 | # [howto/manual]). 72 | latex_documents = [ 73 | ("index", 74 | "%s.tex" % project, 75 | "%s Documentation" % project, 76 | "Mehdi Abaakouk", "manual"), 77 | ] 78 | 79 | 80 | # Example configuration for intersphinx: refer to the Python standard library. 81 | # intersphinx_mapping = {'http://docs.python.org/': None} 82 | -------------------------------------------------------------------------------- /doc/source/contributing.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Contributing 3 | ============ 4 | .. include:: ../../CONTRIBUTING.rst 5 | -------------------------------------------------------------------------------- /doc/source/examples.rst: -------------------------------------------------------------------------------- 1 | ======== 2 | Examples 3 | ======== 4 | 5 | .. literalinclude:: ../../cotyledon/tests/examples.py 6 | :language: python 7 | -------------------------------------------------------------------------------- /doc/source/index.rst: -------------------------------------------------------------------------------- 1 | .. cotyledon documentation master file, created by 2 | sphinx-quickstart on Tue Jul 9 22:26:36 2013. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to cotyledon's documentation! 7 | ======================================================== 8 | 9 | Contents: 10 | ========= 11 | 12 | .. toctree:: 13 | :maxdepth: 2 14 | 15 | installation 16 | api 17 | examples 18 | non-posix-support 19 | oslo-service-migration 20 | contributing 21 | 22 | .. include:: ../../README.rst 23 | 24 | Indices and tables 25 | ================== 26 | 27 | * :ref:`genindex` 28 | * :ref:`modindex` 29 | * :ref:`search` 30 | 31 | -------------------------------------------------------------------------------- /doc/source/installation.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Installation 3 | ============ 4 | 5 | At the command line:: 6 | 7 | $ pip install cotyledon 8 | 9 | Or, if you have virtualenvwrapper installed:: 10 | 11 | $ mkvirtualenv cotyledon 12 | $ pip install cotyledon 13 | -------------------------------------------------------------------------------- /doc/source/non-posix-support.rst: -------------------------------------------------------------------------------- 1 | ============================ 2 | Note about non posix support 3 | ============================ 4 | 5 | On non-posix platform the lib have some limitation. 6 | 7 | When the master process receives a signal, the propagation to children 8 | processes is done manually on known pids instead of the process group. 9 | 10 | SIGHUP is of course not supported. 11 | 12 | Processes termination are not done gracefully. Even we use Popen.terminate(), 13 | children don't received SIGTERM/SIGBREAK as expected. The module 14 | multiprocessing doesn't allow to set CREATE_NEW_PROCESS_GROUP on new processes 15 | and catch SIGBREAK. 16 | 17 | Also signal handlers are only run every second instead of just after the 18 | signal reception because non-posix platform does not support 19 | signal.set_wakeup_fd correctly 20 | 21 | And to finish, the processes names are not set on non-posix platform. 22 | -------------------------------------------------------------------------------- /doc/source/oslo-service-migration.rst: -------------------------------------------------------------------------------- 1 | =============================== 2 | Oslo.service migration examples 3 | =============================== 4 | 5 | This example shows the same application with oslo.service and cotyledon. 6 | It uses a wide range of API of oslo.service, but most applications don't 7 | really uses all of this. In most case cotyledon.ServiceManager don't 8 | need to inherited. 9 | 10 | It doesn't show how to replace the periodic task API, if you use it 11 | you should take a look to `futurist documentation`_ 12 | 13 | 14 | oslo.service typical application: 15 | 16 | .. code-block:: python 17 | 18 | import multiprocessing 19 | from oslo.service import service 20 | from oslo.config import cfg 21 | 22 | class MyService(service.Service): 23 | def __init__(self, conf): 24 | # called before os.fork() 25 | self.conf = conf 26 | self.master_pid = os.getpid() 27 | 28 | self.queue = multiprocessing.Queue() 29 | 30 | def start(self): 31 | # called when application start (parent process start) 32 | # and 33 | # called just after os.fork() 34 | 35 | if self.master_pid == os.getpid(): 36 | do_master_process_start() 37 | else: 38 | task = self.queue.get() 39 | do_child_process_start(task) 40 | 41 | 42 | def stop(self): 43 | # called when children process stop 44 | # and 45 | # called when application stop (parent process stop) 46 | if self.master_pid == os.getpid(): 47 | do_master_process_stop() 48 | else: 49 | do_child_process_stop() 50 | 51 | def restart(self): 52 | # called on SIGHUP 53 | if self.master_pid == os.getpid(): 54 | do_master_process_reload() 55 | else: 56 | # Can't be reach oslo.service currently prefers to 57 | # kill the child process for safety purpose 58 | do_child_process_reload() 59 | 60 | class MyOtherService(service.Service): 61 | pass 62 | 63 | 64 | class MyThirdService(service.Service): 65 | pass 66 | 67 | 68 | def main(): 69 | conf = cfg.ConfigOpts() 70 | service = MyService(conf) 71 | launcher = service.launch(conf, service, workers=2, restart_method='reload') 72 | launcher.launch_service(MyOtherService(), worker=conf.other_workers) 73 | 74 | # Obviously not recommanded, because two objects will handle the 75 | # lifetime of the masterp process but some application does this, so... 76 | launcher2 = service.launch(conf, MyThirdService(), workers=2, restart_method='restart') 77 | 78 | launcher.wait() 79 | launcher2.wait() 80 | 81 | # Here, we have no way to change the number of worker dynamically. 82 | 83 | 84 | Cotyledon version of the typical application: 85 | 86 | .. code-block:: python 87 | 88 | import cotyledon 89 | from cotyledon import oslo_config_glue 90 | 91 | class MyService(cotyledon.Service): 92 | name = "MyService fancy name that will showup in 'ps xaf'" 93 | 94 | # Everything in this object will be called after os.fork() 95 | def __init__(self, worker_id, conf, queue): 96 | self.conf = conf 97 | self.queue = queue 98 | 99 | def run(self): 100 | # Optional method to run the child mainloop or whatever 101 | task = self.queue.get() 102 | do_child_process_start(task) 103 | 104 | def terminate(self): 105 | do_child_process_stop() 106 | 107 | def reload(self): 108 | # Done on SIGHUP after the configuration file reloading 109 | do_child_reload() 110 | 111 | 112 | class MyOtherService(cotyledon.Service): 113 | name = "Second Service" 114 | 115 | 116 | class MyThirdService(cotyledon.Service): 117 | pass 118 | 119 | 120 | class MyServiceManager(cotyledon.ServiceManager): 121 | def __init__(self, conf) 122 | super(MetricdServiceManager, self).__init__() 123 | self.conf = conf 124 | oslo_config_glue.setup(self, self.conf, restart_method='reload') 125 | self.queue = multiprocessing.Queue() 126 | 127 | # the queue is explicitly passed to this child (it will live 128 | # on all of them due to the usage of os.fork() to create children) 129 | sm.add(MyService, workers=2, args=(self.conf, queue)) 130 | self.other_id = sm.add(MyOtherService, workers=conf.other_workers) 131 | sm.add(MyThirdService, workers=2) 132 | 133 | def run(self): 134 | do_master_process_start() 135 | super(MyServiceManager, self).run() 136 | do_master_process_stop() 137 | 138 | def reload(self): 139 | # The cotyledon ServiceManager have already reloaded the oslo.config files 140 | 141 | do_master_process_reload() 142 | 143 | # Allow to change the number of worker for MyOtherService 144 | self.reconfigure(self.other_id, workers=self.conf.other_workers) 145 | 146 | def main(): 147 | conf = cfg.ConfigOpts() 148 | MyServiceManager(conf).run() 149 | 150 | 151 | Other examples can be found here: 152 | 153 | * :doc:`examples` 154 | * https://github.com/openstack/gnocchi/blob/master/gnocchi/cli.py#L287 155 | * https://github.com/openstack/ceilometer/blob/master/ceilometer/cmd/collector.py 156 | 157 | .. _futurist documentation: ` 158 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools", "setuptools_scm"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "cotyledon" 7 | description = "Cotyledon provides a framework for defining long-running services." 8 | readme = "README.rst" 9 | authors = [ 10 | {name = "Mehdi Abaakouk", email = "sileht@sileht.net"} 11 | ] 12 | classifiers = [ 13 | "Intended Audience :: Information Technology", 14 | "Intended Audience :: System Administrators", 15 | "License :: OSI Approved :: Apache Software License", 16 | "Operating System :: POSIX :: Linux", 17 | "Programming Language :: Python", 18 | "Programming Language :: Python :: 3", 19 | "Programming Language :: Python :: 3.9", 20 | "Programming Language :: Python :: 3.10", 21 | "Programming Language :: Python :: 3.11", 22 | "Programming Language :: Python :: 3.12", 23 | ] 24 | dynamic = ["version"] 25 | requires-python = ">= 3.9" 26 | dependencies = [ 27 | "setproctitle; sys_platform != 'win32'" 28 | ] 29 | [project.optional-dependencies] 30 | test = [ 31 | "mock", 32 | "pytest", 33 | "pytest-cov", 34 | "pytest-xdist" 35 | ] 36 | oslo = [ 37 | "oslo.config>=3.14.0", 38 | ] 39 | doc = [ 40 | "sphinx_rtd_theme", 41 | "sphinx" 42 | ] 43 | 44 | [project.entry-points."oslo.config.opts"] 45 | "cotyledon" = "cotyledon.oslo_config_glue:list_opts" 46 | 47 | [project.urls] 48 | "Home Page" = "https://github.com/sileht/cotyledon" 49 | 50 | [tool.setuptools_scm] 51 | 52 | [tool.ruff] 53 | line-length = 88 54 | indent-width = 4 55 | target-version = "py311" 56 | exclude = ["doc/source/conf.py"] 57 | 58 | [tool.ruff.lint] 59 | preview = true 60 | select = [ 61 | "F", 62 | "E", 63 | "W", 64 | "I", 65 | "N", 66 | "UP", 67 | "YTT", 68 | # "ANN", # mypy 69 | "ASYNC", 70 | "S", 71 | "BLE", 72 | "FBT", 73 | "B", 74 | "A", 75 | "COM", 76 | "C4", 77 | "DTZ", 78 | "T10", 79 | "EM", 80 | "FA", 81 | "ISC", 82 | "ICN", 83 | "G", 84 | "INP", 85 | "PIE", 86 | "T20", 87 | "PYI", 88 | "PT", 89 | "Q", 90 | "RSE", 91 | "RET", 92 | "SLF", 93 | "SLOT", 94 | "SIM", 95 | "TID", 96 | "TCH", 97 | "INT", 98 | # "ARG", # unused args 99 | # "PTH", # pathlib.Path 100 | "TD", 101 | "ERA", 102 | "PGH", 103 | "PL", 104 | "TRY", 105 | "FLY", 106 | "NPY", 107 | "PERF", 108 | "FURB", 109 | "LOG", 110 | "RUF", 111 | ] 112 | 113 | ignore = [ 114 | # NOTE(charly): line-length is up to the formatter 115 | "E501", 116 | # NOTE(charly): `subprocess` module is possibly insecure 117 | "S404", 118 | # NOTE(jd): likely a false positive https://github.com/PyCQA/bandit/issues/333 119 | "S603", 120 | # NOTE(charly): Starting a process with a partial executable path 121 | "S607", 122 | # NOTE(charly): Boolean-typed positional argument in function definition. 123 | # Interesting, but require some work. 124 | "FBT001", 125 | # NOTE(charly): Boolean default positional argument in function definition. 126 | # Interesting, but require some work. 127 | "FBT002", 128 | # NOTE(charly): Missing issue link on the line following this TODO 129 | "TD003", 130 | # NOTE(charly): Magic value used in comparison 131 | "PLR2004", 132 | # List comprehensions are most efficient in most cases now 133 | "PLR1702", 134 | # We use mock.patch.object, which automatically pass the mock as an 135 | # argument to the test if no `new` is specified, without needing the mock 136 | # itself. 137 | "PT019", 138 | # We don't want to enforce the number of statements 139 | "PLR0914", "PLR0912", "PLR0915", 140 | ] 141 | [tool.ruff.lint.per-file-ignores] 142 | "cotyledon/tests/*.py" = ["S101", "SLF001"] 143 | 144 | [tool.ruff.lint.isort] 145 | force-single-line = true 146 | force-sort-within-sections = true 147 | lines-after-imports = 2 148 | known-first-party = ["cotyledon"] 149 | 150 | [tool.ruff.lint.flake8-tidy-imports] 151 | ban-relative-imports = "all" 152 | -------------------------------------------------------------------------------- /release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | set -x 5 | 6 | version=$1 7 | [ ! "$version" ] && echo "missing version" && exit 1 8 | 9 | status=$(git status -sz) 10 | [ -z "$status" ] || false 11 | git checkout main 12 | [ -z "$SKIP_TESTS" ] && tox -epy312,pep8 13 | git push 14 | git tag $version -m "Release version ${version}" 15 | git checkout $version 16 | git clean -fd 17 | tox -e build 18 | 19 | set +x 20 | echo 21 | echo "release: Cotyledon ${version}" 22 | echo 23 | echo "SHA1sum: " 24 | sha1sum dist/* 25 | echo "MD5sum: " 26 | md5sum dist/* 27 | echo 28 | echo "uploading..." 29 | echo 30 | set -x 31 | 32 | read 33 | git push --tags 34 | twine upload -r pypi -s dist/cotyledon-${version}.tar.gz dist/cotyledon-${version}-py3-none-any.whl 35 | git checkout main 36 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py39,py310,py311,py312,pep8 3 | minversion = 4.0 4 | skipsdist = true 5 | 6 | [testenv] 7 | deps = -e .[oslo,test] 8 | skip_install = true 9 | commands = 10 | pytest {posargs:cotyledon/tests} 11 | 12 | [testenv:pep8] 13 | deps = ruff 14 | doc8 15 | pygments 16 | commands = 17 | ruff check . 18 | ruff format --check . 19 | doc8 doc/source 20 | 21 | [testenv:format] 22 | deps = ruff 23 | commands = 24 | ruff check --fix . 25 | ruff format . 26 | 27 | [testenv:venv] 28 | commands = {posargs} 29 | 30 | [testenv:build] 31 | deps = build 32 | commands = python -m build 33 | 34 | [testenv:docs] 35 | deps = .[doc,oslo] 36 | commands = sphinx-build -a -W -b html doc/source doc/build 37 | 38 | [pytest] 39 | addopts = --verbose --numprocesses=auto 40 | norecursedirs = .tox 41 | --------------------------------------------------------------------------------