├── .flake8 ├── .github ├── codecov.yml └── workflows │ ├── lint.yaml │ ├── package.yaml │ ├── publish.yaml │ └── test.yaml ├── .gitignore ├── .readthedocs.yaml ├── HISTORY.rst ├── LICENSE ├── MANIFEST.in ├── README.rst ├── docs ├── Makefile ├── advanced_usage.rst ├── basic_usage.rst ├── conf.py ├── conventions.rst ├── gravity-logo.png ├── history.rst ├── index.rst ├── installation.rst ├── make.bat ├── readme.rst ├── requirements.txt └── subcommands.rst ├── gravity ├── __init__.py ├── cli.py ├── commands │ ├── __init__.py │ ├── cmd_exec.py │ ├── cmd_follow.py │ ├── cmd_graceful.py │ ├── cmd_list.py │ ├── cmd_pm.py │ ├── cmd_restart.py │ ├── cmd_show.py │ ├── cmd_shutdown.py │ ├── cmd_start.py │ ├── cmd_status.py │ ├── cmd_stop.py │ └── cmd_update.py ├── config_manager.py ├── io.py ├── options.py ├── process_manager │ ├── __init__.py │ ├── multiprocessing.py │ ├── supervisor.py │ └── systemd.py ├── settings.py ├── state.py └── util │ └── __init__.py ├── pyproject.toml ├── setup.cfg ├── setup.py ├── tests ├── conftest.py ├── test_config_manager.py ├── test_operations.py ├── test_process_manager.py └── test_settings.py └── tox.ini /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | per-file-ignores = 3 | settings.py: E501 4 | -------------------------------------------------------------------------------- /.github/codecov.yml: -------------------------------------------------------------------------------- 1 | comment: false 2 | -------------------------------------------------------------------------------- /.github/workflows/lint.yaml: -------------------------------------------------------------------------------- 1 | name: Lint 2 | on: [push, pull_request] 3 | concurrency: 4 | group: py-lint-${{ github.ref }} 5 | cancel-in-progress: true 6 | jobs: 7 | test: 8 | name: Test 9 | runs-on: ubuntu-latest 10 | strategy: 11 | matrix: 12 | python-version: ['3.9', '3.13'] 13 | steps: 14 | - uses: actions/checkout@v4 15 | - uses: actions/setup-python@v5 16 | with: 17 | python-version: ${{ matrix.python-version }} 18 | - name: Install tox 19 | run: pip install tox 20 | - name: Run linting 21 | run: tox -e lint 22 | -------------------------------------------------------------------------------- /.github/workflows/package.yaml: -------------------------------------------------------------------------------- 1 | name: Package Test 2 | on: [push, pull_request] 3 | concurrency: 4 | group: package-${{ github.ref }} 5 | cancel-in-progress: true 6 | jobs: 7 | package: 8 | name: Package Test 9 | runs-on: ubuntu-latest 10 | strategy: 11 | matrix: 12 | python-version: ['3.9', '3.13'] 13 | steps: 14 | - uses: actions/checkout@v4 15 | - uses: actions/setup-python@v5 16 | with: 17 | python-version: ${{ matrix.python-version }} 18 | - name: Install dependencies 19 | run: | 20 | python3 -m pip install --upgrade pip setuptools 21 | python3 -m pip install --upgrade twine wheel 22 | - name: Create and check packages 23 | run: | 24 | python3 setup.py sdist bdist_wheel 25 | twine check dist/* 26 | ls -l dist 27 | -------------------------------------------------------------------------------- /.github/workflows/publish.yaml: -------------------------------------------------------------------------------- 1 | name: Publish gravity to PyPI 2 | on: 3 | release: 4 | types: [created] 5 | push: 6 | tags: 7 | - '*' 8 | jobs: 9 | build-n-publish: 10 | name: Build and publish Python 🐍 distributions 📦 to PyPI and TestPyPI 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v4 14 | - name: Set up Python 3.9 15 | uses: actions/setup-python@v5 16 | with: 17 | python-version: 3.9 18 | - name: Install dependencies 19 | run: | 20 | python3 -m pip install --upgrade pip setuptools 21 | python3 -m pip install --upgrade twine wheel 22 | - name: Create and check packages 23 | run: | 24 | python3 setup.py sdist bdist_wheel 25 | twine check dist/* 26 | ls -l dist 27 | - name: Publish distribution 📦 to Test PyPI 28 | uses: pypa/gh-action-pypi-publish@master 29 | with: 30 | password: ${{ secrets.TEST_PYPI_API_TOKEN }} 31 | repository_url: https://test.pypi.org/legacy/ 32 | skip_existing: true 33 | - name: Publish distribution 📦 to PyPI 34 | if: github.event_name == 'release' && github.event.action == 'created' 35 | uses: pypa/gh-action-pypi-publish@master 36 | with: 37 | password: ${{ secrets.PYPI_API_TOKEN }} 38 | -------------------------------------------------------------------------------- /.github/workflows/test.yaml: -------------------------------------------------------------------------------- 1 | name: Test 2 | on: [push, pull_request] 3 | concurrency: 4 | group: py-test-${{ github.ref }} 5 | cancel-in-progress: true 6 | jobs: 7 | test: 8 | name: Test 9 | runs-on: ubuntu-22.04 10 | strategy: 11 | fail-fast: false 12 | matrix: 13 | python-version: ['3.9', '3.13'] 14 | galaxy-branch: ['release_23.0', 'dev'] 15 | exclude: 16 | # either the release existed before the python release or some expensive-to-build wheels (e.g. numpy) don't 17 | # exist for the pinned package version / python version combo 18 | - python-version: '3.13' 19 | galaxy-branch: 'release_23.0' 20 | steps: 21 | - uses: actions/checkout@v4 22 | - uses: actions/setup-python@v5 23 | with: 24 | python-version: ${{ matrix.python-version }} 25 | - name: Precreate virtualenv 26 | run: python -m venv tests/galaxy_venv 27 | - name: Install tox 28 | run: pip install tox 29 | - name: Run tests 30 | run: tox -e test 31 | env: 32 | GRAVITY_TEST_GALAXY_BRANCH: ${{ matrix.galaxy-branch }} 33 | - name: "Upload coverage to Codecov" 34 | uses: codecov/codecov-action@v2 35 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *.*.swp 3 | *.pyc 4 | .DS_Store 5 | .coverage 6 | coverage.xml 7 | .tox/ 8 | /build/ 9 | /dist/ 10 | /*.egg-info/ 11 | *.egg 12 | tests/galaxy.git 13 | tests/galaxy_venv 14 | docs/_build 15 | 16 | # dev scripts 17 | /galaxycfg 18 | /galaxyadm 19 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yaml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | # Set the OS, Python version and other tools you might need 9 | build: 10 | os: ubuntu-22.04 11 | tools: 12 | python: "3.13" 13 | 14 | # Build documentation in the docs/ directory with Sphinx 15 | sphinx: 16 | configuration: docs/conf.py 17 | 18 | # Optionally build your docs in additional formats such as PDF and ePub 19 | formats: 20 | - pdf 21 | 22 | # Optional but recommended, declare the Python requirements required 23 | # to build your documentation 24 | # See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html 25 | python: 26 | install: 27 | - path: . 28 | - requirements: docs/requirements.txt 29 | -------------------------------------------------------------------------------- /HISTORY.rst: -------------------------------------------------------------------------------- 1 | ========= 2 | History 3 | ========= 4 | 5 | 1.0.7 6 | ===== 7 | 8 | - Support new Galaxy configuration setting `interactivetoolsproxy_map` by @kysrpex in https://github.com/galaxyproject/gravity/pull/120 9 | - Add a command line flag for controlling systemd user mode by @natefoo in https://github.com/galaxyproject/gravity/pull/122 10 | 11 | 1.0.6 12 | ===== 13 | 14 | - Support pydantic v1 and v2 by @mvdbeek in https://github.com/galaxyproject/gravity/pull/118 15 | - Fix GalaxyReportsService crash and remove config_type by @jvanbraekel in https://github.com/galaxyproject/gravity/pull/116 16 | - Support running multiple tusds and controlling the value of hooks-http by @natefoo in https://github.com/galaxyproject/gravity/pull/119 17 | 18 | 1.0.5 19 | ===== 20 | 21 | - Support pydantic v1 and v2 by @heisner-tillman in https://github.com/galaxyproject/gravity/pull/118 22 | 23 | 1.0.4 24 | ===== 25 | 26 | - Supervisor: more readable program name + reread config on each change by @abretaud in https://github.com/galaxyproject/gravity/pull/110 27 | - Remove unneeded supervisorctl update call by @abretaud in https://github.com/galaxyproject/gravity/pull/112 28 | - Minimal path change needed for galaxy PR 16795 by @sveinugu in https://github.com/galaxyproject/gravity/pull/114 29 | 30 | 1.0.3 31 | ===== 32 | 33 | - Don't create supervisor conf dir unless necessary, create the gravity data dir as the correct user by @natefoo in https://github.com/galaxyproject/gravity/pull/105 34 | 35 | 1.0.2 36 | ===== 37 | 38 | - Pin a minimum package version of gx-it-proxy by @natefoo in https://github.com/galaxyproject/gravity/pull/102 39 | 40 | 1.0.1 41 | ===== 42 | 43 | - Added configuration of gx-it-proxy to support path-based proxying by @sveinugu in https://github.com/galaxyproject/gravity/pull/100 44 | 45 | 1.0.0 46 | ===== 47 | 48 | Version 1.0.0 represents a significant update to Gravity, its features and functionality. Although Gravity 1.x is intended to be backwards compatible with 0.x, you are strongly encouraged to [read the documentation](https://gravity.readthedocs.io/en/latest/) if upgrading to Gravity 1.x or to Galaxy 23.0 (which depends on Gravity 1.x) in order to get the most out of the new features. 49 | 50 | - Support systemd as a process manager by @natefoo in https://github.com/galaxyproject/gravity/pull/77 51 | - Full stateless mode when working with single instances and other improvements for 1.0 by @natefoo in https://github.com/galaxyproject/gravity/pull/80 52 | - Multi-unicorn rolling restart and general multi-instance service support by @natefoo in https://github.com/galaxyproject/gravity/pull/81 53 | - Don't clobber other Galaxies' systemd units when managed by different Gravity config files by @natefoo in https://github.com/galaxyproject/gravity/pull/83 54 | - Don't restart tusd on graceful by @natefoo in https://github.com/galaxyproject/gravity/pull/85 55 | - Read job_conf.yml by default if job_config_file is unset by @natefoo in https://github.com/galaxyproject/gravity/pull/86 56 | - Fixes for spaces in the galaxy root path, fix the `galaxy` entrypoint by @natefoo in https://github.com/galaxyproject/gravity/pull/87 57 | - Update existing env with program env when running exec, rather than the other way around by @natefoo in https://github.com/galaxyproject/gravity/pull/93 58 | - Hide the "exec" ServiceCommandStyle from documentation since it is only used internally by @natefoo in https://github.com/galaxyproject/gravity/pull/94 59 | - Updates for settings documentation generation by @natefoo in https://github.com/galaxyproject/gravity/pull/95 60 | - Set `$VIRTUAL_ENV` if `virtualenv` is set in config by @natefoo in https://github.com/galaxyproject/gravity/pull/97 61 | - Always add venv bin dir to `$PATH` if `virtualenv` is set by @natefoo in https://github.com/galaxyproject/gravity/pull/98 62 | 63 | 0.13.6 64 | ====== 65 | 66 | - Fix graceful method for gunicorn ``--preload`` by @Slugger70 in https://github.com/galaxyproject/gravity/pull/76 67 | - Add ``--version`` option to get Gravity version by @natefoo in https://github.com/galaxyproject/gravity/pull/79 68 | - Fix stopping of gx-it-proxy by @abretaud in https://github.com/galaxyproject/gravity/pull/91 69 | 70 | 0.13.5 71 | ====== 72 | 73 | - If virtualenv is set in the Gravity config, automatically add its bin dir to $PATH if the gx-it-proxy is enabled by @natefoo in https://github.com/galaxyproject/gravity/pull/71 74 | - Support converting settings to command line arguments in a generalized way by @natefoo in https://github.com/galaxyproject/gravity/pull/73 75 | 76 | 0.13.4 77 | ====== 78 | 79 | - Fixes for startup test by @natefoo in https://github.com/galaxyproject/gravity/pull/68 80 | - Fix setting environment vars on handlers by @natefoo in https://github.com/galaxyproject/gravity/pull/70 81 | 82 | 0.13.3 83 | ====== 84 | 85 | - Don't use gunicorn logging options with unicornherder by @natefoo in https://github.com/galaxyproject/gravity/pull/65 86 | 87 | 0.13.2 88 | ====== 89 | 90 | - Don't override PATH in subprocess call by @jdavcs in https://github.com/galaxyproject/gravity/pull/62 91 | - Only send pre create hook by @mvdbeek in https://github.com/galaxyproject/gravity/pull/64 92 | 93 | 0.13.1 94 | ====== 95 | 96 | - Set correct default for environment settings by @natefoo in https://github.com/galaxyproject/gravity/pull/58 97 | - Don't catch exceptions in the deregister, register, and rename subcommands by @natefoo in https://github.com/galaxyproject/gravity/pull/59 98 | - ``processes`` in the ``handling`` dict in the job conf dict is a dict, not a list by @natefoo in https://github.com/galaxyproject/gravity/pull/60 99 | 100 | 0.13.0 101 | ====== 102 | 103 | - Add options to enable/disable gunicorn, celery, and celery-beat services by @natefoo in https://github.com/galaxyproject/gravity/pull/47 104 | - Add ability to include gravity config from a separate file and document by @natefoo in https://github.com/galaxyproject/gravity/pull/48 105 | - Only default to preload = true for gunicorn if not using unicornherder by @natefoo in https://github.com/galaxyproject/gravity/pull/49 106 | - Add option to specify tusd path by @natefoo in https://github.com/galaxyproject/gravity/pull/50 107 | - Support setting per-service environment variables by @natefoo in https://github.com/galaxyproject/gravity/pull/56 108 | 109 | 0.12.0 110 | ====== 111 | 112 | - Fix typo in ``log_dir`` description by @nsoranzo in https://github.com/galaxyproject/gravity/pull/44 113 | - Shortcut individual services fix by @natefoo in https://github.com/galaxyproject/gravity/pull/45 114 | - Add additional options to celery beat / celery workers by @mvdbeek in https://github.com/galaxyproject/gravity/pull/46 115 | 116 | 0.11.0 117 | ====== 118 | 119 | - Allow setting supervisor socket path via environment variable by @mvdbeek in https://github.com/galaxyproject/gravity/pull/36 120 | - Automatically switch to non-sample galaxy.yml if it exists by @mvdbeek in https://github.com/galaxyproject/gravity/pull/39 121 | - Add pydantic config schema by @mvdbeek in https://github.com/galaxyproject/gravity/pull/42 122 | - Add --quiet option to galaxy and galaxyctl start by @mvdbeek in https://github.com/galaxyproject/gravity/pull/40 123 | - Add support for yaml job config by @mvdbeek in https://github.com/galaxyproject/gravity/pull/37 124 | - Add --preload support for gunicorn by @mvdbeek in https://github.com/galaxyproject/gravity/pull/41 125 | - Support running tusd by @natefoo in https://github.com/galaxyproject/gravity/pull/23 126 | 127 | 0.10.0 128 | ====== 129 | 130 | - Fix for the case where a job_conf.xml exists but no handlers are defined by @natefoo in https://github.com/galaxyproject/gravity/pull/24 131 | - Do not raise error if config file section is empty by @nsoranzo in https://github.com/galaxyproject/gravity/pull/25 132 | - Add tests for static handlers and a defined job_conf.xml with no handlers by @natefoo in https://github.com/galaxyproject/gravity/pull/26 133 | - Fix minor typos in readme by @ic4f in https://github.com/galaxyproject/gravity/pull/27 134 | - Move configuration to gravity key of galaxy.yml file by @mvdbeek in https://github.com/galaxyproject/gravity/pull/28 135 | - Fix for resolved galaxy.yml.sample symlink by @mvdbeek in https://github.com/galaxyproject/gravity/pull/31 136 | - Support managing gx-it-proxy via gravity by @mvdbeek in https://github.com/galaxyproject/gravity/pull/32 137 | 138 | 0.9 139 | === 140 | 141 | - Gunicorn/fastAPI support, click support, tests by @mvdbeek in https://github.com/galaxyproject/gravity/pull/14 142 | - Don't test on Python 3.6, which is unsupported by @natefoo in https://github.com/galaxyproject/gravity/pull/17 143 | - Update README. Also some various small bugfixes and fixes for other stuff mentioned in the README by @natefoo in https://github.com/galaxyproject/gravity/pull/18 144 | - Add unicornherder support by @natefoo in https://github.com/galaxyproject/gravity/pull/15 145 | - Expose the log following used by `start -f` as its own subcommand. by @natefoo in https://github.com/galaxyproject/gravity/pull/16 146 | - Better integration with Galaxy's run.sh by @natefoo in https://github.com/galaxyproject/gravity/pull/19 147 | - Use relative paths in supervisord by @natefoo in https://github.com/galaxyproject/gravity/pull/21 148 | - Converted CLI from `argparse`_ to `click`_. 149 | - Stole ideas and code from `planemo`_ in general. 150 | - Improve the AttributeDict so that it can have "hidden" items (anything that 151 | starts with a ``_``) that won't be serialized. Also, it serializes itself and 152 | can be created via deserialization from a classmethod. This simplifies using 153 | it to persist state data in the new GravityState subclass. 154 | 155 | .. _argparse: https://docs.python.org/3/library/argparse.html 156 | .. _click: http://click.pocoo.org/ 157 | .. _planemo: https://github.com/galaxyproject/planemo 158 | 159 | 0.8.3 160 | ===== 161 | 162 | - Merge ``galaxycfg`` and ``galaxyadm`` commands to ``galaxy``. 163 | 164 | 0.8.2 165 | ===== 166 | 167 | - Allow for passing names of individual services directly to ``supervisorctl`` 168 | via the ``start``, ``stop``, and ``restart`` methods. 169 | - Fix a bug where uWSGI would not start when using the automatic virtualenv 170 | install method. 171 | 172 | 0.8.1 173 | ===== 174 | 175 | - Version bump because I deleted the 0.8 files from PyPI, and despite the fact 176 | that it lets you delete them, it doesn't let you upload once they have been 177 | uploaded once... 178 | 179 | 0.8 180 | === 181 | 182 | - Add auto-register to ``galaxy start`` if it's called from the root (or 183 | subdirectory) of a Galaxy root directory. 184 | - Make ``galaxycfg remove`` accept instance names as params in addition to 185 | config file paths. 186 | - Use the same hash generated for an instance name as the hash for a generated 187 | virtualenv name, so virtualenvs are more easily identified as belonging to a 188 | config. 189 | - Renamed from ``galaxyadmin`` to ``gravity`` (thanks John Chilton). 190 | 191 | 0.7 192 | === 193 | 194 | - Added the ``galaxyadm`` subcommand ``graceful`` on a suggestion from Nicola 195 | Soranzo. 196 | - Install uWSGI into the config's virtualenv if requested. 197 | - Removed any dependence on Galaxy and eggs. 198 | - Moved project to its own repository from the Galaxy clone I'd been working 199 | from. 200 | 201 | Older 202 | ===== 203 | 204 | - Works in progress as part of the Galaxy code. 205 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015 The Pennsylvania State University 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst 2 | include HISTORY.rst 3 | include LICENSE 4 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | .. figure:: https://raw.githubusercontent.com/galaxyproject/gravity/main/docs/gravity-logo.png 2 | :alt: Gravity Logo 3 | :align: center 4 | :figwidth: 100% 5 | :target: https://github.com/galaxyproject/gravity 6 | 7 | Process management for `Galaxy`_ servers. 8 | 9 | .. image:: https://readthedocs.org/projects/gravity/badge/?version=latest 10 | :target: http://gravity.readthedocs.io/en/latest/ 11 | :alt: Documentation Status 12 | 13 | .. image:: https://badge.fury.io/py/gravity.svg 14 | :target: https://pypi.python.org/pypi/gravity/ 15 | :alt: Gravity on the Python Package Index (PyPI) 16 | 17 | .. image:: https://github.com/galaxyproject/gravity/actions/workflows/test.yaml/badge.svg 18 | :target: https://github.com/galaxyproject/gravity/actions/workflows/test.yaml 19 | 20 | * License: MIT 21 | * Documentation: https://gravity.readthedocs.io 22 | * Code: https://github.com/galaxyproject/gravity 23 | 24 | Overview 25 | ======== 26 | 27 | Modern Galaxy servers run multiple disparate processes: `gunicorn`_ for serving the web application, `celery`_ for 28 | asynchronous tasks, `tusd`_ for fault-tolerant uploads, standalone Galaxy processes for job handling, and more. Gravity 29 | is Galaxy's process manager, to make configuring and running these services simple. 30 | 31 | Installing Gravity will give you two executables, ``galaxyctl`` which is used to manage the starting, stopping, and 32 | logging of Galaxy's various processes, and ``galaxy``, which can be used to run a Galaxy server in the foreground. 33 | 34 | Quick Start 35 | =========== 36 | 37 | Installation 38 | ------------ 39 | 40 | Python 3.7 or later is required. Gravity can be installed independently of Galaxy, but it is also a dependency of 41 | Galaxy since Galaxy 22.01. If you've installed Galaxy, then Gravity is already installed in Galaxy's virtualenv. 42 | 43 | To install independently: 44 | 45 | .. code:: console 46 | 47 | $ pip install gravity 48 | 49 | Usage 50 | ----- 51 | 52 | From the root directory of a source checkout of Galaxy, after first run (or running Galaxy's 53 | ``./scripts/common_startup.sh``), activate Galaxy's virtualenv, which will put Gravity's ``galaxyctl`` and ``galaxy`` 54 | commands on your ``$PATH``: 55 | 56 | .. code:: console 57 | 58 | $ . ./.venv/bin/activate 59 | $ galaxyctl --help 60 | Usage: galaxyctl [OPTIONS] COMMAND [ARGS]... 61 | 62 | Manage Galaxy server configurations and processes. 63 | 64 | ... additional help output 65 | 66 | You can start and run Galaxy in the foreground using the ``galaxy`` command: 67 | 68 | .. code:: console 69 | 70 | $ galaxy 71 | Registered galaxy config: /srv/galaxy/config/galaxy.yml 72 | Creating or updating service gunicorn 73 | Creating or updating service celery 74 | Creating or updating service celery-beat 75 | celery: added process group 76 | 2022-01-20 14:44:24,619 INFO spawned: 'celery' with pid 291651 77 | celery-beat: added process group 78 | 2022-01-20 14:44:24,620 INFO spawned: 'celery-beat' with pid 291652 79 | gunicorn: added process group 80 | 2022-01-20 14:44:24,622 INFO spawned: 'gunicorn' with pid 291653 81 | celery STARTING 82 | celery-beat STARTING 83 | gunicorn STARTING 84 | ==> /srv/galaxy/var/gravity/log/gunicorn.log <== 85 | ...log output follows... 86 | 87 | Galaxy will continue to run and output logs to stdout until terminated with ``CTRL+C``. 88 | 89 | More detailed configuration and usage examples, especially those concerning production Galaxy servers, can be found in 90 | `the full documentation`_. 91 | 92 | .. _Galaxy: http://galaxyproject.org/ 93 | .. _gunicorn: https://gunicorn.org/ 94 | .. _celery: https://docs.celeryq.dev/ 95 | .. _tusd: https://tus.io/ 96 | .. _the full documentation: https://gravity.readthedocs.io 97 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/advanced_usage.rst: -------------------------------------------------------------------------------- 1 | Advanced Usage 2 | ============== 3 | 4 | Zero-Downtime Restarts 5 | ---------------------- 6 | 7 | Prior to Gravity 1.0, the preferred solution for performing zero-downtime restarts was `unicornherder`_. However, due to 8 | limitations in the unicornherder software, it does not always successfully perform zero-downtime restarts. Because of 9 | this, Gravity is now able to perform rolling restarts of gunicorn services if more than one gunicorn is configured. 10 | 11 | To run multiple gunicorn processes, configure the ``gunicorn`` section of the Gravity configuration as a *list*. Each 12 | item in the list is a gunicorn configuration, and can have all of the same parameters as a single gunicorn 13 | configuration: 14 | 15 | .. code:: yaml 16 | 17 | gravity: 18 | gunicorn: 19 | - bind: unix:/srv/galaxy/var/gunicorn0.sock 20 | workers: 4 21 | - bind: unix:/srv/galaxy/var/gunicorn1.sock 22 | workers: 4 23 | 24 | .. caution:: 25 | 26 | This will start multiple Galaxy servers with the same ``server_name``. If you have not configured separate Galaxy 27 | processes to act as job handlers, your gunicorn processes will handle them, resulting in job errors due to handling 28 | the same job multiple times. See the Gravity and Galaxy documentation on configuring handlers. 29 | 30 | Your proxy server can balance load between the two gunicorns. For example, with nginx: 31 | 32 | .. code:: nginx 33 | 34 | upstream galaxy { 35 | server unix:/srv/galaxy/var/gunicorn0.sock; 36 | server unix:/srv/galaxy/var/gunicorn1.sock; 37 | } 38 | 39 | http { 40 | location / { 41 | proxy_pass http://galaxy; 42 | } 43 | } 44 | 45 | By default, Gravity will wait 300 seconds for the gunicorn server to respond to web requests after initiating the 46 | restart. To change this timeout this, set the ``restart_timeout`` option on each configured ``gunicorn`` instance. 47 | 48 | Service Instances 49 | ----------------- 50 | 51 | In the case of multiple gunicorn instances as described in :ref:`Zero-Downtime Restarts` and multiple dynamic handlers 52 | as described in :ref:`Galaxy Job Handlers`, Gravity will create multiple *service instances* of each service. This 53 | allows multiple processes to be run from a single service definition. 54 | 55 | In supervisor, this means that the service names as presented by supervisor are appended with ``:INSTANCE_NUMBER``, 56 | e.g.: 57 | 58 | .. code:: console 59 | 60 | $ galaxyctl status 61 | celery RUNNING pid 121363, uptime 0:02:33 62 | celery-beat RUNNING pid 121364, uptime 0:02:33 63 | gunicorn:gunicorn_0 RUNNING pid 121365, uptime 0:02:33 64 | gunicorn:gunicorn_1 RUNNING pid 121366, uptime 0:02:33 65 | 66 | However, ``galaxyctl`` commands that take a service name still use the base service name, e.g.: 67 | 68 | .. code:: console 69 | 70 | $ galaxyctl stop gunicorn 71 | gunicorn:gunicorn_0: stopped 72 | gunicorn:gunicorn_1: stopped 73 | Not all processes stopped, supervisord not shut down (hint: see `galaxyctl status`) 74 | 75 | In systemd, the service names as presented by systemd are appended with ``@INSTANCE_NUMBER``, 76 | e.g.: 77 | 78 | .. code:: console 79 | 80 | $ galaxyctl status 81 | UNIT LOAD ACTIVE SUB DESCRIPTION 82 | galaxy-celery-beat.service loaded active running Galaxy celery-beat 83 | galaxy-celery.service loaded active running Galaxy celery 84 | galaxy-gunicorn@0.service loaded active running Galaxy gunicorn (process 0) 85 | galaxy-gunicorn@1.service loaded active running Galaxy gunicorn (process 1) 86 | galaxy.target loaded active active Galaxy 87 | 88 | As with supervisor, ``galaxyctl`` commands that take a service name still use the base service name. 89 | 90 | If you prefer not to work with service instances and want Galaxy to write a service configuration file for each instance 91 | of each service, you can do so by setting ``use_service_instances`` in the Gravity configuration to ``false``. 92 | 93 | Managing Multiple Galaxies 94 | -------------------------- 95 | 96 | Gravity can manage multiple instances of Galaxy simultaneously. This is useful especially in the case where you have 97 | multiple production Galaxy instances on a single server and are managing them with Gravity installed outside of a Galaxy 98 | virtualenv, as root. There are multiple ways to achieve this: 99 | 100 | 1. Pass multiple ``--config-file`` options to ``galaxyctl``, or set a list of colon-separated config paths in 101 | ``$GRAVITY_CONFIG_FILE``: 102 | 103 | .. code:: console 104 | 105 | $ galaxyctl --config-file /srv/galaxy/test/config/galaxy.yml \ 106 | --config-file /srv/galaxy/main/config/galaxy.yml list --version 107 | TYPE INSTANCE NAME VERSION CONFIG PATH 108 | galaxy test 22.05 /srv/galaxy/test/config/galaxy.yml 109 | galaxy main 22.09.dev0 /srv/galaxy/main/config/galaxy.yml 110 | $ export GRAVITY_CONFIG_FILE='/srv/galaxy/test/config/galaxy.yml:/srv/galaxy/main/config/galaxy.yml' 111 | $ galaxyctl list --version 112 | TYPE INSTANCE NAME VERSION CONFIG PATH 113 | galaxy test 22.05 /srv/galaxy/test/config/galaxy.yml 114 | galaxy main 22.09.dev0 /srv/galaxy/main/config/galaxy.yml 115 | 116 | 2. If running as root, any config files located in ``/etc/galaxy/gravity.d`` will automatically be loaded. 117 | 118 | 3. Specify multiple Gravity configurations in a single config file, as a list. In this case, the Galaxy and Gravity 119 | configurations must be in separate files as described in :ref:`Splitting Gravity and Galaxy Configurations`: 120 | 121 | .. code:: yaml 122 | 123 | gravity: 124 | - instance_name: test 125 | process_manager: systemd 126 | galaxy_config_file: /srv/galaxy/test/config/galaxy.yml 127 | galaxy_root: /srv/galaxy/test/server 128 | virtualenv: /srv/galaxy/test/venv 129 | galaxy_user: gxtest 130 | gunicorn: 131 | bind: unix:/srv/galaxy/test/var/gunicorn.sock 132 | handlers: 133 | handler: 134 | pools: 135 | - job-handlers 136 | - workflow-schedulers 137 | 138 | - instance_name: main 139 | process_manager: systemd 140 | galaxy_config_file: /srv/galaxy/main/config/galaxy.yml 141 | galaxy_root: /srv/galaxy/main/server 142 | virtualenv: /srv/galaxy/main/venv 143 | galaxy_user: gxmain 144 | gunicorn: 145 | bind: unix:/srv/galaxy/main/var/gunicorn.sock 146 | workers: 8 147 | handlers: 148 | handler: 149 | processes: 4 150 | pools: 151 | - job-handlers 152 | - workflow-schedulers 153 | 154 | In all cases, when using multiple Gravity instances, each Galaxy instance managed by Gravity must have a unique 155 | **instance name**. When working with a single instance, the default name ``_default_`` is used automatically and mostly 156 | hidden from you. When working with multiple instances, set the ``instance_name`` option in each instance's Gravity 157 | config to a unique name. 158 | 159 | Although it is strongly encouraged to use systemd for running multiple instances, it is possible to use supervisor. 160 | Please see the :ref:`Gravity State` section for important details on how and where Gravity stores the supervisor 161 | configuration and log files. 162 | 163 | .. _unicornherder: https://github.com/alphagov/unicornherder 164 | -------------------------------------------------------------------------------- /docs/basic_usage.rst: -------------------------------------------------------------------------------- 1 | Basic Usage 2 | =========== 3 | 4 | A basic example of starting and running a simple Galaxy server from a source clone in the foreground is provided in the 5 | ref:`Quick Start` guide. This section covers more typical usage for production Galaxy servers. 6 | 7 | Managing a Single Galaxy 8 | ------------------------ 9 | 10 | If you have not installed Gravity separate from the Galaxy virtualenv, simply activate Galaxy's virtualenv, which will 11 | put Gravity's ``galaxyctl`` and ``galaxy`` commands on your ``$PATH``: 12 | 13 | .. code:: console 14 | 15 | $ . /srv/galaxy/venv/bin/activate 16 | $ galaxyctl --help 17 | Usage: galaxyctl [OPTIONS] COMMAND [ARGS]... 18 | 19 | Manage Galaxy server configurations and processes. 20 | 21 | Options: 22 | -d, --debug Enables debug mode. 23 | -c, --config-file FILE Gravity (or Galaxy) config file to operate on. Can 24 | also be set with $GRAVITY_CONFIG_FILE or 25 | $GALAXY_CONFIG_FILE 26 | --state-dir DIRECTORY Where process management configs and state will be 27 | stored. 28 | -h, --help Show this message and exit. 29 | 30 | Commands: 31 | configs List registered config files. 32 | deregister Deregister config file(s). 33 | exec Run a single Galaxy service in the foreground, with logging... 34 | follow Follow log files of configured instances. 35 | graceful Gracefully reload configured services. 36 | instances List all known instances. 37 | pm Invoke process manager (supervisorctl, systemctl) directly. 38 | register Register config file(s). 39 | rename Update path of registered config file. 40 | restart Restart configured services. 41 | show Show details of registered config. 42 | shutdown Shut down process manager. 43 | start Start configured services. 44 | status Display server status. 45 | stop Stop configured services. 46 | update Update process manager from config changes. 47 | 48 | If you run ``galaxy`` or ``galaxyctl`` from the root of a Galaxy source checkout and do not specify the config file 49 | option, ``config/galaxy.yml`` or ``config/galaxy.yml.sample`` will be automatically used. This is handy for working with 50 | local clones of Galaxy for testing or development. You can skip Galaxy's lengthy and repetitive ``run.sh`` configuration 51 | steps when starting and stopping Galaxy in between code updates (you should still run ``run.sh`` after performing a 52 | ``git pull`` to make sure your dependencies are up to date). 53 | 54 | Gravity can either run Galaxy via the `supervisor`_ process manager (the default) or `systemd`_. For production servers, 55 | **it is recommended that you run Gravity as root in systemd mode**. See the :ref:`Managing a Production Galaxy` section 56 | for details. 57 | 58 | As shown in the Quick Start, the ``galaxy`` command will run a Galaxy server in the foreground. The ``galaxy`` command 59 | is actually a shortcut for two separate steps: 1. read the provided ``galaxy.yml`` and write out the corresponding 60 | process manager configurations, and 2. start and run Galaxy in the foreground using the process manager (`supervisor`_). 61 | You can perform these steps separately (and in this example, start Galaxy as a backgrounded daemon instead of in the 62 | foreground): 63 | 64 | .. code:: console 65 | 66 | $ galaxyctl update 67 | Adding service gunicorn 68 | Adding service celery 69 | Adding service celery-beat 70 | $ galaxyctl start 71 | celery STARTING 72 | celery-beat STARTING 73 | gunicorn STARTING 74 | Log files are in /srv/galaxy/var/gravity/log 75 | 76 | When running as a daemon, the ``stop`` subcommand stops your Galaxy server: 77 | 78 | .. code:: console 79 | 80 | $ galaxyctl stop 81 | celery-beat: stopped 82 | gunicorn: stopped 83 | celery: stopped 84 | All processes stopped, supervisord will exit 85 | Shut down 86 | 87 | Most Gravity subcommands (such as ``stop``, ``start``, ``restart``, ...) are straightforward, but a few subcommands are 88 | worth pointing out: :ref:`update`, :ref:`graceful`, and :ref:`exec`. All subcommands are documented in the 89 | :ref:`Subcommands` section and their respective ``--help`` output. 90 | 91 | Managing a Production Galaxy 92 | ---------------------------- 93 | 94 | By default, Gravity runs Galaxy processes under `supervisor`_, but setting the ``process_manager`` option to ``systemd`` 95 | in Gravity's configuration will cause it to run under `systemd`_ instead. systemd is the default init system under most 96 | modern Linux distributions, and using systemd is strongly encouraged for production Galaxy deployments. 97 | 98 | Gravity manages `systemd service unit files`_ corresponding to all of the Galaxy services that it is aware of, much like 99 | how it manages supervisor program config files in supervisor mode. If you run ``galaxyctl update`` as a non-root user, 100 | the unit files will be installed in ``~/.config/systemd/user`` and run via `systemd user mode`_. This can be useful for 101 | testing and development, but in production it is recommended to run Gravity as root, so that it installs the service 102 | units in ``/etc/systemd/system`` and are managed by the privileged systemd instance. Even when Gravity is run as root, 103 | Galaxy itself still runs as a non-root user, specified by the ``galaxy_user`` option in the Gravity configuration. 104 | 105 | It is also recommended, when running as root, that you install Gravity independent of Galaxy, rather than use the copy 106 | installed in Galaxy's virtualenv: 107 | 108 | .. code:: console 109 | 110 | # python3 -m venv /opt/gravity 111 | # /opt/gravity/bin/pip install gravity 112 | 113 | .. caution:: 114 | 115 | Because systemd unit file names have semantic meaning (the filename is the service's name) and systemd does not have 116 | a facility for isolating unit files controlled by an application, Gravity considers all unit files in the unit dir 117 | (``/etc/systemd/system``) that are named like ``galaxy-*`` to be controlled by Gravity. **If you have existing unit 118 | files that are named as such, Gravity will overwrite or remove them.** 119 | 120 | In systemd mode, and especially when run as root, some Gravity options are required: 121 | 122 | .. code:: yaml 123 | 124 | gravity: 125 | process_manager: systemd 126 | 127 | # required if running as root 128 | galaxy_user: GALAXY-USERNAME 129 | # optional, defaults to primary group of the user set above 130 | galaxy_group: GALAXY-GROUPNAME 131 | 132 | # required 133 | virtualenv: /srv/galaxy/venv 134 | # probably necessary if your galaxy.yml is not in galaxy_root/config 135 | galaxy_root: /srv/galaxy/server 136 | 137 | See the :ref:`Configuration` section for more details on these options and others. 138 | 139 | The ``log_dir`` option is ignored when using systemd. Logs are instead captured by systemd's logging facility, 140 | ``journald``. 141 | 142 | You can use ``galaxyctl`` to manage Galaxy process starts/stops/restarts/etc. and follow the logs, just as you do under 143 | supervisor, but you can also use ``systemctl`` and ``journalctl`` directly to manage process states and inspect logs 144 | (respectively). Only ``galaxyctl update`` is necessary, in order to write and/or remove the appropriate systemd service 145 | units based on your configuration. For example: 146 | 147 | .. code:: console 148 | 149 | # export GRAVITY_CONFIG_FILE=/srv/galaxy/config/galaxy.yml 150 | # . /srv/galaxy/venv/bin/activate 151 | (venv) # galaxyctl update 152 | Adding service galaxy-gunicorn.service 153 | Adding service galaxy-celery.service 154 | Adding service galaxy-celery-beat.service 155 | 156 | After this point, operations can be performed with either ``galaxyctl`` or ``systemctl``. Some examples of equivalent 157 | commands: 158 | 159 | =================================== ================================================================== 160 | Gravity systemd 161 | =================================== ================================================================== 162 | ``galaxy`` ``systemctl start galaxy.target && journalctl -f -u 'galaxy-*'`` 163 | ``galaxyctl start`` ``systemctl start galaxy.target`` 164 | ``galaxyctl start SERVICE ...`` ``systemctl start galaxy-SERVICE.service galaxy-...`` 165 | ``galaxyctl restart`` ``systemctl restart galaxy.target`` 166 | ``galaxyctl restart SERVICE ...`` ``systemctl restart galaxy-SERVICE.service galaxy-...`` 167 | ``galaxyctl graceful`` ``systemctl reload-or-restart galaxy.target`` 168 | ``galaxyctl graceful SERVICE ...`` ``systemctl reload-or-restart galaxy-SERVICE.service galaxy-...`` 169 | ``galaxyctl stop`` ``systemctl start galaxy.target`` 170 | ``galayxctl follow`` ``journalctl -f -u 'galaxy-*'`` 171 | =================================== ================================================================== 172 | 173 | .. _supervisor: http://supervisord.org/ 174 | .. _systemd: https://www.freedesktop.org/wiki/Software/systemd/ 175 | .. _systemd service unit files: https://www.freedesktop.org/software/systemd/man/systemd.unit.html 176 | .. _systemd user mode: https://www.freedesktop.org/software/systemd/man/user@.service.html 177 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # For the full list of built-in configuration values, see the documentation: 4 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 5 | 6 | # -- Project information ----------------------------------------------------- 7 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information 8 | import os 9 | import sys 10 | 11 | import sphinx_rtd_theme 12 | 13 | sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) 14 | 15 | from gravity import __version__ # noqa: E402 16 | 17 | project = 'Gravity' 18 | copyright = '2022, The Galaxy Project' 19 | author = 'The Galaxy Project' 20 | release = __version__ 21 | 22 | # -- General configuration --------------------------------------------------- 23 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration 24 | 25 | master_doc = 'index' 26 | 27 | extensions = ['sphinx.ext.autosectionlabel'] 28 | 29 | templates_path = ['_templates'] 30 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 31 | 32 | 33 | # -- Options for HTML output ------------------------------------------------- 34 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output 35 | 36 | html_theme = "sphinx_rtd_theme" 37 | html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] 38 | -------------------------------------------------------------------------------- /docs/conventions.rst: -------------------------------------------------------------------------------- 1 | Documentation Conventions 2 | ========================= 3 | 4 | Examples in this documentation assume a Galaxy layout like the one used in the `Galaxy Installation with Ansible`_ 5 | tutorial:: 6 | 7 | /srv/galaxy/server # Galaxy code 8 | /srv/galaxy/config # config files 9 | /srv/galaxy/venv # virtualenv 10 | 11 | .. _Galaxy Installation with Ansible: https://training.galaxyproject.org/training-material/topics/admin/tutorials/ansible-galaxy/tutorial.html 12 | -------------------------------------------------------------------------------- /docs/gravity-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/galaxyproject/gravity/20038d06c677ead44e555a0bc1faeb3f475d85b3/docs/gravity-logo.png -------------------------------------------------------------------------------- /docs/history.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../HISTORY.rst 2 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | Gravity 3 | ======= 4 | 5 | .. toctree:: 6 | :maxdepth: 2 7 | :caption: Contents: 8 | 9 | readme 10 | conventions 11 | installation 12 | basic_usage 13 | advanced_usage 14 | subcommands 15 | history 16 | 17 | 18 | Indices and tables 19 | ================== 20 | 21 | * :ref:`genindex` 22 | * :ref:`search` 23 | -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- 1 | Installation and Configuration 2 | ============================== 3 | 4 | Installation 5 | ------------ 6 | 7 | Python 3.7 or later is required. Gravity can be installed independently of Galaxy, but it is also a dependency of 8 | Galaxy since Galaxy 22.01. If you've installed Galaxy, then Gravity is already installed in Galaxy's virtualenv. 9 | 10 | To install independently: 11 | 12 | .. code:: console 13 | 14 | $ pip install gravity 15 | 16 | To make your life easier, you are encourged to install into a `virtualenv`_. The easiest way to do this is with Python's 17 | built-in `venv`_ module: 18 | 19 | .. code:: console 20 | 21 | $ python3 -m venv ~/gravity 22 | $ . ~/gravity/bin/activate 23 | 24 | Configuration 25 | ------------- 26 | 27 | Gravity needs to know where your Galaxy configuration file is, and depending on your Galaxy layout, some additional 28 | details like the paths to its virtualenv and root directory. By default, Gravity's configuration is defined in Galaxy's 29 | configuration file (``galaxy.yml``) to be easy and familiar for Galaxy administrators. Gravity's configuration is 30 | defined underneath the ``gravity`` key, and Galaxy's configuration is defined underneath the ``galaxy`` key. For 31 | example: 32 | 33 | .. code:: yaml 34 | 35 | --- 36 | gravity: 37 | gunicorn: 38 | bind: localhost:8192 39 | galaxy: 40 | database_connection: postgresql:///galaxy 41 | 42 | Configuration Search Paths 43 | """""""""""""""""""""""""" 44 | 45 | If you run ``galaxy`` or ``galaxyctl`` from the root of a Galaxy source checkout and do not specify the config file 46 | option, ``config/galaxy.yml`` or ``config/galaxy.yml.sample`` will be automatically used. To avoid having to run from 47 | the Galaxy root directory or to work with a config file in a different location, you can explicitly point Gravity at 48 | your Galaxy configuration file with the ``--config-file`` (``-c``) option or the ``$GRAVITY_CONFIG_FILE`` (or 49 | ``$GALAXY_CONFIG_FILE``, as set by Galaxy's ``run.sh`` script) environment variable. Then it's possible to run the 50 | ``galaxyctl`` command from anywhere. 51 | 52 | Often times it's convenient to put the environment variable in the Galaxy user's shell environment file, e.g.: 53 | 54 | .. code:: console 55 | 56 | $ echo "export GRAVITY_CONFIG_FILE='/srv/galaxy/config/galaxy.yml'" >> ~/.bash_profile 57 | 58 | When running Gravity as root, the following configuration files will automatically be searched for and read, unless 59 | ``--config-file`` is specified or ``$GRAVITY_CONFIG_FILE`` is set: 60 | 61 | - ``/etc/galaxy/gravity.yml`` 62 | - ``/etc/galaxy/galaxy.yml`` 63 | - ``/etc/galaxy/gravity.d/*.y(a?)ml`` 64 | 65 | Splitting Gravity and Galaxy Configurations 66 | """"""""""""""""""""""""""""""""""""""""""" 67 | 68 | For more advanced deployments, it is *not* necessary to write your entire Galaxy configuration to the Gravity config 69 | file. You can write only the Gravity configuration, and then point to your Galaxy config file with the 70 | ``galaxy_config_file`` option in the Gravity config. This can be useful for cases such as your Galaxy server being split 71 | across multiple hosts. 72 | 73 | For example, on a deployment where the web (gunicorn) and job handler processes run on different hosts, one might have: 74 | 75 | In ``gravity.yml`` on the web host: 76 | 77 | .. code:: yaml 78 | 79 | --- 80 | gravity: 81 | galaxy_config_file: galaxy.yml 82 | log_dir: /var/log/galaxy 83 | gunicorn: 84 | bind: localhost:8888 85 | celery: 86 | enable: false 87 | enable_beat: false 88 | 89 | In ``gravity.yml`` on the job handler host: 90 | 91 | .. code:: yaml 92 | 93 | --- 94 | gravity: 95 | galaxy_config_file: galaxy.yml 96 | log_dir: /var/log/galaxy 97 | gunicorn: 98 | enable: false 99 | celery: 100 | enable: true 101 | enable_beat: true 102 | handlers: 103 | handler: 104 | processes: 2 105 | 106 | See the :ref:`Managing Multiple Galaxies` section for additional examples. 107 | 108 | Configuration Options 109 | --------------------- 110 | 111 | The following options in the ``gravity`` section of ``galaxy.yml`` can be used to configure Gravity: 112 | 113 | .. code:: yaml 114 | 115 | # Configuration for Gravity process manager. 116 | # ``uwsgi:`` section will be ignored if Galaxy is started via Gravity commands (e.g ``./run.sh``, ``galaxy`` or ``galaxyctl``). 117 | gravity: 118 | 119 | # Process manager to use. 120 | # ``supervisor`` is the default process manager when Gravity is invoked as a non-root user. 121 | # ``systemd`` is the default when Gravity is invoked as root. 122 | # Valid options are: supervisor, systemd 123 | # process_manager: 124 | 125 | # What command to write to the process manager configs 126 | # `gravity` (`galaxyctl exec `) is the default 127 | # `direct` (each service's actual command) is also supported. 128 | # Valid options are: gravity, direct 129 | # service_command_style: gravity 130 | 131 | # Use the process manager's *service instance* functionality for services that can run multiple instances. 132 | # Presently this includes services like gunicorn and Galaxy dynamic job handlers. Service instances are only supported if 133 | # ``service_command_style`` is ``gravity``, and so this option is automatically set to ``false`` if 134 | # ``service_command_style`` is set to ``direct``. 135 | # use_service_instances: true 136 | 137 | # umask under which services should be executed. Setting ``umask`` on an individual service overrides this value. 138 | # umask: '022' 139 | 140 | # Memory limit (in GB), processes exceeding the limit will be killed. Default is no limit. If set, this is default value 141 | # for all services. Setting ``memory_limit`` on an individual service overrides this value. Ignored if ``process_manager`` 142 | # is ``supervisor``. 143 | # memory_limit: 144 | 145 | # Specify Galaxy config file (galaxy.yml), if the Gravity config is separate from the Galaxy config. Assumed to be the 146 | # same file as the Gravity config if a ``galaxy`` key exists at the root level, otherwise, this option is required. 147 | # galaxy_config_file: 148 | 149 | # Specify Galaxy's root directory. 150 | # Gravity will attempt to find the root directory, but you can set the directory explicitly with this option. 151 | # galaxy_root: 152 | 153 | # User to run Galaxy as, required when using the systemd process manager as root. 154 | # Ignored if ``process_manager`` is ``supervisor`` or user-mode (non-root) ``systemd``. 155 | # galaxy_user: 156 | 157 | # Group to run Galaxy as, optional when using the systemd process manager as root. 158 | # Ignored if ``process_manager`` is ``supervisor`` or user-mode (non-root) ``systemd``. 159 | # galaxy_group: 160 | 161 | # Set to a directory that should contain log files for the processes controlled by Gravity. 162 | # If not specified defaults to ``/gravity/log``. 163 | # log_dir: 164 | 165 | # Set to Galaxy's virtualenv directory. 166 | # If not specified, Gravity assumes all processes are on PATH. This option is required in most circumstances when using 167 | # the ``systemd`` process manager. 168 | # virtualenv: 169 | 170 | # Select the application server. 171 | # ``gunicorn`` is the default application server. 172 | # ``unicornherder`` is a production-oriented manager for (G)unicorn servers that automates zero-downtime Galaxy server restarts, 173 | # similar to uWSGI Zerg Mode used in the past. 174 | # Valid options are: gunicorn, unicornherder 175 | # app_server: gunicorn 176 | 177 | # Override the default instance name. 178 | # this is hidden from you when running a single instance. 179 | # instance_name: _default_ 180 | 181 | # Configuration for Gunicorn. Can be a list to run multiple gunicorns for rolling restarts. 182 | gunicorn: 183 | 184 | # Enable Galaxy gunicorn server. 185 | # enable: true 186 | 187 | # The socket to bind. A string of the form: ``HOST``, ``HOST:PORT``, ``unix:PATH``, ``fd://FD``. An IP is a valid HOST. 188 | # bind: localhost:8080 189 | 190 | # Controls the number of Galaxy application processes Gunicorn will spawn. 191 | # Increased web performance can be attained by increasing this value. 192 | # If Gunicorn is the only application on the server, a good starting value is the number of CPUs * 2 + 1. 193 | # 4-12 workers should be able to handle hundreds if not thousands of requests per second. 194 | # workers: 1 195 | 196 | # Gunicorn workers silent for more than this many seconds are killed and restarted. 197 | # Value is a positive number or 0. Setting it to 0 has the effect of infinite timeouts by disabling timeouts for all workers entirely. 198 | # If you disable the ``preload`` option workers need to have finished booting within the timeout. 199 | # timeout: 300 200 | 201 | # Extra arguments to pass to Gunicorn command line. 202 | # extra_args: 203 | 204 | # Use Gunicorn's --preload option to fork workers after loading the Galaxy Application. 205 | # Consumes less memory when multiple processes are configured. Default is ``false`` if using unicornherder, else ``true``. 206 | # preload: 207 | 208 | # umask under which service should be executed 209 | # umask: 210 | 211 | # Value of supervisor startsecs, systemd TimeoutStartSec 212 | # start_timeout: 15 213 | 214 | # Value of supervisor stopwaitsecs, systemd TimeoutStopSec 215 | # stop_timeout: 65 216 | 217 | # Amount of time to wait for a server to become alive when performing rolling restarts. 218 | # restart_timeout: 300 219 | 220 | # Memory limit (in GB). If the service exceeds the limit, it will be killed. Default is no limit or the value of the 221 | # ``memory_limit`` setting at the top level of the Gravity configuration, if set. Ignored if ``process_manager`` is 222 | # ``supervisor``. 223 | # memory_limit: 224 | 225 | # Extra environment variables and their values to set when running the service. A dictionary where keys are the variable 226 | # names. 227 | # environment: {} 228 | 229 | # Configuration for Celery Processes. 230 | celery: 231 | 232 | # Enable Celery distributed task queue. 233 | # enable: true 234 | 235 | # Enable Celery Beat periodic task runner. 236 | # enable_beat: true 237 | 238 | # Number of Celery Workers to start. 239 | # concurrency: 2 240 | 241 | # Log Level to use for Celery Worker. 242 | # Valid options are: DEBUG, INFO, WARNING, ERROR 243 | # loglevel: DEBUG 244 | 245 | # Queues to join 246 | # queues: celery,galaxy.internal,galaxy.external 247 | 248 | # Pool implementation 249 | # Valid options are: prefork, eventlet, gevent, solo, processes, threads 250 | # pool: threads 251 | 252 | # Extra arguments to pass to Celery command line. 253 | # extra_args: 254 | 255 | # umask under which service should be executed 256 | # umask: 257 | 258 | # Value of supervisor startsecs, systemd TimeoutStartSec 259 | # start_timeout: 10 260 | 261 | # Value of supervisor stopwaitsecs, systemd TimeoutStopSec 262 | # stop_timeout: 10 263 | 264 | # Memory limit (in GB). If the service exceeds the limit, it will be killed. Default is no limit or the value of the 265 | # ``memory_limit`` setting at the top level of the Gravity configuration, if set. Ignored if ``process_manager`` is 266 | # ``supervisor``. 267 | # memory_limit: 268 | 269 | # Extra environment variables and their values to set when running the service. A dictionary where keys are the variable 270 | # names. 271 | # environment: {} 272 | 273 | # Configuration for gx-it-proxy. 274 | gx_it_proxy: 275 | 276 | # Set to true to start gx-it-proxy 277 | # enable: false 278 | 279 | # gx-it-proxy version 280 | # version: '>=0.0.6' 281 | 282 | # Public-facing IP of the proxy 283 | # ip: localhost 284 | 285 | # Public-facing port of the proxy 286 | # port: 4002 287 | 288 | # Database to monitor. 289 | # Should be set to the same value as ``interactivetools_map`` (or ``interactivetoolsproxy_map``) in the ``galaxy:`` section. This is 290 | # ignored if either ``interactivetools_map`` or ``interactivetoolsproxy_map`` are set. 291 | # sessions: database/interactivetools_map.sqlite 292 | 293 | # Include verbose messages in gx-it-proxy 294 | # verbose: true 295 | 296 | # Forward all requests to IP. 297 | # This is an advanced option that is only needed when proxying to remote interactive tool container that cannot be reached through the local network. 298 | # forward_ip: 299 | 300 | # Forward all requests to port. 301 | # This is an advanced option that is only needed when proxying to remote interactive tool container that cannot be reached through the local network. 302 | # forward_port: 303 | 304 | # Rewrite location blocks with proxy port. 305 | # This is an advanced option that is only needed when proxying to remote interactive tool container that cannot be reached through the local network. 306 | # reverse_proxy: false 307 | 308 | # umask under which service should be executed 309 | # umask: 310 | 311 | # Value of supervisor startsecs, systemd TimeoutStartSec 312 | # start_timeout: 10 313 | 314 | # Value of supervisor stopwaitsecs, systemd TimeoutStopSec 315 | # stop_timeout: 10 316 | 317 | # Memory limit (in GB). If the service exceeds the limit, it will be killed. Default is no limit or the value of the 318 | # ``memory_limit`` setting at the top level of the Gravity configuration, if set. Ignored if ``process_manager`` is 319 | # ``supervisor``. 320 | # memory_limit: 321 | 322 | # Extra environment variables and their values to set when running the service. A dictionary where keys are the variable 323 | # names. 324 | # environment: {} 325 | 326 | # Configuration for tusd server (https://github.com/tus/tusd). 327 | # The ``tusd`` binary must be installed manually and made available on PATH (e.g in galaxy's .venv/bin directory). 328 | tusd: 329 | 330 | # Enable tusd server. 331 | # If enabled, you also need to set up your proxy as outlined in https://docs.galaxyproject.org/en/latest/admin/nginx.html#receiving-files-via-the-tus-protocol. 332 | # enable: false 333 | 334 | # Path to tusd binary 335 | # tusd_path: tusd 336 | 337 | # Host to bind the tusd server to 338 | # host: localhost 339 | 340 | # Port to bind the tusd server to 341 | # port: 1080 342 | 343 | # Directory to store uploads in. 344 | # Must match ``tus_upload_store`` setting in ``galaxy:`` section. 345 | # upload_dir: 346 | 347 | # Comma-separated string of enabled tusd hooks. 348 | # 349 | # Leave at the default value to require authorization at upload creation time. 350 | # This means Galaxy's web process does not need to be running after creating the initial 351 | # upload request. 352 | # 353 | # Set to empty string to disable all authorization. This means data can be uploaded (but not processed) 354 | # without the Galaxy web process being available. 355 | # 356 | # You can find a list of available hooks at https://github.com/tus/tusd/blob/master/docs/hooks.md#list-of-available-hooks. 357 | # hooks_enabled_events: pre-create 358 | 359 | # Extra arguments to pass to tusd command line. 360 | # extra_args: 361 | 362 | # umask under which service should be executed 363 | # umask: 364 | 365 | # Value of supervisor startsecs, systemd TimeoutStartSec 366 | # start_timeout: 10 367 | 368 | # Value of supervisor stopwaitsecs, systemd TimeoutStopSec 369 | # stop_timeout: 10 370 | 371 | # Memory limit (in GB). If the service exceeds the limit, it will be killed. Default is no limit or the value of the 372 | # ``memory_limit`` setting at the top level of the Gravity configuration, if set. Ignored if ``process_manager`` is 373 | # ``supervisor``. 374 | # memory_limit: 375 | 376 | # Extra environment variables and their values to set when running the service. A dictionary where keys are the variable 377 | # names. 378 | # environment: {} 379 | 380 | # Configuration for Galaxy Reports. 381 | reports: 382 | 383 | # Enable Galaxy Reports server. 384 | # enable: false 385 | 386 | # Path to reports.yml, relative to galaxy.yml if not absolute 387 | # config_file: reports.yml 388 | 389 | # The socket to bind. A string of the form: ``HOST``, ``HOST:PORT``, ``unix:PATH``, ``fd://FD``. An IP is a valid HOST. 390 | # bind: localhost:9001 391 | 392 | # Controls the number of Galaxy Reports application processes Gunicorn will spawn. 393 | # It is not generally necessary to increase this for the low-traffic Reports server. 394 | # workers: 1 395 | 396 | # Gunicorn workers silent for more than this many seconds are killed and restarted. 397 | # Value is a positive number or 0. Setting it to 0 has the effect of infinite timeouts by disabling timeouts for all workers entirely. 398 | # timeout: 300 399 | 400 | # URL prefix to serve from. 401 | # The corresponding nginx configuration is (replace and with the values from these options): 402 | # 403 | # location // { 404 | # proxy_pass http:///; 405 | # } 406 | # 407 | # If is a unix socket, you will need a ``:`` after the socket path but before the trailing slash like so: 408 | # proxy_pass http://unix:/run/reports.sock:/; 409 | # url_prefix: 410 | 411 | # Extra arguments to pass to Gunicorn command line. 412 | # extra_args: 413 | 414 | # umask under which service should be executed 415 | # umask: 416 | 417 | # Value of supervisor startsecs, systemd TimeoutStartSec 418 | # start_timeout: 10 419 | 420 | # Value of supervisor stopwaitsecs, systemd TimeoutStopSec 421 | # stop_timeout: 10 422 | 423 | # Memory limit (in GB). If the service exceeds the limit, it will be killed. Default is no limit or the value of the 424 | # ``memory_limit`` setting at the top level of the Gravity configuration, if set. Ignored if ``process_manager`` is 425 | # ``supervisor``. 426 | # memory_limit: 427 | 428 | # Extra environment variables and their values to set when running the service. A dictionary where keys are the variable 429 | # names. 430 | # environment: {} 431 | 432 | # Configure dynamic handlers in this section. 433 | # See https://docs.galaxyproject.org/en/latest/admin/scaling.html#dynamically-defined-handlers for details. 434 | # handlers: {} 435 | 436 | Galaxy Job Handlers 437 | ------------------- 438 | 439 | Gravity has support for reading Galaxy's job configuration: it can read statically configured job handlers in the 440 | ``job_conf.yml`` or ``job_conf.xml`` files, or the job configuration inline from the ``job_config`` option in 441 | ``galaxy.yml``. However, unless you need to statically define handlers, it is simpler to configure Gravity to run 442 | `dynamically defined handlers`_ as detailed in the Galaxy scaling documentation. 443 | 444 | When using dynamically defined handlers, be sure to explicitly set the `job handler assignment method`_ to 445 | ``db-skip-locked`` or ``db-transaction-isolation`` to prevent the web process from also handling jobs. 446 | 447 | Gravity State 448 | ------------- 449 | 450 | Older versions of Gravity stored a considerable amount of *config state* in ``$GRAVITY_STATE_DIR/configstate.yaml``. As 451 | of version 1.0.0, Gravity does not store state information, and this file can be removed if left over from an older 452 | installation. 453 | 454 | Although Gravity no longer uses the config state file, it does still use a state directory for storing supervisor 455 | configs, the default log directory (if ``log_dir`` is unchanged), and the celery-beat database. This directory defaults 456 | to ``/database/gravity/`` by way of the ``data_dir`` option in the ``galaxy`` section of ``galaxy.yml`` 457 | (which defaults to ``/database/``). 458 | 459 | If running multiple Galaxy servers with the same Gravity configuration as described in :ref:`Managing Multiple Galaxies` 460 | and if doing so using supervisor rather than systemd, the supervisor configurations will be stored in 461 | ``$XDG_CONFIG_HOME/galaxy-gravity`` (``$XDG_CONFIG_HOME`` defaults to ``~/.config/galaxy-gravity``) 462 | 463 | In any case, you can override the path to the state directory using the ``--state-dir`` option, or the 464 | ``$GRAVITY_STATE_DIR`` environment variable. 465 | 466 | .. note:: 467 | 468 | Galaxy 22.01 and 22.05 automatically set ``$GRAVITY_STATE_DIR`` to ``/database/gravity`` in the 469 | virtualenv's activation script, ``activate``. This can be removed from the activate script when using Gravity 1.0.0 470 | or later. 471 | 472 | .. _virtualenv: https://virtualenv.pypa.io/ 473 | .. _venv: https://docs.python.org/3/library/venv.html 474 | .. _dynamically defined handlers: https://docs.galaxyproject.org/en/latest/admin/scaling.html#dynamically-defined-handlers 475 | .. _job handler assignment method: https://docs.galaxyproject.org/en/master/admin/scaling.html#job-handler-assignment-methods 476 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | %SPHINXBUILD% >NUL 2>NUL 14 | if errorlevel 9009 ( 15 | echo. 16 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 17 | echo.installed, then set the SPHINXBUILD environment variable to point 18 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 19 | echo.may add the Sphinx directory to PATH. 20 | echo. 21 | echo.If you don't have Sphinx installed, grab it from 22 | echo.https://www.sphinx-doc.org/ 23 | exit /b 1 24 | ) 25 | 26 | if "%1" == "" goto help 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/readme.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../README.rst 2 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx>=2 2 | sphinx-rtd-theme>=0.5.2 3 | -------------------------------------------------------------------------------- /docs/subcommands.rst: -------------------------------------------------------------------------------- 1 | Subcommands 2 | =========== 3 | 4 | Use ``galaxyctl --help`` for help. Subcommands also support ``--help``, e.g. ``galaxy register --help`` 5 | 6 | start 7 | ----- 8 | 9 | Start and run Galaxy and associated processes in daemonized (background) mode, or ``-f`` to run in the foreground and 10 | follow log files. The ``galaxy`` command is a shortcut for ``galaxyctl start -f``. 11 | 12 | stop 13 | ---- 14 | 15 | Stop daemonized Galaxy server processes. If using supervisor mode and no processes remain running after this step (which 16 | should be the case when working with a single Galaxy instance), ``supervisord`` will terminate. 17 | 18 | restart 19 | ------- 20 | 21 | Restart Galaxy server processes. This is done in a relatively "brutal" fashion: processes are signaled (by the process 22 | manager) to exit, and then are restarted. See the ``graceful`` subcommand to restart gracefully. 23 | 24 | graceful 25 | -------- 26 | 27 | Restart Galaxy with minimal interruption. 28 | 29 | If running with a single `gunicorn`_ without ``preload``, this means holding the web socket open while restarting 30 | (connections to Galaxy will block). With ``preload``, gunicorn is restarted and some clients may experience connection 31 | failures. 32 | 33 | If running with multiple gunicorns, a rolling restart is performed, where Gravity restarts each gunicorn, waits for it 34 | to respond to requests after restarting, and then moves to the next one. This process should be transparent to clients. 35 | See :ref:`Zero-Downtime Restarts` for configuration details. 36 | 37 | If running with `unicornherder`_, a new Galaxy application will be started and the old one shut down only once the new 38 | one is accepting connections. This should also be transparent to clients, but limitations in the unicornherder software 39 | may allow interruptions to occur. 40 | 41 | update 42 | ------ 43 | 44 | Figure out what has changed in the Galaxy/Gravity config(s), which could be: 45 | 46 | - changes to the Gravity configuration options in ``galaxy.yml`` 47 | - adding or removing handlers in ``job_conf.yml`` or ``job_conf.xml`` 48 | 49 | This may cause service restarts if there are any changes. 50 | 51 | Any needed changes to supervisor or systemd configs will be performed and then ``supervisorctl update`` or ``systemctl 52 | daemon-reload`` will be called. 53 | 54 | If you wish to *remove* any existing process manager configurations for Galaxy servers managed by Gravity, the 55 | ``--clean`` flag to ``update`` can be used for this purpose. 56 | 57 | shutdown 58 | -------- 59 | 60 | Stop all processes and cause ``supervisord`` to terminate. Similar to ``stop`` but there is no ambiguity as to whether 61 | ``supervisord`` remains running. The equivalent of ``stop`` when using systemd. 62 | 63 | follow 64 | ------ 65 | 66 | Follow (e.g. using ``tail -f`` (supervisor) or ``journalctl -f`` (systemd)) log files of all Galaxy services, or a 67 | subset (if named as arguments). 68 | 69 | list 70 | ---- 71 | 72 | List config files known to Gravity. 73 | 74 | show 75 | ---- 76 | 77 | Show Gravity configuration details for a Galaxy instance. 78 | 79 | pm 80 | -- 81 | 82 | Pass through directly to the process manager (e.g. supervisor). Run ``galaxyctl pm`` to invoke the supervisorctl shell, 83 | or ``galaxyctl pm [command]`` to call a supervisorctl or systemctl command directly. See the `supervisor`_ documentation 84 | or ``galaxyctl pm help`` for help. 85 | 86 | exec 87 | ---- 88 | 89 | Directly execute a single Galaxy service in the foreground, e.g. ``galaxyctl exec gunicorn``, ``galaxyctl exec tusd``, 90 | etc. 91 | 92 | When Gravity writes out configs for the underlying process manager, it must provide a *command* (program and arguments) 93 | to execute and some number of *environment variables* that must be set for each individual Galaxy service (gunicorn, 94 | celery, etc.) to execute. By default, rather than write this information directly to the process manager configuration, 95 | Gravity sets the command to ``galaxyctl exec --config-file= ``. The ``exec`` 96 | subcommand instructs Gravity to use the `exec(3)`_ system call to execute the actual service command with its proper 97 | arguments and environment. 98 | 99 | This is done so that it is is not necesary to rewrite the process manager configs and update the process manager every 100 | time a parameter is changed, only when services are added or removed entirely. Gravity can instead be instructed to 101 | write the actual service command and environment variables directly to the process manager configurations by setting 102 | ``service_command_style`` to ``direct``. 103 | 104 | Thus, although ``exec`` is mostly an internal subcommand, developers and admins may find it useful when debugging in 105 | order to quickly and easily start just a single service and view only that service's logs in the foreground. 106 | 107 | .. _gunicorn: https://gunicorn.org/ 108 | .. _unicornherder: https://github.com/alphagov/unicornherder 109 | .. _supervisor: http://supervisord.org/ 110 | .. _exec(3): https://pubs.opengroup.org/onlinepubs/9699919799/functions/exec.html 111 | -------------------------------------------------------------------------------- /gravity/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | __version__ = "1.0.7" 4 | -------------------------------------------------------------------------------- /gravity/cli.py: -------------------------------------------------------------------------------- 1 | """ Command line utilities for managing Galaxy servers 2 | """ 3 | 4 | import os 5 | 6 | import click 7 | 8 | from gravity import io 9 | from gravity import options 10 | from gravity.settings import ProcessManager 11 | 12 | 13 | CONTEXT_SETTINGS = { 14 | "auto_envvar_prefix": "GRAVITY", 15 | "help_option_names": ["-h", "--help"] 16 | } 17 | 18 | COMMAND_ALIASES = { 19 | "configs": "list", 20 | "get": "show", 21 | "reload": "graceful", 22 | "supervisorctl": "pm", 23 | } 24 | 25 | 26 | cmd_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "commands")) 27 | 28 | 29 | def set_debug(debug_opt): 30 | if debug_opt: 31 | io.DEBUG = True 32 | 33 | 34 | def list_cmds(): 35 | rv = [] 36 | for filename in os.listdir(cmd_folder): 37 | if filename.endswith(".py") and filename.startswith("cmd_"): 38 | rv.append(filename[len("cmd_"): -len(".py")]) 39 | rv.sort() 40 | return rv 41 | 42 | 43 | def name_to_command(name): 44 | try: 45 | mod_name = "gravity.commands.cmd_" + name 46 | mod = __import__(mod_name, None, None, ["cli"]) 47 | except ImportError as e: 48 | io.error(f"Problem loading command {name}, exception {e}") 49 | return 50 | return mod.cli 51 | 52 | 53 | class GravityCLI(click.MultiCommand): 54 | def list_commands(self, ctx): 55 | return list_cmds() 56 | 57 | def get_command(self, ctx, name): 58 | if name in COMMAND_ALIASES: 59 | name = COMMAND_ALIASES[name] 60 | return name_to_command(name) 61 | 62 | 63 | # Shortcut for running Galaxy in the foreground 64 | @click.command(context_settings=CONTEXT_SETTINGS) 65 | @click.version_option() 66 | @options.debug_option() 67 | @options.config_file_option() 68 | @options.state_dir_option() 69 | @options.no_log_option() 70 | @options.single_user_option() 71 | @click.pass_context 72 | def galaxy(ctx, debug, config_file, state_dir, quiet, single_user): 73 | """Run Galaxy server in the foreground""" 74 | set_debug(debug) 75 | ctx.cm_kwargs = { 76 | "config_file": config_file, 77 | "state_dir": state_dir, 78 | "process_manager": ProcessManager.multiprocessing.value, 79 | } 80 | if single_user: 81 | os.environ["GALAXY_CONFIG_SINGLE_USER"] = single_user 82 | os.environ["GALAXY_CONFIG_ADMIN_USERS"] = single_user 83 | mod = __import__("gravity.commands.cmd_start", None, None, ["cli"]) 84 | return ctx.invoke(mod.cli, foreground=True, quiet=quiet) 85 | 86 | 87 | @click.command(cls=GravityCLI, context_settings=CONTEXT_SETTINGS) 88 | @click.version_option() 89 | @options.debug_option() 90 | @options.config_file_option() 91 | @options.state_dir_option() 92 | @options.user_mode_option() 93 | @click.pass_context 94 | def galaxyctl(ctx, debug, config_file, state_dir, user): 95 | """Manage Galaxy server configurations and processes.""" 96 | set_debug(debug) 97 | ctx.cm_kwargs = { 98 | "config_file": config_file, 99 | "state_dir": state_dir, 100 | "user_mode": user, 101 | } 102 | -------------------------------------------------------------------------------- /gravity/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/galaxyproject/gravity/20038d06c677ead44e555a0bc1faeb3f475d85b3/gravity/commands/__init__.py -------------------------------------------------------------------------------- /gravity/commands/cmd_exec.py: -------------------------------------------------------------------------------- 1 | import click 2 | 3 | from gravity import options 4 | from gravity import process_manager 5 | 6 | 7 | @click.command("exec") 8 | @click.option("--no-exec", "-n", is_flag=True, default=False, help="Don't exec, just print the command that would be run") 9 | @click.option("--service-instance", "-i", type=int, help="For multi-instance services, which instance to exec") 10 | @options.instances_services_arg() 11 | @click.pass_context 12 | def cli(ctx, instances_services, no_exec, service_instance): 13 | """Run a single Galaxy service in the foreground, with logging output to stdout/stderr. 14 | 15 | Zero or one instance names can be provided in INSTANCES, it is required if more than one Galaxy instance is 16 | configured in Gravity. 17 | 18 | Exactly one service name is required in SERVICES. 19 | """ 20 | with process_manager.process_manager(**ctx.parent.cm_kwargs) as pm: 21 | pm.exec(instance_names=instances_services, service_instance_number=service_instance, no_exec=no_exec) 22 | -------------------------------------------------------------------------------- /gravity/commands/cmd_follow.py: -------------------------------------------------------------------------------- 1 | import click 2 | 3 | from gravity import options 4 | from gravity import process_manager 5 | 6 | 7 | @click.command("follow") 8 | @options.instances_services_arg() 9 | @click.pass_context 10 | def cli(ctx, instances_services): 11 | """Follow log files of configured instances. 12 | 13 | If no INSTANCES or SERVICES are provided, logs of all configured services of all configured instances are followed. 14 | 15 | Specifying INSTANCES and SERVICES limits the operation to only the provided instance name(s) and/or service(s). 16 | """ 17 | with process_manager.process_manager(**ctx.parent.cm_kwargs) as pm: 18 | pm.follow(instance_names=instances_services) 19 | -------------------------------------------------------------------------------- /gravity/commands/cmd_graceful.py: -------------------------------------------------------------------------------- 1 | import click 2 | 3 | from gravity import options 4 | from gravity import process_manager 5 | 6 | 7 | @click.command("graceful") 8 | @options.instances_services_arg() 9 | @click.pass_context 10 | def cli(ctx, instances_services): 11 | """Gracefully reload configured services. 12 | 13 | If no INSTANCES or SERVICES are provided, all configured services of all configured instances are gracefully 14 | reloaded. 15 | 16 | Specifying INSTANCES and SERVICES limits the operation to only the provided instance name(s) and/or service(s). 17 | """ 18 | with process_manager.process_manager(**ctx.parent.cm_kwargs) as pm: 19 | pm.graceful(instance_names=instances_services) 20 | -------------------------------------------------------------------------------- /gravity/commands/cmd_list.py: -------------------------------------------------------------------------------- 1 | import click 2 | 3 | from gravity import config_manager 4 | 5 | 6 | @click.command("list") 7 | @click.option("--version", "-v", is_flag=True, default=False, help="Include Galaxy version in output") 8 | @click.pass_context 9 | def cli(ctx, version): 10 | """List configured instances. 11 | 12 | aliases: configs 13 | """ 14 | cols = ["{:<18}", "{}"] 15 | head = ["INSTANCE NAME", "CONFIG PATH"] 16 | if version: 17 | cols.insert(1, "{:<12}") 18 | head.insert(1, "VERSION") 19 | cols_str = " ".join(cols) 20 | with config_manager.config_manager(**ctx.parent.cm_kwargs) as cm: 21 | configs = cm.get_configs() 22 | if configs: 23 | click.echo(cols_str.format(*head)) 24 | for config in configs: 25 | row = [ 26 | config.instance_name, 27 | config.gravity_config_file, 28 | ] 29 | if version: 30 | row.insert(1, config.galaxy_version) 31 | click.echo(cols_str.format(*row)) 32 | else: 33 | click.echo("No configured instances") 34 | -------------------------------------------------------------------------------- /gravity/commands/cmd_pm.py: -------------------------------------------------------------------------------- 1 | import click 2 | 3 | from gravity import process_manager 4 | 5 | 6 | @click.command("pm") 7 | @click.argument("pm_args", nargs=-1) 8 | @click.pass_context 9 | def cli(ctx, pm_args): 10 | """Invoke process manager (supervisorctl, systemctl) directly. 11 | 12 | Any args in PM_ARGS are passed to the process manager command. 13 | """ 14 | with process_manager.process_manager(**ctx.parent.cm_kwargs) as pm: 15 | pm.pm(*pm_args) 16 | -------------------------------------------------------------------------------- /gravity/commands/cmd_restart.py: -------------------------------------------------------------------------------- 1 | import click 2 | 3 | from gravity import options 4 | from gravity import process_manager 5 | 6 | 7 | @click.command("restart") 8 | @options.instances_services_arg() 9 | @click.pass_context 10 | def cli(ctx, instances_services): 11 | """Restart configured services. 12 | 13 | If no INSTANCES or SERVICES are provided, all configured services of all configured instances are restarted. 14 | 15 | Specifying INSTANCES and SERVICES limits the operation to only the provided instance name(s) and/or service(s). 16 | """ 17 | with process_manager.process_manager(**ctx.parent.cm_kwargs) as pm: 18 | pm.restart(instance_names=instances_services) 19 | -------------------------------------------------------------------------------- /gravity/commands/cmd_show.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import click 4 | 5 | from gravity import config_manager 6 | 7 | 8 | @click.command("show") 9 | @click.argument("instance", required=False) 10 | @click.pass_context 11 | def cli(ctx, instance): 12 | """Show details of instance config. 13 | 14 | INSTANCE is optional unless there is more than one Galaxy instance configured. 15 | 16 | aliases: get 17 | """ 18 | with config_manager.config_manager(**ctx.parent.cm_kwargs) as cm: 19 | config_data = cm.get_config(instance_name=instance) 20 | click.echo(json.dumps(config_data.dict(), indent=4)) 21 | -------------------------------------------------------------------------------- /gravity/commands/cmd_shutdown.py: -------------------------------------------------------------------------------- 1 | import click 2 | 3 | from gravity import process_manager 4 | 5 | 6 | @click.command("status") 7 | @click.pass_context 8 | def cli(ctx): 9 | """Shut down process manager.""" 10 | with process_manager.process_manager(**ctx.parent.cm_kwargs) as pm: 11 | pm.shutdown() 12 | -------------------------------------------------------------------------------- /gravity/commands/cmd_start.py: -------------------------------------------------------------------------------- 1 | import click 2 | 3 | from gravity import options 4 | from gravity import process_manager 5 | from gravity.io import info 6 | 7 | 8 | @click.command("start") 9 | @options.instances_services_arg() 10 | @click.option("-f", "--foreground", is_flag=True, default=False, help="Run in foreground") 11 | @options.no_log_option() 12 | @click.pass_context 13 | def cli(ctx, instances_services, foreground, quiet=False): 14 | """Start configured services. 15 | 16 | If no INSTANCES or SERVICES are provided, all configured services of all configured instances are started. 17 | 18 | Specifying INSTANCES and SERVICES limits the operation to only the provided instance name(s) and/or service(s). 19 | """ 20 | with process_manager.process_manager(foreground=foreground, **ctx.parent.cm_kwargs) as pm: 21 | pm.update() 22 | pm.start(instance_names=instances_services) 23 | if foreground: 24 | pm.follow(instance_names=instances_services, quiet=quiet) 25 | elif pm.config_manager.single_instance: 26 | config = pm.config_manager.get_config() 27 | if config.process_manager != "systemd": 28 | info(f"Log files are in {config.log_dir}") 29 | else: 30 | for config in pm.config_manager.get_configs(instances=instances_services or None): 31 | if config.process_manager != "systemd": 32 | info(f"Log files for {config.instance_name} are in {config.log_dir}") 33 | -------------------------------------------------------------------------------- /gravity/commands/cmd_status.py: -------------------------------------------------------------------------------- 1 | import click 2 | 3 | from gravity import options 4 | from gravity import process_manager 5 | 6 | 7 | @click.command("status") 8 | @options.instances_services_arg() 9 | @click.pass_context 10 | def cli(ctx, instances_services): 11 | """Display server status. 12 | 13 | If no INSTANCES or SERVICES are provided, the status of all configured services of all configured instances is 14 | displayed. 15 | 16 | Specifying INSTANCES and SERVICES limits the operation to only the provided instance name(s) and/or service(s). 17 | """ 18 | with process_manager.process_manager(**ctx.parent.cm_kwargs) as pm: 19 | pm.status(instance_names=instances_services) 20 | -------------------------------------------------------------------------------- /gravity/commands/cmd_stop.py: -------------------------------------------------------------------------------- 1 | import click 2 | 3 | from gravity import options 4 | from gravity import process_manager 5 | 6 | 7 | @click.command("stop") 8 | @options.instances_services_arg() 9 | @click.pass_context 10 | def cli(ctx, instances_services): 11 | """Stop configured services. 12 | 13 | If no INSTANCES or SERVICES are provided, all configured services of all configured instances are stopped. 14 | 15 | Specifying INSTANCES and SERVICES limits the operation to only the provided instance name(s) and/or service(s). 16 | """ 17 | with process_manager.process_manager(**ctx.parent.cm_kwargs) as pm: 18 | pm.stop(instance_names=instances_services) 19 | -------------------------------------------------------------------------------- /gravity/commands/cmd_update.py: -------------------------------------------------------------------------------- 1 | import click 2 | 3 | from gravity import process_manager 4 | 5 | 6 | @click.command("update") 7 | @click.option("--force", is_flag=True, help="Force rewriting of process config files") 8 | @click.option("--clean", is_flag=True, help="Remove process config files if they exist") 9 | @click.pass_context 10 | def cli(ctx, force, clean): 11 | """Update process manager from config changes.""" 12 | with process_manager.process_manager(**ctx.parent.cm_kwargs) as pm: 13 | pm.update(force=force, clean=clean) 14 | -------------------------------------------------------------------------------- /gravity/config_manager.py: -------------------------------------------------------------------------------- 1 | """ Galaxy Process Management superclass and utilities 2 | """ 3 | import contextlib 4 | import glob 5 | import logging 6 | import os 7 | import xml.etree.ElementTree as elementtree 8 | from typing import Union 9 | 10 | try: 11 | from pydantic.v1 import ValidationError 12 | except ImportError: 13 | from pydantic import ValidationError 14 | from yaml import safe_load 15 | 16 | import gravity.io 17 | from gravity.settings import ( 18 | ProcessManager, 19 | Settings, 20 | ) 21 | from gravity.state import ( 22 | ConfigFile, 23 | service_for_service_type, 24 | galaxy_installed, 25 | ) 26 | from gravity.util import recursive_update 27 | 28 | log = logging.getLogger(__name__) 29 | 30 | # Falling back to job_conf.xml when job_config_file is unset and job_conf.yml doesn't exist is deprecated in Galaxy, and 31 | # support for it can be removed from Gravity when it is removed from Galaxy 32 | DEFAULT_JOB_CONFIG_FILES = ("job_conf.yml", "job_conf.xml") 33 | if "XDG_CONFIG_HOME" in os.environ: 34 | DEFAULT_STATE_DIR = os.path.join(os.environ["XDG_CONFIG_HOME"], "galaxy-gravity") 35 | 36 | OPTIONAL_APP_KEYS = ( 37 | "interactivetools_map", 38 | "interactivetools_base_path", 39 | "interactivetools_prefix", 40 | "galaxy_url_prefix", 41 | ) 42 | 43 | 44 | @contextlib.contextmanager 45 | def config_manager(config_file=None, state_dir=None, user_mode=None, process_manager=None): 46 | yield ConfigManager( 47 | config_file=config_file, 48 | state_dir=state_dir, 49 | user_mode=user_mode, 50 | process_manager=process_manager 51 | ) 52 | 53 | 54 | class ConfigManager(object): 55 | galaxy_server_config_section = "galaxy" 56 | gravity_config_section = "gravity" 57 | app_config_file_option = "galaxy_config_file" 58 | 59 | def __init__(self, config_file=None, state_dir=None, user_mode=None, process_manager=None): 60 | self.__configs = {} 61 | self.state_dir = None 62 | if state_dir is not None: 63 | # convert from pathlib.Path 64 | self.state_dir = str(state_dir) 65 | self.user_mode = user_mode 66 | self.process_manager = process_manager or ProcessManager.supervisor.value 67 | 68 | gravity.io.debug(f"Gravity state dir: {state_dir}") 69 | 70 | if config_file: 71 | for cf in config_file: 72 | self.load_config_file(cf) 73 | else: 74 | self.auto_load() 75 | 76 | @property 77 | def is_root(self): 78 | return os.geteuid() == 0 79 | 80 | def load_config_file(self, config_file): 81 | with open(config_file) as config_fh: 82 | try: 83 | config_dict = safe_load(config_fh) 84 | except Exception as exc: 85 | # this should always be a parse error, access errors will be caught by click 86 | gravity.io.error(f"Failed to parse config: {config_file}") 87 | gravity.io.exception(exc) 88 | 89 | if type(config_dict) is not dict: 90 | gravity.io.exception(f"Config file does not look like valid Galaxy or Gravity configuration file: {config_file}") 91 | 92 | gravity_config_dict = config_dict.get(self.gravity_config_section) or {} 93 | 94 | if type(gravity_config_dict) is list: 95 | self.__load_config_list(config_file, config_dict) 96 | return 97 | 98 | app_config = None 99 | server_section = self.galaxy_server_config_section 100 | if self.gravity_config_section in config_dict and server_section not in config_dict: 101 | app_config_file = config_dict[self.gravity_config_section].get(self.app_config_file_option) 102 | if app_config_file: 103 | app_config = self.__load_app_config_file(config_file, app_config_file) 104 | else: 105 | gravity.io.warn( 106 | f"Config file appears to be a Gravity config but contains no {server_section} section, " 107 | f"Galaxy defaults will be used: {config_file}") 108 | elif self.gravity_config_section not in config_dict and server_section in config_dict: 109 | gravity.io.warn( 110 | f"Config file appears to be a Galaxy config but contains no {self.gravity_config_section} section, " 111 | f"Gravity defaults will be used: {config_file}") 112 | elif self.gravity_config_section not in config_dict and server_section not in config_dict: 113 | gravity.io.exception(f"Config file does not look like valid Galaxy or Gravity configuration file: {config_file}") 114 | 115 | app_config = app_config or config_dict.get(server_section) or {} 116 | gravity_config_dict["__file__"] = config_file 117 | self.__load_config(gravity_config_dict, app_config) 118 | 119 | def __load_app_config_file(self, gravity_config_file, app_config_file): 120 | server_section = self.galaxy_server_config_section 121 | if not os.path.isabs(app_config_file): 122 | app_config_file = os.path.join(os.path.dirname(gravity_config_file), app_config_file) 123 | try: 124 | with open(app_config_file) as config_fh: 125 | _app_config_dict = safe_load(config_fh) 126 | if server_section not in _app_config_dict: 127 | # we let a missing galaxy config slide in other scenarios but if you set the option to something 128 | # that doesn't contain a galaxy section that's almost surely a mistake 129 | gravity.io.exception(f"Galaxy config file does not contain a {server_section} section: {app_config_file}") 130 | app_config = _app_config_dict[server_section] or {} 131 | app_config["__file__"] = app_config_file 132 | return app_config 133 | except Exception as exc: 134 | gravity.io.exception(exc) 135 | 136 | def __load_config_list(self, config_file, config_dict): 137 | try: 138 | assert self.galaxy_server_config_section not in config_dict, ( 139 | "Multiple Gravity configurations in a shared Galaxy configuration file is ambiguous, set " 140 | f"`{self.app_config_file_option}` and remove the Galaxy configuration: {config_file}" 141 | ) 142 | for gravity_config_dict in config_dict[self.gravity_config_section]: 143 | assert "galaxy_config_file" in gravity_config_dict, ( 144 | "The `{self.app_config_file_option}` option must be set when multiple Gravity configurations are " 145 | f"present: {config_file}" 146 | ) 147 | app_config = self.__load_app_config_file(config_file, gravity_config_dict[self.app_config_file_option]) 148 | gravity_config_dict["__file__"] = config_file 149 | self.__load_config(gravity_config_dict, app_config) 150 | except AssertionError as exc: 151 | gravity.io.exception(exc) 152 | 153 | def __load_config(self, gravity_config_dict, app_config): 154 | defaults = {} 155 | try: 156 | gravity_settings = Settings(**recursive_update(defaults, gravity_config_dict)) 157 | except ValidationError as exc: 158 | # suppress the traceback and just report the error 159 | gravity.io.exception(exc) 160 | 161 | if gravity_settings.instance_name in self.__configs: 162 | gravity.io.error( 163 | f"Galaxy instance {gravity_settings.instance_name} already loaded from file: " 164 | f"{self.__configs[gravity_settings.instance_name].gravity_config_file}") 165 | gravity.io.exception(f"Duplicate instance name {gravity_settings.instance_name}, instance names must be unique") 166 | 167 | gravity_config_file = gravity_config_dict.get("__file__") 168 | galaxy_config_file = app_config.get("__file__", gravity_config_file) 169 | galaxy_root = gravity_settings.galaxy_root or app_config.get("root") 170 | 171 | # TODO: document that the default state_dir is data_dir/gravity and that setting state_dir overrides this 172 | default_data_dir = "data" if galaxy_installed else "database" 173 | gravity_data_dir = self.state_dir or os.path.join(app_config.get("data_dir", default_data_dir), "gravity") 174 | log_dir = gravity_settings.log_dir or os.path.join(gravity_data_dir, "log") 175 | 176 | # TODO: this should use galaxy.util.properties.load_app_properties() so that env vars work 177 | app_config_dict = { 178 | "galaxy_infrastructure_url": app_config.get("galaxy_infrastructure_url", "").rstrip("/"), 179 | "interactivetools_enable": app_config.get("interactivetools_enable"), 180 | } 181 | 182 | # some things should only be included if set 183 | for app_key in OPTIONAL_APP_KEYS: 184 | if app_key in app_config: 185 | app_config_dict[app_key] = app_config[app_key] 186 | 187 | config = ConfigFile( 188 | app_config=app_config_dict, 189 | gravity_config_file=gravity_config_file, 190 | galaxy_config_file=galaxy_config_file, 191 | instance_name=gravity_settings.instance_name, 192 | process_manager=gravity_settings.process_manager or self.process_manager, 193 | service_command_style=gravity_settings.service_command_style, 194 | app_server=gravity_settings.app_server, 195 | virtualenv=gravity_settings.virtualenv, 196 | galaxy_root=galaxy_root, 197 | galaxy_user=gravity_settings.galaxy_user, 198 | galaxy_group=gravity_settings.galaxy_group, 199 | umask=gravity_settings.umask, 200 | memory_limit=gravity_settings.memory_limit, 201 | gravity_data_dir=gravity_data_dir, 202 | log_dir=log_dir, 203 | ) 204 | 205 | # add standard services if enabled 206 | for service_type in (config.app_server, "celery", "celery-beat", "tusd", "gx-it-proxy", "reports"): 207 | config.services.extend(service_for_service_type(service_type).services_if_enabled(config, gravity_settings)) 208 | 209 | # load any static handlers defined in the galaxy job config 210 | assign_with = self.create_static_handler_services(config, app_config) 211 | 212 | # load any dynamic handlers defined in the gravity config 213 | self.create_dynamic_handler_services(gravity_settings, config, assign_with) 214 | 215 | for service in config.services: 216 | gravity.io.debug(f"Configured {service.service_type} type service: {service.service_name}") 217 | gravity.io.debug(f"Loaded instance {config.instance_name} from Gravity config file: {config.gravity_config_file}") 218 | 219 | self.__configs[config.instance_name] = config 220 | return config 221 | 222 | def create_static_handler_services(self, config: ConfigFile, app_config: dict): 223 | assign_with = None 224 | if not app_config.get("job_config_file") and app_config.get("job_config"): 225 | # config embedded directly in Galaxy config 226 | job_config = app_config["job_config"] 227 | else: 228 | # config in an external file 229 | config_dir = os.path.dirname(config.galaxy_config_file or os.getcwd()) 230 | job_config = app_config.get("job_config_file") 231 | if not job_config: 232 | for job_config in [os.path.abspath(os.path.join(config_dir, c)) for c in DEFAULT_JOB_CONFIG_FILES]: 233 | if os.path.exists(job_config): 234 | break 235 | else: 236 | job_config = None 237 | elif not os.path.isabs(job_config): 238 | job_config = os.path.abspath(os.path.join(config_dir, job_config)) 239 | if not os.path.exists(job_config): 240 | job_config = None 241 | if job_config: 242 | # parse job conf for any *static* standalone handlers 243 | assign_with, handler_settings_list = ConfigManager.get_job_config(job_config) 244 | for handler_settings in handler_settings_list: 245 | config.services.append(service_for_service_type("standalone")( 246 | config=config, 247 | service_name=handler_settings.pop("service_name"), 248 | settings=handler_settings, 249 | )) 250 | return assign_with 251 | 252 | def create_dynamic_handler_services(self, gravity_settings: Settings, config: ConfigFile, assign_with): 253 | # we push environment from settings to services but the rest of the services pull their env options from 254 | # settings directly. this can be a bit confusing but is probably ok since there are 3 ways to configure 255 | # handlers, and gravity is only 1 of them. 256 | assign_with = assign_with or [] 257 | expanded_handlers = self.expand_handlers(gravity_settings, config) 258 | if expanded_handlers and "db-skip-locked" not in assign_with and "db-transaction-isolation" not in assign_with: 259 | gravity.io.warn( 260 | "Dynamic handlers are configured in Gravity but Galaxy is not configured to assign jobs to handlers " 261 | "dynamically, so these handlers will not handle jobs. Set the job handler assignment method in the " 262 | "Galaxy job configuration to `db-skip-locked` or `db-transaction-isolation` to fix this.") 263 | for service_name, handler_settings in expanded_handlers.items(): 264 | config.services.extend( 265 | service_for_service_type("standalone").services_if_enabled( 266 | config, 267 | gravity_settings=gravity_settings, 268 | settings=handler_settings, 269 | service_name=service_name, 270 | )) 271 | 272 | @staticmethod 273 | def expand_handlers(gravity_settings: Settings, config: ConfigFile): 274 | use_list = gravity_settings.use_service_instances 275 | handlers = gravity_settings.handlers or {} 276 | expanded_handlers = {} 277 | default_name_template = "{name}_{process}" 278 | for service_name, handler_config in handlers.items(): 279 | handler_config["enable"] = True 280 | count = handler_config.get("processes", 1) 281 | if "pools" in handler_config: 282 | handler_config["server_pools"] = handler_config.pop("pools") 283 | name_template = handler_config.get("name_template") 284 | if name_template is None: 285 | if count == 1 and service_name[-1].isdigit(): 286 | # Assume we have an explicit handler name, don't apply pattern 287 | expanded_handlers[service_name] = handler_config 288 | continue 289 | name_template = (name_template or default_name_template).strip() 290 | instances = [] 291 | for index in range(count): 292 | expanded_service_name = name_template.format(name=service_name, process=index, instance_name=config.instance_name) 293 | if use_list: 294 | instance = handler_config.copy() 295 | instance["server_name"] = expanded_service_name 296 | instances.append(instance) 297 | elif expanded_service_name not in expanded_handlers: 298 | expanded_handlers[expanded_service_name] = handler_config 299 | else: 300 | gravity.io.warn(f"Duplicate handler name after expansion: {expanded_service_name}") 301 | if use_list: 302 | expanded_handlers[service_name] = instances 303 | return expanded_handlers 304 | 305 | @staticmethod 306 | def get_job_config(conf: Union[str, dict]): 307 | """Extract handler names from job_conf.xml""" 308 | # TODO: use galaxy job conf parsing 309 | assign_with = None 310 | rval = [] 311 | if isinstance(conf, str): 312 | if conf.endswith('.xml'): 313 | root = elementtree.parse(conf).getroot() 314 | handlers = root.find("handlers") 315 | assign_with = (handlers or {}).get("assign_with") 316 | if assign_with: 317 | assign_with = [a.strip() for a in assign_with.split(",")] 318 | for handler in (handlers or []): 319 | rval.append({"service_name": handler.attrib["id"]}) 320 | elif conf.endswith(('.yml', '.yaml')): 321 | with open(conf) as job_conf_fh: 322 | conf = safe_load(job_conf_fh.read()) 323 | else: 324 | gravity.io.exception(f"Unknown job config file type: {conf}") 325 | if isinstance(conf, dict): 326 | handling = conf.get('handling') or {} 327 | assign_with = handling.get('assign', []) 328 | processes = handling.get('processes') or {} 329 | for handler_name, handler_options in processes.items(): 330 | rval.append({ 331 | "service_name": handler_name, 332 | "environment": (handler_options or {}).get("environment", None) 333 | }) 334 | return (assign_with, rval) 335 | 336 | @property 337 | def instance_count(self): 338 | """The number of configured instances""" 339 | return len(self.__configs) 340 | 341 | @property 342 | def single_instance(self): 343 | """Indicate if there is only one configured instance""" 344 | return self.instance_count == 1 345 | 346 | def is_loaded(self, config_file): 347 | return config_file in self.get_configured_files() 348 | 349 | def get_configs(self, instances=None, process_manager=None): 350 | """Return the persisted values of all config files registered with the config manager.""" 351 | rval = [] 352 | for instance_name, config in list(self.__configs.items()): 353 | if ((instances is not None and instance_name in instances) or instances is None) and ( 354 | (process_manager is not None and config.process_manager == process_manager) or process_manager is None 355 | ): 356 | rval.append(config) 357 | return rval 358 | 359 | def get_config(self, instance_name=None): 360 | if instance_name is None: 361 | if self.instance_count > 1: 362 | gravity.io.exception("An instance name is required when more than one instance is configured") 363 | elif self.instance_count == 0: 364 | gravity.io.exception("No configured Galaxy instances") 365 | instance_name = list(self.__configs.keys())[0] 366 | try: 367 | return self.__configs[instance_name] 368 | except KeyError: 369 | gravity.io.exception(f"Unknown instance name: {instance_name}") 370 | 371 | def get_configured_service_names(self): 372 | rval = set() 373 | for config in self.get_configs(): 374 | for service in config.services: 375 | rval.add(service.service_name) 376 | return rval 377 | 378 | def get_configured_instance_names(self): 379 | return list(self.__configs.keys()) 380 | 381 | def get_configured_files(self): 382 | return list(c.gravity_config_file for c in self.__configs.values()) 383 | 384 | def auto_load(self): 385 | """Attempt to automatically load a config file if none are loaded.""" 386 | load_all = False 387 | if self.instance_count != 0: 388 | return 389 | if os.environ.get("GALAXY_CONFIG_FILE"): 390 | configs = [os.environ["GALAXY_CONFIG_FILE"]] 391 | elif self.is_root: 392 | load_all = True 393 | configs = ( 394 | "/etc/galaxy/gravity.yml", 395 | "/etc/galaxy/galaxy.yml", 396 | *glob.glob("/etc/galaxy/gravity.d/*.yml"), 397 | *glob.glob("/etc/galaxy/gravity.d/*.yaml"), 398 | ) 399 | else: 400 | configs = (os.path.join("config", "galaxy.yml"), "galaxy.yml", os.path.join("config", "galaxy.yml.sample")) 401 | configs = tuple(config for config in configs if os.path.exists(config)) 402 | if not configs and galaxy_installed: 403 | gravity.io.warn( 404 | "Warning: No configuration file found but Galaxy is installed in this Python environment, running with " 405 | "default config. Use -c / --config-file or set $GALAXY_CONFIG_FILE to specify a config file." 406 | ) 407 | self.__load_config({}, {}) 408 | for config in configs: 409 | self.load_config_file(os.path.abspath(config)) 410 | if not load_all: 411 | return 412 | -------------------------------------------------------------------------------- /gravity/io.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import traceback 3 | 4 | import click 5 | 6 | 7 | DEBUG = False 8 | 9 | 10 | def debug(message, *args): 11 | if args: 12 | message = message % args 13 | if DEBUG: 14 | click.echo(message) 15 | 16 | 17 | def info(message, *args, bright=True): 18 | if args: 19 | message = message % args 20 | style_kwargs = {} 21 | if bright: 22 | style_kwargs = {"bold": True, "fg": "green"} 23 | click.echo(click.style(message, **style_kwargs)) 24 | 25 | 26 | def error(message, *args): 27 | if args: 28 | message = message % args 29 | if DEBUG and sys.exc_info()[0] is not None: 30 | click.echo(traceback.format_exc(), nl=False) 31 | click.echo(click.style(message, bold=True, fg="red"), err=True) 32 | 33 | 34 | def warn(message, *args): 35 | if args: 36 | message = message % args 37 | click.echo(click.style(message, fg="yellow"), err=True) 38 | 39 | 40 | def exception(message): 41 | raise click.ClickException(click.style(message, bold=True, fg="red")) 42 | -------------------------------------------------------------------------------- /gravity/options.py: -------------------------------------------------------------------------------- 1 | """ Click definitions for various shared options and arguments. 2 | """ 3 | import click 4 | 5 | 6 | def debug_option(): 7 | return click.option("-d", "--debug", is_flag=True, help="Enables debug mode.") 8 | 9 | 10 | def state_dir_option(): 11 | return click.option( 12 | "--state-dir", type=click.Path(file_okay=False, writable=True, resolve_path=True), help="Where process management configs and state will be stored." 13 | ) 14 | 15 | 16 | def config_file_option(): 17 | return click.option( 18 | "-c", 19 | "--config-file", 20 | type=click.Path(exists=True, dir_okay=False, resolve_path=True), 21 | multiple=True, 22 | help="Gravity (or Galaxy) config file to operate on. Can also be set with $GRAVITY_CONFIG_FILE or $GALAXY_CONFIG_FILE", 23 | ) 24 | 25 | 26 | def user_mode_option(): 27 | return click.option( 28 | "--user/--no-user", 29 | default=None, 30 | help="Use `systemctl/journalctl --user` (default: automatic depending on whether run as root)", 31 | ) 32 | 33 | 34 | def single_user_option(): 35 | return click.option( 36 | "-s", 37 | "--single-user", 38 | default=None, 39 | help="Run Galaxy in single user mode with the specified account email" 40 | ) 41 | 42 | 43 | def no_log_option(): 44 | return click.option( 45 | '--quiet', is_flag=True, default=False, help="Only output supervisor logs, do not include process logs" 46 | ) 47 | 48 | 49 | def required_config_arg(name="config", exists=False, nargs=None): 50 | arg_type = click.Path( 51 | exists=exists, 52 | file_okay=True, 53 | dir_okay=False, 54 | readable=True, 55 | resolve_path=True, 56 | ) 57 | if nargs is None: 58 | return click.argument(name, type=arg_type) 59 | else: 60 | return click.argument(name, nargs=nargs, type=arg_type) 61 | 62 | 63 | def instances_services_arg(): 64 | return click.argument("instances_services", metavar="[INSTANCES] [SERVICES]", nargs=-1) 65 | -------------------------------------------------------------------------------- /gravity/process_manager/__init__.py: -------------------------------------------------------------------------------- 1 | """ Galaxy Process Management superclass and utilities 2 | """ 3 | import contextlib 4 | import errno 5 | import importlib 6 | import inspect 7 | import os 8 | import shlex 9 | import sys 10 | from abc import ABCMeta, abstractmethod 11 | from functools import partial, wraps 12 | 13 | import gravity.io 14 | from gravity.config_manager import ConfigManager 15 | from gravity.settings import DEFAULT_INSTANCE_NAME, ServiceCommandStyle 16 | from gravity.state import VALID_SERVICE_NAMES 17 | from gravity.util import which 18 | 19 | 20 | @contextlib.contextmanager 21 | def process_manager(*args, **kwargs): 22 | pm = ProcessManagerRouter(*args, **kwargs) 23 | try: 24 | yield pm 25 | finally: 26 | pm.terminate() 27 | 28 | 29 | def _route(func, all_process_managers=False): 30 | """Given instance names, populates kwargs with instance configs for the given PM, and calls the PM-routed function 31 | """ 32 | @wraps(func) 33 | def decorator(self, *args, instance_names=None, **kwargs): 34 | configs_by_pm = {} 35 | pm_names = self.process_managers.keys() 36 | instance_names, service_names = self._instance_service_names(instance_names) 37 | configs = self.config_manager.get_configs(instances=instance_names or None) 38 | if not configs: 39 | gravity.io.exception("No configured Galaxy instances") 40 | for config in configs: 41 | try: 42 | configs_by_pm[config.process_manager].append(config) 43 | except KeyError: 44 | configs_by_pm[config.process_manager] = [config] 45 | if not all_process_managers: 46 | pm_names = configs_by_pm.keys() 47 | for pm_name in pm_names: 48 | routed_func = getattr(self.process_managers[pm_name], func.__name__) 49 | routed_func_params = list(inspect.signature(routed_func).parameters) 50 | if "configs" in routed_func_params: 51 | pm_configs = configs_by_pm.get(pm_name, []) 52 | kwargs["configs"] = pm_configs 53 | gravity.io.debug(f"Calling {func.__name__} in process manager {pm_name} for instances: {[c.instance_name for c in pm_configs]}") 54 | else: 55 | gravity.io.debug(f"Calling {func.__name__} in process manager {pm_name} for all instances") 56 | if "service_names" in routed_func_params: 57 | kwargs["service_names"] = service_names 58 | routed_func(*args, **kwargs) 59 | # note we don't ever actually call the decorated function, we call the routed one(s) 60 | return decorator 61 | 62 | 63 | route = partial(_route, all_process_managers=False) 64 | route_to_all = partial(_route, all_process_managers=True) 65 | 66 | 67 | class BaseProcessExecutionEnvironment(metaclass=ABCMeta): 68 | def __init__(self, state_dir=None, config_file=None, config_manager=None, user_mode=None, process_executor=None): 69 | self.config_manager = config_manager or ConfigManager(state_dir=state_dir, config_file=config_file, user_mode=user_mode) 70 | self.tail = which("tail") 71 | 72 | @abstractmethod 73 | def _service_environment_formatter(self, environment, format_vars): 74 | raise NotImplementedError() 75 | 76 | def _service_default_path(self): 77 | return os.environ["PATH"] 78 | 79 | def _service_program_name(self, instance_name, service): 80 | return f"{instance_name}_{service.service_type}_{service.service_name}" 81 | 82 | def _service_format_vars(self, config, service, pm_format_vars=None): 83 | pm_format_vars = pm_format_vars or {} 84 | virtualenv_dir = config.virtualenv 85 | virtualenv_bin = shlex.quote(f'{os.path.join(virtualenv_dir, "bin")}{os.path.sep}') if virtualenv_dir else "" 86 | 87 | format_vars = { 88 | "server_name": service.service_name, 89 | "galaxy_umask": service.settings.get("umask") or config.umask, 90 | "galaxy_conf": config.galaxy_config_file, 91 | # TODO: this is used as the runtime directory, but it should probably be something else 92 | "galaxy_root": config.galaxy_root or os.getcwd(), 93 | "virtualenv_bin": virtualenv_bin, 94 | "gravity_data_dir": shlex.quote(config.gravity_data_dir), 95 | "app_config": config.app_config, 96 | } 97 | 98 | format_vars["settings"] = service.settings 99 | format_vars["service_instance_count"] = service.count 100 | 101 | # update here from PM overrides 102 | format_vars.update(pm_format_vars) 103 | 104 | # template the command template 105 | if config.service_command_style in (ServiceCommandStyle.direct, ServiceCommandStyle.exec): 106 | format_vars["command_arguments"] = service.get_command_arguments(format_vars) 107 | format_vars["command"] = service.command_template.format(**format_vars) 108 | 109 | # template env vars 110 | environment = service.environment 111 | virtualenv_bin = format_vars["virtualenv_bin"] # could have been changed by pm_format_vars 112 | if virtualenv_bin and service.add_virtualenv_to_path: 113 | path = environment.get("PATH", self._service_default_path()) 114 | environment["PATH"] = ":".join([virtualenv_bin, path]) 115 | else: 116 | config_file_option = "" 117 | if config.gravity_config_file: 118 | config_file_option = f" --config-file {shlex.quote(config.gravity_config_file)}" 119 | # is there a click way to do this? 120 | galaxyctl = sys.argv[0] 121 | if galaxyctl.endswith(f"{os.path.sep}galaxy"): 122 | # handle when called using the `galaxy` entrypoint 123 | galaxyctl += "ctl" 124 | if not galaxyctl.endswith(f"{os.path.sep}galaxyctl"): 125 | gravity.io.warn(f"Unable to determine galaxyctl command, sys.argv[0] is: {galaxyctl}") 126 | galaxyctl = shlex.quote(galaxyctl) 127 | instance_number_opt = "" 128 | if service.count > 1: 129 | instance_number_opt = f" --service-instance {pm_format_vars['instance_number']}" 130 | format_vars["command"] = f"{galaxyctl}{config_file_option} exec{instance_number_opt} {config.instance_name} {service.service_name}" 131 | environment = {} 132 | format_vars["environment"] = self._service_environment_formatter(environment, format_vars) 133 | 134 | return format_vars 135 | 136 | 137 | class BaseProcessManager(BaseProcessExecutionEnvironment, metaclass=ABCMeta): 138 | def __init__(self, *args, foreground=False, **kwargs): 139 | super().__init__(*args, **kwargs) 140 | self._service_changes = None 141 | 142 | @property 143 | def _use_instance_name(self): 144 | return ((not self.config_manager.single_instance) 145 | or self.config_manager.get_config().instance_name != DEFAULT_INSTANCE_NAME) 146 | 147 | def _remove_unintended_pm_files_for_configs(self, configs): 148 | unintended_pm_files = set() 149 | for config in configs: 150 | intended_pm_files = self._intended_pm_files_for_config(config) 151 | present_pm_files = self._present_pm_files_for_config(config) 152 | unintended_pm_files.update(present_pm_files - intended_pm_files) 153 | self._disable_and_remove_pm_files(unintended_pm_files) 154 | 155 | def _remove_all_pm_files_for_configs(self, configs): 156 | for config in configs: 157 | pm_files = self._present_pm_files_for_config(config) 158 | self._disable_and_remove_pm_files(pm_files) 159 | 160 | def _remove_all_pm_files(self): 161 | # the kevin uxbridge method 162 | pm_files = self._all_present_pm_files() 163 | self._disable_and_remove_pm_files(pm_files) 164 | 165 | def _pre_update(self, configs, force, clean): 166 | all_configs = set(self.config_manager.get_configs()) 167 | if not clean: 168 | # no --clean and either possibility of --force 169 | # remove any pm files for configs known to this gravity but managed by other PMs 170 | self._remove_all_pm_files_for_configs(all_configs - set(configs)) 171 | # always remove any unintended pm files for known configs managed by this PM 172 | self._remove_unintended_pm_files_for_configs(configs) 173 | elif not force: 174 | # --clean but no --force, so remove everything we know about 175 | self._remove_all_pm_files_for_configs(all_configs) 176 | pm_files = self._all_present_pm_files() 177 | if pm_files: 178 | gravity.io.warn(f"Configs not managed by this Gravity remain after cleaning, use --force to remove: {', '.join(pm_files)}") 179 | else: 180 | # --clean and --force 181 | self._remove_all_pm_files() 182 | 183 | def _create_dir_for(self, path): 184 | try: 185 | os.makedirs(os.path.dirname(path)) 186 | except OSError as exc: 187 | if exc.errno != errno.EEXIST: 188 | raise 189 | 190 | def _file_needs_update(self, path, contents): 191 | """Update if contents differ""" 192 | if os.path.exists(path): 193 | # check first whether there are changes 194 | with open(path) as fh: 195 | existing_contents = fh.read() 196 | return existing_contents != contents 197 | return True 198 | 199 | def _update_file(self, path, contents, name, file_type, force): 200 | if force or self._file_needs_update(path, contents): 201 | verb = "Updating" if os.path.exists(path) else "Adding" 202 | gravity.io.info(f"{verb} {file_type} {name}") 203 | self._create_dir_for(path) 204 | with open(path, "w") as out: 205 | out.write(contents) 206 | self._service_changes = True 207 | return True 208 | else: 209 | gravity.io.debug(f"No changes to existing config for {file_type} {name}: {path}") 210 | return False 211 | 212 | @abstractmethod 213 | def follow(self, configs=None, service_names=None, quiet=False): 214 | """ """ 215 | 216 | @abstractmethod 217 | def start(self, configs=None, service_names=None): 218 | """ """ 219 | 220 | @abstractmethod 221 | def stop(self, configs=None, service_names=None): 222 | """ """ 223 | 224 | @abstractmethod 225 | def restart(self, configs=None, service_names=None): 226 | """ """ 227 | 228 | @abstractmethod 229 | def graceful(self, configs=None, service_names=None): 230 | """ """ 231 | 232 | @abstractmethod 233 | def status(self): 234 | """ """ 235 | 236 | @abstractmethod 237 | def update(self, configs=None, service_names=None, force=False, clean=False): 238 | """ """ 239 | 240 | @abstractmethod 241 | def shutdown(self): 242 | """ """ 243 | 244 | @abstractmethod 245 | def terminate(self): 246 | """ """ 247 | 248 | @abstractmethod 249 | def pm(self, *args, **kwargs): 250 | """Direct pass-thru to process manager.""" 251 | 252 | 253 | class ProcessExecutor(BaseProcessExecutionEnvironment): 254 | def _service_environment_formatter(self, environment, format_vars): 255 | return {k: v.format(**format_vars) for k, v in environment.items()} 256 | 257 | def exec(self, config, service, service_instance_number=None, no_exec=False): 258 | service_name = service.service_name 259 | 260 | # if this is an instance of a service, we need to ensure that instance_number is formatted in as needed 261 | instance_count = service.count 262 | if instance_count > 1: 263 | msg = f"Cannot exec '{service_name}': This service is configured to use multiple instances and " 264 | if service_instance_number is None: 265 | gravity.io.exception(msg + "--service-instance was not set") 266 | if service_instance_number not in range(0, instance_count): 267 | gravity.io.exception(msg + "--service-instance is out of range") 268 | service_instance = service.get_service_instance(service_instance_number) 269 | else: 270 | service_instance = service 271 | 272 | # force generation of real commands 273 | config.service_command_style = ServiceCommandStyle.exec 274 | format_vars = self._service_format_vars(config, service_instance) 275 | print_env = ' '.join('{}={}'.format(k, shlex.quote(v)) for k, v in format_vars["environment"].items()) 276 | 277 | cmd = shlex.split(format_vars["command"]) 278 | env = {**dict(os.environ), **format_vars["environment"]} 279 | cwd = format_vars["galaxy_root"] or os.getcwd() 280 | 281 | # ensure the data dir exists 282 | try: 283 | os.makedirs(config.gravity_data_dir) 284 | except OSError as exc: 285 | if exc.errno != errno.EEXIST: 286 | raise 287 | 288 | gravity.io.info(f"Working directory: {cwd}") 289 | gravity.io.info(f"Executing: {print_env} {format_vars['command']}") 290 | 291 | if not no_exec: 292 | os.chdir(cwd) 293 | os.execvpe(cmd[0], cmd, env) 294 | 295 | 296 | class ProcessManagerRouter: 297 | def __init__(self, state_dir=None, config_file=None, config_manager=None, user_mode=None, process_manager=None, **kwargs): 298 | self.config_manager = config_manager or ConfigManager(state_dir=state_dir, 299 | config_file=config_file, 300 | user_mode=user_mode, 301 | process_manager=process_manager) 302 | self._process_executor = ProcessExecutor(config_manager=self.config_manager) 303 | self._load_pm_modules(**kwargs) 304 | 305 | def _load_pm_modules(self, *args, **kwargs): 306 | self.process_managers = {} 307 | for filename in os.listdir(os.path.dirname(__file__)): 308 | if filename.endswith(".py") and not filename.startswith("_"): 309 | mod = importlib.import_module("gravity.process_manager." + filename[: -len(".py")]) 310 | for name in dir(mod): 311 | obj = getattr(mod, name) 312 | if not name.startswith("_") and inspect.isclass(obj) and issubclass(obj, BaseProcessManager) and obj != BaseProcessManager: 313 | pm = obj(*args, config_manager=self.config_manager, process_executor=self._process_executor, **kwargs) 314 | self.process_managers[pm.name] = pm 315 | 316 | def _instance_service_names(self, names): 317 | instance_names = [] 318 | service_names = [] 319 | configured_instance_names = self.config_manager.get_configured_instance_names() 320 | configured_service_names = self.config_manager.get_configured_service_names() 321 | if names: 322 | for name in names: 323 | if name in configured_instance_names: 324 | instance_names.append(name) 325 | elif name in configured_service_names | VALID_SERVICE_NAMES: 326 | service_names.append(name) 327 | else: 328 | gravity.io.warn(f"Warning: Not a known instance or service name: {name}") 329 | if not instance_names and not service_names: 330 | gravity.io.exception("No provided names are known instance or service names") 331 | return (instance_names, service_names) 332 | 333 | def exec(self, instance_names=None, service_instance_number=None, no_exec=False): 334 | """ """ 335 | instance_names, service_names = self._instance_service_names(instance_names) 336 | 337 | if len(instance_names) == 0 and self.config_manager.single_instance: 338 | instance_names = None 339 | elif len(instance_names) != 1: 340 | gravity.io.exception("Only zero or one instance name can be provided") 341 | 342 | config = self.config_manager.get_configs(instances=instance_names)[0] 343 | service_list = ", ".join(s.service_name for s in config.services) 344 | 345 | if len(service_names) != 1: 346 | gravity.io.exception(f"Exactly one service name must be provided. Configured service(s): {service_list}") 347 | 348 | service_name = service_names[0] 349 | services = config.get_services(service_names) 350 | if not services: 351 | gravity.io.exception(f"Service '{service_name}' is not configured. Configured service(s): {service_list}") 352 | 353 | service = services[0] 354 | return self._process_executor.exec(config, service, service_instance_number=service_instance_number, no_exec=no_exec) 355 | 356 | @route 357 | def follow(self, instance_names=None, quiet=None): 358 | """ """ 359 | 360 | @route 361 | def start(self, instance_names=None): 362 | """ """ 363 | 364 | @route 365 | def stop(self, instance_names=None): 366 | """ """ 367 | 368 | @route 369 | def restart(self, instance_names=None): 370 | """ """ 371 | 372 | @route 373 | def graceful(self, instance_names=None): 374 | """ """ 375 | 376 | @route 377 | def status(self, instance_names=None): 378 | """ """ 379 | 380 | @route_to_all 381 | def update(self, instance_names=None, force=False, clean=False): 382 | """ """ 383 | 384 | @route 385 | def shutdown(self): 386 | """ """ 387 | 388 | @route 389 | def terminate(self): 390 | """ """ 391 | 392 | @route 393 | def pm(self, *args): 394 | """ """ 395 | -------------------------------------------------------------------------------- /gravity/process_manager/multiprocessing.py: -------------------------------------------------------------------------------- 1 | """ 2 | """ 3 | import multiprocessing 4 | 5 | from gravity.process_manager import BaseProcessManager, ProcessExecutor 6 | from gravity.settings import ProcessManager 7 | 8 | 9 | class MultiprocessingProcessManager(BaseProcessManager): 10 | 11 | name = ProcessManager.multiprocessing 12 | 13 | def __init__(self, process_executor=None, **kwargs): 14 | super().__init__(**kwargs) 15 | 16 | assert process_executor is not None, f"Process executor is required for {self.__class__.__name__}" 17 | self.process_executor = process_executor 18 | self.processes = [] 19 | 20 | def follow(self, configs=None, service_names=None, quiet=False): 21 | """ """ 22 | 23 | def start(self, configs=None, service_names=None): 24 | for config in configs: 25 | for service in config.services: 26 | process = multiprocessing.Process(target=self.process_executor.exec, args=(config, service)) 27 | process.start() 28 | self.processes.append(process) 29 | for process in self.processes: 30 | process.join() 31 | 32 | def pm(self, *args, **kwargs): 33 | """ """ 34 | 35 | def stop(self, configs=None, service_names=None): 36 | """ """ 37 | 38 | def _present_pm_files_for_config(self, config): 39 | """ """ 40 | 41 | def _disable_and_remove_pm_files(self, pm_files): 42 | """ """ 43 | 44 | def restart(self, configs=None, service_names=None): 45 | """ """ 46 | 47 | def graceful(self, configs=None, service_names=None): 48 | """ """ 49 | 50 | def status(self, configs=None, service_names=None): 51 | """ """ 52 | 53 | def terminate(self): 54 | """ """ 55 | 56 | def shutdown(self): 57 | """ """ 58 | 59 | def update(self, configs=None, force=False, clean=False): 60 | """ """ 61 | 62 | _service_environment_formatter = ProcessExecutor._service_environment_formatter 63 | -------------------------------------------------------------------------------- /gravity/process_manager/supervisor.py: -------------------------------------------------------------------------------- 1 | """ 2 | """ 3 | import os 4 | import shlex 5 | import subprocess 6 | import time 7 | from functools import partial 8 | from glob import glob 9 | 10 | import gravity.io 11 | from gravity.process_manager import BaseProcessManager 12 | from gravity.settings import ProcessManager 13 | from gravity.state import GracefulMethod 14 | from gravity.util import which 15 | 16 | from supervisor import supervisorctl # type: ignore 17 | 18 | SUPERVISORD_START_TIMEOUT = 60 19 | DEFAULT_SUPERVISOR_SOCKET_PATH = os.environ.get("SUPERVISORD_SOCKET", '%(here)s/supervisor.sock') 20 | 21 | SUPERVISORD_CONF_TEMPLATE = f"""; 22 | ; This file is maintained by Galaxy - CHANGES WILL BE OVERWRITTEN 23 | ; 24 | 25 | [unix_http_server] 26 | file = {DEFAULT_SUPERVISOR_SOCKET_PATH} 27 | 28 | [supervisord] 29 | logfile = %(here)s/supervisord.log 30 | pidfile = %(here)s/supervisord.pid 31 | loglevel = info 32 | nodaemon = false 33 | 34 | [rpcinterface:supervisor] 35 | supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface 36 | 37 | [supervisorctl] 38 | serverurl = unix://{DEFAULT_SUPERVISOR_SOCKET_PATH} 39 | 40 | [include] 41 | files = supervisord.conf.d/*.d/*.conf supervisord.conf.d/*.conf 42 | """ 43 | 44 | SUPERVISORD_SERVICE_TEMPLATE = """; 45 | ; This file is maintained by Gravity - CHANGES WILL BE OVERWRITTEN 46 | ; 47 | 48 | [program:{supervisor_program_name}] 49 | command = {command} 50 | directory = {galaxy_root} 51 | umask = {galaxy_umask} 52 | autostart = true 53 | autorestart = true 54 | stopasgroup = true 55 | startsecs = {settings[start_timeout]} 56 | stopwaitsecs = {settings[stop_timeout]} 57 | environment = {environment} 58 | numprocs = {service_instance_count} 59 | numprocs_start = {supervisor_numprocs_start} 60 | process_name = {supervisor_process_name} 61 | stdout_logfile = {log_file} 62 | redirect_stderr = true 63 | """ 64 | 65 | 66 | SUPERVISORD_GROUP_TEMPLATE = """; 67 | ; This file is maintained by Galaxy - CHANGES WILL BE OVERWRITTEN 68 | ; 69 | 70 | [group:{instance_name}] 71 | programs = {programs} 72 | """ 73 | 74 | DEFAULT_STATE_DIR = os.path.expanduser(os.path.join("~", ".config", "galaxy-gravity")) 75 | if "XDG_CONFIG_HOME" in os.environ: 76 | DEFAULT_STATE_DIR = os.path.join(os.environ["XDG_CONFIG_HOME"], "galaxy-gravity") 77 | 78 | 79 | class SupervisorProgram: 80 | # converts between different formats 81 | def __init__(self, config, service, use_instance_name): 82 | self.config = config 83 | self.service = service 84 | self._use_instance_name = use_instance_name 85 | 86 | self.config_process_name = "%(program_name)s" 87 | if self._use_instance_name: 88 | self.config_process_name = service.service_name 89 | 90 | self.config_numprocs = service.count 91 | self.config_numprocs_start = 0 92 | 93 | self.config_instance_program_name = self.config_program_name 94 | self.log_file_name_template = self.config_program_name 95 | 96 | if self.config_numprocs > 1: 97 | if self._use_instance_name: 98 | self.config_process_name = f"{service.service_name}%(process_num)d" 99 | else: 100 | self.config_process_name = "%(program_name)s_%(process_num)d" 101 | self.config_instance_program_name += "_%(process_num)d" 102 | self.log_file_name_template += "_{instance_number}" 103 | self.log_file_name_template += ".log" 104 | 105 | @property 106 | def config_file_name(self): 107 | service = self.service 108 | return f"{service.service_type}_{service.service_name}.conf" 109 | 110 | @property 111 | def config_program_name(self): 112 | """The representation in [program:NAME] in the supervisor config""" 113 | service = self.service 114 | if self._use_instance_name: 115 | instance_name = self.config.instance_name 116 | return f"{instance_name}_{service.service_type}_{service.service_name}" 117 | else: 118 | return service.service_name 119 | 120 | @property 121 | def config_log_file_name(self): 122 | return self.config_instance_program_name + ".log" 123 | 124 | @property 125 | def program_names(self): 126 | """The representation when performing commands, after group and procnums expansion""" 127 | instance_name = None 128 | if self._use_instance_name: 129 | instance_name = self.config.instance_name 130 | service_name = self.service.service_name 131 | instance_count = self.config_numprocs 132 | instance_number_start = self.config_numprocs_start 133 | return supervisor_program_names(service_name, instance_count, instance_number_start, instance_name=instance_name) 134 | 135 | @property 136 | def log_file_names(self): 137 | return list(self.log_file_name_template.format(instance_number=i) for i in range(0, self.config_numprocs)) 138 | 139 | 140 | class SupervisorProcessManager(BaseProcessManager): 141 | 142 | name = ProcessManager.supervisor 143 | 144 | def __init__(self, foreground=False, **kwargs): 145 | super().__init__(**kwargs) 146 | 147 | if self.config_manager.state_dir is not None: 148 | state_dir = self.config_manager.state_dir 149 | elif self.config_manager.instance_count > 1: 150 | state_dir = DEFAULT_STATE_DIR 151 | gravity.io.info(f"Supervisor configuration will be stored in {state_dir}, set --state-dir ($GRAVITY_STATE_DIR) to override") 152 | else: 153 | state_dir = self.config_manager.get_config().gravity_data_dir 154 | 155 | self.supervisord_exe = which("supervisord") 156 | self.supervisor_state_dir = os.path.join(state_dir, "supervisor") 157 | self.supervisord_conf_path = os.path.join(self.supervisor_state_dir, "supervisord.conf") 158 | self.supervisord_conf_dir = os.path.join(self.supervisor_state_dir, "supervisord.conf.d") 159 | self.supervisord_pid_path = os.path.join(self.supervisor_state_dir, "supervisord.pid") 160 | self.supervisord_sock_path = os.environ.get("SUPERVISORD_SOCKET", os.path.join(self.supervisor_state_dir, "supervisor.sock")) 161 | self.__supervisord_popen = None 162 | self.foreground = foreground 163 | 164 | @property 165 | def log_file(self): 166 | return os.path.join(self.supervisor_state_dir, "supervisord.log") 167 | 168 | def __supervisord_is_running(self): 169 | try: 170 | assert os.path.exists(self.supervisord_pid_path) 171 | assert os.path.exists(self.supervisord_sock_path) 172 | os.kill(int(open(self.supervisord_pid_path).read()), 0) 173 | return True 174 | except Exception: 175 | return False 176 | 177 | def __supervisord(self): 178 | format_vars = {"supervisor_state_dir": self.supervisor_state_dir, "supervisord_conf_dir": self.supervisord_conf_dir} 179 | supervisord_cmd = [self.supervisord_exe, "-c", self.supervisord_conf_path] 180 | if self.foreground: 181 | supervisord_cmd.append('--nodaemon') 182 | if not self.__supervisord_is_running(): 183 | # any time that supervisord is not running, let's rewrite supervisord.conf 184 | if not os.path.exists(self.supervisord_conf_dir): 185 | os.makedirs(self.supervisord_conf_dir) 186 | open(self.supervisord_conf_path, "w").write(SUPERVISORD_CONF_TEMPLATE.format(**format_vars)) 187 | self.__supervisord_popen = subprocess.Popen(supervisord_cmd, env=os.environ) 188 | rc = self.__supervisord_popen.poll() 189 | if rc: 190 | gravity.io.error("supervisord exited with code %d" % rc) 191 | start = time.time() 192 | while not os.path.exists(self.supervisord_pid_path) or not os.path.exists(self.supervisord_sock_path): 193 | if (time.time() - start) > SUPERVISORD_START_TIMEOUT: 194 | gravity.io.exception("Timed out waiting for supervisord to start") 195 | gravity.io.debug(f"Waiting for {self.supervisord_pid_path}") 196 | time.sleep(0.5) 197 | 198 | def __get_supervisor(self): 199 | """Return the supervisor proxy object 200 | 201 | Should probably use this more rather than supervisorctl directly 202 | """ 203 | options = supervisorctl.ClientOptions() 204 | options.realize(args=["-c", self.supervisord_conf_path]) 205 | return supervisorctl.Controller(options).get_supervisor() 206 | 207 | def _service_default_path(self): 208 | return "%(ENV_PATH)s" 209 | 210 | def _service_environment_formatter(self, environment, format_vars): 211 | return ",".join("{}={}".format(k, shlex.quote(v.format(**format_vars))) for k, v in environment.items()) 212 | 213 | def terminate(self): 214 | if self.foreground: 215 | # if running in foreground, if terminate is called, then supervisord should've already received a SIGINT 216 | self.__supervisord_popen and self.__supervisord_popen.wait() 217 | 218 | def _disable_and_remove_pm_files(self, pm_files): 219 | # don't need to stop anything - `supervisorctl update` afterward will take care of it 220 | if pm_files: 221 | gravity.io.info(f"Removing supervisor configs: {', '.join(pm_files)}") 222 | list(map(os.unlink, pm_files)) 223 | for instance_dir in set(os.path.dirname(f) for f in pm_files): 224 | # should maybe warn if the intent was to remove the entire instance 225 | if not os.listdir(instance_dir): 226 | gravity.io.info(f"Removing empty instance dir: {instance_dir}") 227 | os.rmdir(instance_dir) 228 | 229 | def _present_pm_files_for_config(self, config): 230 | pm_files = set() 231 | instance_name = config.instance_name 232 | instance_conf_dir = os.path.join(self.supervisord_conf_dir, f"{instance_name}.d") 233 | group_file = os.path.join(self.supervisord_conf_dir, f"group_{instance_name}.conf") 234 | if os.path.exists(group_file): 235 | pm_files.add(group_file) 236 | pm_files.update(glob(os.path.join(instance_conf_dir, "*"))) 237 | return pm_files 238 | 239 | def _intended_pm_files_for_config(self, config): 240 | pm_files = set() 241 | instance_name = config.instance_name 242 | instance_conf_dir = os.path.join(self.supervisord_conf_dir, f"{instance_name}.d") 243 | for service in config.services: 244 | program = SupervisorProgram(config, service, self._use_instance_name) 245 | pm_files.add(os.path.join(instance_conf_dir, program.config_file_name)) 246 | if self._use_instance_name: 247 | pm_files.add(os.path.join(self.supervisord_conf_dir, f"group_{instance_name}.conf")) 248 | return pm_files 249 | 250 | def _all_present_pm_files(self): 251 | return (glob(os.path.join(self.supervisord_conf_dir, "*.d", "*")) + 252 | glob(os.path.join(self.supervisord_conf_dir, "group_*.conf"))) 253 | 254 | def __update_service(self, config, service, instance_conf_dir, instance_name, force): 255 | program = SupervisorProgram(config, service, self._use_instance_name) 256 | # supervisor-specific format vars 257 | supervisor_format_vars = { 258 | "log_dir": config.log_dir, 259 | "log_file": os.path.join(config.log_dir, program.config_log_file_name), 260 | "instance_number": "%(process_num)d", 261 | "supervisor_program_name": program.config_program_name, 262 | "supervisor_process_name": program.config_process_name, 263 | "supervisor_numprocs_start": program.config_numprocs_start, 264 | } 265 | 266 | format_vars = self._service_format_vars(config, service, supervisor_format_vars) 267 | 268 | conf = os.path.join(instance_conf_dir, program.config_file_name) 269 | template = SUPERVISORD_SERVICE_TEMPLATE 270 | contents = template.format(**format_vars) 271 | name = service.service_name if not self._use_instance_name else f"{instance_name}:{service.service_name}" 272 | if self._update_file(conf, contents, name, "service", force): 273 | self.supervisorctl('reread') 274 | return conf 275 | 276 | def __process_config(self, config, force): 277 | """Perform necessary supervisor config updates as per current Galaxy/Gravity configuration. 278 | 279 | Does not call ``supervisorctl update``. 280 | """ 281 | instance_name = config.instance_name 282 | instance_conf_dir = os.path.join(self.supervisord_conf_dir, f"{instance_name}.d") 283 | 284 | programs = [] 285 | for service in config.services: 286 | self.__update_service(config, service, instance_conf_dir, instance_name, force) 287 | programs.append(f"{instance_name}_{service.service_type}_{service.service_name}") 288 | 289 | group_conf = os.path.join(self.supervisord_conf_dir, f"group_{instance_name}.conf") 290 | if self._use_instance_name: 291 | format_vars = {"instance_name": instance_name, "programs": ",".join(programs)} 292 | contents = SUPERVISORD_GROUP_TEMPLATE.format(**format_vars) 293 | if self._update_file(group_conf, contents, instance_name, "supervisor group", force): 294 | self.supervisorctl('reread') 295 | elif os.path.exists(group_conf): 296 | os.unlink(group_conf) 297 | 298 | def __process_configs(self, configs, force): 299 | for config in configs: 300 | self.__process_config(config, force) 301 | if not os.path.exists(config.log_dir): 302 | os.makedirs(config.log_dir) 303 | 304 | def __supervisor_programs(self, config, service_names): 305 | services = config.get_services(service_names) 306 | return [SupervisorProgram(config, service, self._use_instance_name) for service in services] 307 | 308 | def __supervisor_program_names(self, config, service_names): 309 | program_names = [] 310 | for program in self.__supervisor_programs(config, service_names): 311 | program_names.extend(program.program_names) 312 | return program_names 313 | 314 | def __op_on_programs(self, op, configs, service_names): 315 | targets = [] 316 | for config in configs: 317 | if service_names: 318 | targets.extend(self.__supervisor_program_names(config, service_names)) 319 | elif self._use_instance_name: 320 | targets.append(f"{config.instance_name}:*") 321 | else: 322 | targets.append("all") 323 | self.supervisorctl(op, *targets) 324 | 325 | def __reload_graceful(self, configs, service_names): 326 | for config in configs: 327 | services = config.get_services(service_names) 328 | for service in services: 329 | program = self.__supervisor_programs(config, [service.service_name])[0] 330 | graceful_method = service.graceful_method 331 | if graceful_method == GracefulMethod.SIGHUP: 332 | self.supervisorctl("signal", "SIGHUP", *program.program_names) 333 | elif graceful_method == GracefulMethod.ROLLING: 334 | self.__rolling_restart(config, service, program) 335 | elif graceful_method != GracefulMethod.NONE: 336 | self.supervisorctl("restart", *program.program_names) 337 | 338 | def __rolling_restart(self, config, service, program): 339 | restart_callbacks = list(partial(self.supervisorctl, "restart", p) for p in program.program_names) 340 | service.rolling_restart(restart_callbacks) 341 | 342 | def follow(self, configs=None, service_names=None, quiet=False): 343 | # supervisor has a built-in tail command but it only works on a single log file. `galaxyctl pm tail ...` can be 344 | # used if desired, though 345 | if not self.tail: 346 | gravity.io.exception("`tail` not found on $PATH, please install it") 347 | log_files = [] 348 | if quiet: 349 | cmd = [self.tail, "-f", self.log_file] 350 | tail_popen = subprocess.Popen(cmd) 351 | tail_popen.wait() 352 | else: 353 | for config in configs: 354 | log_dir = config.log_dir 355 | programs = self.__supervisor_programs(config, service_names) 356 | for program in programs: 357 | log_files.extend(os.path.join(log_dir, f) for f in program.log_file_names) 358 | cmd = [self.tail, "-f"] + log_files 359 | tail_popen = subprocess.Popen(cmd) 360 | tail_popen.wait() 361 | 362 | def start(self, configs=None, service_names=None): 363 | self.update(configs=configs) 364 | self.__supervisord() 365 | self.__op_on_programs("start", configs, service_names) 366 | self.supervisorctl("status") 367 | 368 | def stop(self, configs=None, service_names=None): 369 | self.__op_on_programs("stop", configs, service_names) 370 | # Exit supervisor if all processes are stopped 371 | supervisor = self.__get_supervisor() 372 | if self.__supervisord_is_running(): 373 | proc_infos = supervisor.getAllProcessInfo() 374 | if all([i["state"] == 0 for i in proc_infos]): 375 | gravity.io.info("All processes stopped, supervisord will exit") 376 | self.shutdown() 377 | else: 378 | gravity.io.info("Not all processes stopped, supervisord not shut down (hint: see `galaxyctl status`)") 379 | 380 | def restart(self, configs=None, service_names=None): 381 | self.update(configs=configs) 382 | if not self.__supervisord_is_running(): 383 | self.__supervisord() 384 | gravity.io.warn("supervisord was not previously running; it has been started, so the 'restart' command has been ignored") 385 | else: 386 | self.__op_on_programs("restart", configs, service_names) 387 | 388 | def graceful(self, configs=None, service_names=None): 389 | self.update(configs=configs) 390 | if not self.__supervisord_is_running(): 391 | self.__supervisord() 392 | gravity.io.warn("supervisord was not previously running; it has been started, so the 'graceful' command has been ignored") 393 | else: 394 | self.__reload_graceful(configs, service_names) 395 | 396 | def status(self, configs=None, service_names=None): 397 | # TODO: create our own formatted output 398 | # supervisor = self.get_supervisor() 399 | # all_infos = supervisor.getAllProcessInfo() 400 | self.__op_on_programs("status", configs, service_names) 401 | 402 | def shutdown(self): 403 | self.supervisorctl("shutdown") 404 | while self.__supervisord_is_running(): 405 | gravity.io.debug("Waiting for supervisord to terminate") 406 | time.sleep(0.5) 407 | gravity.io.info("supervisord has terminated") 408 | 409 | def update(self, configs=None, force=False, clean=False): 410 | """Add newly defined servers, remove any that are no longer present""" 411 | self._pre_update(configs, force, clean) 412 | if not clean: 413 | self.__process_configs(configs, force) 414 | # only need to update if supervisord is running, otherwise changes will be picked up at next start 415 | if self.__supervisord_is_running(): 416 | self.supervisorctl("update") 417 | 418 | def supervisorctl(self, *args): 419 | if not self.__supervisord_is_running(): 420 | gravity.io.warn("supervisord is not running") 421 | return 422 | try: 423 | gravity.io.debug("Calling supervisorctl with args: %s", list(args)) 424 | supervisorctl.main(args=["-c", self.supervisord_conf_path] + list(args)) 425 | except SystemExit as e: 426 | # supervisorctl.main calls sys.exit(), so we catch that 427 | if e.code == 0: 428 | pass 429 | else: 430 | raise 431 | 432 | pm = supervisorctl 433 | 434 | 435 | def supervisor_program_names(service_name, instance_count, instance_number_start, instance_name=None): 436 | # this is what supervisor turns the service name into depending on groups and numprocs 437 | if instance_count > 1 and instance_name is not None: 438 | return [f"{instance_name}:{service_name}{i + instance_number_start}" for i in range(0, instance_count)] 439 | 440 | if instance_count > 1: 441 | program_names = [f"{service_name}:{service_name}_{i + instance_number_start}" for i in range(0, instance_count)] 442 | else: 443 | program_names = [service_name] 444 | 445 | if instance_name is not None: 446 | return [f"{instance_name}:{program_name}" for program_name in program_names] 447 | else: 448 | return program_names 449 | -------------------------------------------------------------------------------- /gravity/process_manager/systemd.py: -------------------------------------------------------------------------------- 1 | """ 2 | """ 3 | import os 4 | import re 5 | import shlex 6 | import subprocess 7 | from glob import glob 8 | from functools import partial 9 | 10 | import gravity.io 11 | from gravity.process_manager import BaseProcessManager 12 | from gravity.settings import ProcessManager 13 | from gravity.state import GracefulMethod 14 | 15 | SYSTEMD_TARGET_HASH_RE = r";\s*GRAVITY=([0-9a-f]+)" 16 | 17 | SYSTEMD_SERVICE_TEMPLATE = """; 18 | ; This file is maintained by Gravity - CHANGES WILL BE OVERWRITTEN 19 | ; 20 | 21 | [Unit] 22 | Description={systemd_description} 23 | After=network.target 24 | After=time-sync.target 25 | PartOf={systemd_target} 26 | 27 | [Service] 28 | UMask={galaxy_umask} 29 | Type=simple 30 | {systemd_user_group} 31 | WorkingDirectory={galaxy_root} 32 | TimeoutStartSec={settings[start_timeout]} 33 | TimeoutStopSec={settings[stop_timeout]} 34 | ExecStart={command} 35 | {systemd_exec_reload} 36 | {environment} 37 | {systemd_memory_limit} 38 | Restart=always 39 | 40 | MemoryAccounting=yes 41 | CPUAccounting=yes 42 | BlockIOAccounting=yes 43 | 44 | [Install] 45 | WantedBy=multi-user.target 46 | """ 47 | 48 | SYSTEMD_TARGET_TEMPLATE = """; 49 | ; This file is maintained by Gravity - CHANGES WILL BE OVERWRITTEN 50 | ; 51 | 52 | ; This allows Gravity to keep track of which units it controls and should not be changed 53 | ; GRAVITY={gravity_config_hash} 54 | 55 | [Unit] 56 | Description={systemd_description} 57 | After=network.target 58 | After=time-sync.target 59 | Wants={systemd_target_wants} 60 | 61 | [Install] 62 | WantedBy=multi-user.target 63 | """ 64 | 65 | 66 | class SystemdService: 67 | # converts between different formats 68 | def __init__(self, config, service, use_instance_name): 69 | self.config = config 70 | self.service = service 71 | self._use_instance_name = use_instance_name 72 | 73 | if use_instance_name: 74 | prefix_instance_name = f"-{config.instance_name}" 75 | description_instance_name = f" {config.instance_name}" 76 | else: 77 | prefix_instance_name = "" 78 | description_instance_name = "" 79 | 80 | if self.service.count > 1: 81 | description_process = " (process %i)" 82 | else: 83 | description_process = "" 84 | 85 | self.unit_prefix = f"galaxy{prefix_instance_name}-{service.service_name}" 86 | self.description = f"Galaxy{description_instance_name}{service.service_name}{description_process}" 87 | 88 | @property 89 | def unit_file_name(self): 90 | instance_count = self.service.count 91 | if instance_count > 1: 92 | return f"{self.unit_prefix}@.service" 93 | else: 94 | return f"{self.unit_prefix}.service" 95 | 96 | @property 97 | def unit_names(self): 98 | """The representation when performing commands, after instance expansion""" 99 | instance_count = self.service.count 100 | if instance_count > 1: 101 | unit_names = [f"{self.unit_prefix}@{i}.service" for i in range(0, instance_count)] 102 | else: 103 | unit_names = [f"{self.unit_prefix}.service"] 104 | return unit_names 105 | 106 | 107 | class SystemdProcessManager(BaseProcessManager): 108 | 109 | name = ProcessManager.systemd 110 | 111 | def __init__(self, foreground=False, **kwargs): 112 | super(SystemdProcessManager, self).__init__(**kwargs) 113 | self.user_mode = self.config_manager.user_mode 114 | if self.user_mode is None: 115 | self.user_mode = not self.config_manager.is_root 116 | 117 | @property 118 | def __systemd_unit_dir(self): 119 | unit_path = os.environ.get("GRAVITY_SYSTEMD_UNIT_PATH") 120 | if not unit_path: 121 | unit_path = "/etc/systemd/system" if not self.user_mode else os.path.expanduser("~/.config/systemd/user") 122 | return unit_path 123 | 124 | def __systemctl(self, *args, ignore_rc=None, not_found_rc=None, capture=False, **kwargs): 125 | args = list(args) 126 | not_found_rc = not_found_rc or () 127 | call = subprocess.check_call 128 | extra_args = os.environ.get("GRAVITY_SYSTEMCTL_EXTRA_ARGS") 129 | if extra_args: 130 | args = shlex.split(extra_args) + args 131 | if self.user_mode: 132 | args = ["--user"] + args 133 | gravity.io.debug("Calling systemctl with args: %s", args) 134 | if capture: 135 | call = subprocess.check_output 136 | try: 137 | return call(["systemctl"] + args, text=True) 138 | except subprocess.CalledProcessError as exc: 139 | if exc.returncode in not_found_rc: 140 | gravity.io.exception("Some expected systemd units were not found, did you forget to run `galaxyctl update`?") 141 | if ignore_rc is None or exc.returncode not in ignore_rc: 142 | raise 143 | 144 | def __journalctl(self, *args, **kwargs): 145 | args = list(args) 146 | if self.user_mode: 147 | args = ["--user"] + args 148 | gravity.io.debug("Calling journalctl with args: %s", args) 149 | subprocess.check_call(["journalctl"] + args) 150 | 151 | def _service_default_path(self): 152 | environ = self.__systemctl("show-environment", capture=True) 153 | for line in environ.splitlines(): 154 | if line.startswith("PATH="): 155 | return line.split("=", 1)[1] 156 | 157 | def _service_environment_formatter(self, environment, format_vars): 158 | return "\n".join("Environment={}={}".format(k, shlex.quote(v.format(**format_vars))) for k, v in environment.items()) 159 | 160 | def terminate(self): 161 | # this is used to stop a foreground supervisord in the supervisor PM, so it is a no-op here 162 | pass 163 | 164 | def __target_unit_name(self, config): 165 | instance_name = f"-{config.instance_name}" if self._use_instance_name else "" 166 | return f"galaxy{instance_name}.target" 167 | 168 | def __unit_files_to_active_unit_names(self, unit_files): 169 | unit_names = [] 170 | for unit_file in unit_files: 171 | unit_file = os.path.basename(unit_file) 172 | if "@" in unit_file: 173 | at_position = unit_file.index("@") 174 | unit_arg = unit_file[:at_position + 1] + "*" + unit_file[at_position + 1:] 175 | else: 176 | unit_arg = unit_file 177 | list_output = self.__systemctl("list-units", "--plain", "--no-legend", unit_arg, capture=True) 178 | unit_names.extend(line.split()[0] for line in list_output.splitlines()) 179 | return unit_names 180 | 181 | def _disable_and_remove_pm_files(self, unit_files): 182 | for target in [u for u in unit_files if u.endswith(".target")]: 183 | self.__systemctl("disable", "--now", os.path.basename(target)) 184 | # stopping all the targets should also stop all the services, but we'll check to be sure 185 | active_unit_names = self.__unit_files_to_active_unit_names(unit_files) 186 | if active_unit_names: 187 | gravity.io.info(f"Stopping active units: {', '.join(active_unit_names)}") 188 | self.__systemctl("disable", "--now", *active_unit_names) 189 | if unit_files: 190 | gravity.io.info(f"Removing systemd configs: {', '.join(unit_files)}") 191 | list(map(os.unlink, unit_files)) 192 | self._service_changes = True 193 | 194 | def __read_gravity_config_hash_from_target(self, target_path): 195 | # systemd has no mechanism for isolated unit dirs, so if you were to invoke gravity's update command on two 196 | # different gravity config files in succession, the second call would see the unit files written by the first as 197 | # "unintended" and remove them. there is also no way to separate such "foreign" unit files (from another gravity 198 | # config) from ones generated by this gravity config but left behind after the instance_name is changed. to deal 199 | # with this, gravity stores a hash of the path of the gravity config used to generate a target unit file in the 200 | # file, so that it can determine whether or not it "owns" a given set of unit files, and only clean those. 201 | with open(target_path) as fh: 202 | for line in fh: 203 | match = re.match(SYSTEMD_TARGET_HASH_RE, line) 204 | if match: 205 | return match.group(1) 206 | 207 | def _present_pm_files_for_config(self, config): 208 | unit_files = set() 209 | instance_name = f"-{config.instance_name}" if self._use_instance_name else "" 210 | target = os.path.join(self.__systemd_unit_dir, f"galaxy{instance_name}.target") 211 | if os.path.exists(target): 212 | target_hash = self.__read_gravity_config_hash_from_target(target) 213 | if target_hash == config.path_hash: 214 | unit_files.add(target) 215 | unit_files.update(glob(f"{os.path.splitext(target)[0]}-*.service")) 216 | return unit_files 217 | 218 | def _intended_pm_files_for_config(self, config): 219 | unit_files = set() 220 | for service in config.services: 221 | systemd_service = SystemdService(config, service, self._use_instance_name) 222 | unit_files.add(os.path.join(self.__systemd_unit_dir, systemd_service.unit_file_name)) 223 | target_unit_name = self.__target_unit_name(config) 224 | unit_files.add(os.path.join(self.__systemd_unit_dir, target_unit_name)) 225 | return unit_files 226 | 227 | def _all_present_pm_files(self): 228 | return (glob(os.path.join(self.__systemd_unit_dir, "galaxy-*.service")) + 229 | glob(os.path.join(self.__systemd_unit_dir, "galaxy-*.target")) + 230 | glob(os.path.join(self.__systemd_unit_dir, "galaxy.target"))) 231 | 232 | def __update_service(self, config, service, systemd_service: SystemdService, force: bool): 233 | # under supervisor we expect that gravity is installed in the galaxy venv and the venv is active when gravity 234 | # runs, but under systemd this is not the case. we do assume $VIRTUAL_ENV is the galaxy venv if running as an 235 | # unprivileged user, though. 236 | virtualenv_dir = config.virtualenv 237 | environ_virtual_env = os.environ.get("VIRTUAL_ENV") 238 | if not virtualenv_dir and self.user_mode and environ_virtual_env: 239 | gravity.io.warn(f"Assuming Galaxy virtualenv is value of $VIRTUAL_ENV: {environ_virtual_env}") 240 | gravity.io.warn("Set `virtualenv` in Gravity configuration to override") 241 | virtualenv_dir = environ_virtual_env 242 | elif not virtualenv_dir: 243 | gravity.io.exception("The `virtualenv` Gravity config option must be set when using the systemd process manager") 244 | 245 | memory_limit = service.settings.get("memory_limit") or config.memory_limit 246 | if memory_limit: 247 | memory_limit = f"MemoryLimit={memory_limit}G" 248 | 249 | exec_reload = None 250 | if service.graceful_method == GracefulMethod.SIGHUP: 251 | exec_reload = "ExecReload=/bin/kill -HUP $MAINPID" 252 | 253 | # systemd-specific format vars 254 | systemd_format_vars = { 255 | "virtualenv_bin": shlex.quote(f'{os.path.join(virtualenv_dir, "bin")}{os.path.sep}'), 256 | "instance_number": "%i", 257 | "systemd_user_group": "", 258 | "systemd_exec_reload": exec_reload or "", 259 | "systemd_memory_limit": memory_limit or "", 260 | "systemd_description": systemd_service.description, 261 | "systemd_target": self.__target_unit_name(config), 262 | } 263 | if not self.user_mode: 264 | systemd_format_vars["systemd_user_group"] = f"User={config.galaxy_user}" 265 | if config.galaxy_group is not None: 266 | systemd_format_vars["systemd_user_group"] += f"\nGroup={config.galaxy_group}" 267 | 268 | format_vars = self._service_format_vars(config, service, systemd_format_vars) 269 | 270 | unit_file = systemd_service.unit_file_name 271 | conf = os.path.join(self.__systemd_unit_dir, unit_file) 272 | template = SYSTEMD_SERVICE_TEMPLATE 273 | contents = template.format(**format_vars) 274 | self._update_file(conf, contents, unit_file, "systemd unit", force) 275 | 276 | def __process_config(self, config, force): 277 | service_units = [] 278 | for service in config.services: 279 | systemd_service = SystemdService(config, service, self._use_instance_name) 280 | self.__update_service(config, service, systemd_service, force) 281 | service_units.extend(systemd_service.unit_names) 282 | 283 | # create systemd target 284 | target_unit_name = self.__target_unit_name(config) 285 | target_conf = os.path.join(self.__systemd_unit_dir, target_unit_name) 286 | format_vars = { 287 | "gravity_config_hash": config.path_hash, 288 | "systemd_description": "Galaxy", 289 | "systemd_target_wants": " ".join(service_units), 290 | } 291 | if self._use_instance_name: 292 | format_vars["systemd_description"] += f" {config.instance_name}" 293 | contents = SYSTEMD_TARGET_TEMPLATE.format(**format_vars) 294 | if self._update_file(target_conf, contents, target_unit_name, "systemd unit", force): 295 | self.__systemctl("enable", target_conf) 296 | 297 | def __process_configs(self, configs, force): 298 | for config in configs: 299 | self.__process_config(config, force) 300 | 301 | def __unit_names(self, configs, service_names, use_target=True, include_services=False): 302 | unit_names = [] 303 | for config in configs: 304 | services = config.services 305 | if not service_names and use_target: 306 | unit_names.append(self.__target_unit_name(config)) 307 | if not include_services: 308 | services = [] 309 | elif service_names: 310 | services = config.get_services(service_names) 311 | systemd_services = [SystemdService(config, s, self._use_instance_name) for s in services] 312 | for systemd_service in systemd_services: 313 | unit_names.extend(systemd_service.unit_names) 314 | return unit_names 315 | 316 | def follow(self, configs=None, service_names=None, quiet=False): 317 | """ """ 318 | unit_names = self.__unit_names(configs, service_names, use_target=False) 319 | u_args = [i for sl in list(zip(["-u"] * len(unit_names), unit_names)) for i in sl] 320 | self.__journalctl("-f", *u_args) 321 | 322 | def start(self, configs=None, service_names=None): 323 | """ """ 324 | self.update(configs=configs) 325 | unit_names = self.__unit_names(configs, service_names) 326 | self.__systemctl("start", *unit_names, not_found_rc=(5,)) 327 | self.status(configs=configs, service_names=service_names) 328 | 329 | def stop(self, configs=None, service_names=None): 330 | """ """ 331 | unit_names = self.__unit_names(configs, service_names) 332 | self.__systemctl("stop", *unit_names, not_found_rc=(5,)) 333 | self.status(configs=configs, service_names=service_names) 334 | 335 | def restart(self, configs=None, service_names=None): 336 | """ """ 337 | # this can result in a double restart if your configs changed, not ideal but we can't really control that 338 | self.update(configs=configs) 339 | unit_names = self.__unit_names(configs, service_names) 340 | self.__systemctl("restart", *unit_names, not_found_rc=(5,)) 341 | self.status(configs=configs, service_names=service_names) 342 | 343 | def __graceful_service(self, config, service, service_names): 344 | systemd_service = SystemdService(config, service, self._use_instance_name) 345 | if service.graceful_method == GracefulMethod.ROLLING: 346 | restart_callbacks = list(partial(self.__systemctl, "reload-or-restart", u) for u in systemd_service.unit_names) 347 | service.rolling_restart(restart_callbacks) 348 | elif service.graceful_method != GracefulMethod.NONE: 349 | self.__systemctl("reload-or-restart", *systemd_service.unit_names, not_found_rc=(5,)) 350 | gravity.io.info(f"Restarted: {', '.join(systemd_service.unit_names)}") 351 | 352 | def graceful(self, configs=None, service_names=None): 353 | """ """ 354 | self.update(configs=configs) 355 | # reload-or-restart on a target does a restart on its services, so we use the services directly 356 | for config in configs: 357 | services = config.get_services(service_names) 358 | for service in services: 359 | self.__graceful_service(config, service, service_names) 360 | 361 | def status(self, configs=None, service_names=None): 362 | """ """ 363 | unit_names = self.__unit_names(configs, service_names, include_services=True) 364 | if service_names: 365 | self.__systemctl("status", *unit_names, ignore_rc=(3,), not_found_rc=(4,)) 366 | else: 367 | self.__systemctl("list-units", "--all", *unit_names) 368 | 369 | def update(self, configs=None, force=False, clean=False): 370 | """ """ 371 | self._pre_update(configs, force, clean) 372 | if not clean: 373 | self.__process_configs(configs, force) 374 | if self._service_changes: 375 | self.__systemctl("daemon-reload") 376 | else: 377 | gravity.io.debug("No service changes, daemon-reload not performed") 378 | 379 | def shutdown(self): 380 | """ """ 381 | if self._use_instance_name: 382 | configs = self.config_manager.get_configs(process_manager=self.name) 383 | self.__systemctl("stop", *[f"galaxy-{c.instance_name}.target" for c in configs]) 384 | else: 385 | self.__systemctl("stop", "galaxy.target") 386 | 387 | def pm(self, *args): 388 | """ """ 389 | self.__systemctl(*args) 390 | -------------------------------------------------------------------------------- /gravity/settings.py: -------------------------------------------------------------------------------- 1 | import os 2 | from enum import Enum 3 | from typing import Any, Dict, List, Optional, Union 4 | 5 | try: 6 | from pydantic.v1 import BaseModel, BaseSettings, Extra, Field, validator 7 | except ImportError: 8 | from pydantic import BaseModel, BaseSettings, Extra, Field, validator 9 | 10 | DEFAULT_INSTANCE_NAME = "_default_" 11 | GX_IT_PROXY_MIN_VERSION = "0.0.6" 12 | 13 | 14 | def none_to_default(cls, v, field): 15 | if all( 16 | ( 17 | # Cater for the occasion where field.default in (0, False) 18 | getattr(field, "default", None) is not None, 19 | v is None, 20 | ) 21 | ): 22 | return field.default 23 | else: 24 | return v 25 | 26 | 27 | class LogLevel(str, Enum): 28 | debug = "DEBUG" 29 | info = "INFO" 30 | warning = "WARNING" 31 | error = "ERROR" 32 | 33 | 34 | class ProcessManager(str, Enum): 35 | supervisor = "supervisor" 36 | systemd = "systemd" 37 | multiprocessing = "multiprocessing" 38 | 39 | 40 | class ServiceCommandStyle(str, Enum): 41 | gravity = "gravity" 42 | direct = "direct" 43 | exec = "_exec" 44 | 45 | 46 | class AppServer(str, Enum): 47 | gunicorn = "gunicorn" 48 | unicornherder = "unicornherder" 49 | 50 | 51 | class Pool(str, Enum): 52 | prefork = "prefork" 53 | eventlet = "eventlet" 54 | gevent = "gevent" 55 | solo = "solo" 56 | processes = "processes" 57 | threads = "threads" 58 | 59 | 60 | class TusdSettings(BaseModel): 61 | enable: bool = Field(False, description=""" 62 | Enable tusd server. 63 | If enabled, you also need to set up your proxy as outlined in https://docs.galaxyproject.org/en/latest/admin/nginx.html#receiving-files-via-the-tus-protocol. 64 | """) 65 | tusd_path: str = Field(default="tusd", description="""Path to tusd binary""") 66 | host: str = Field("localhost", description="Host to bind the tusd server to") 67 | port: int = Field(1080, description="Port to bind the tusd server to") 68 | upload_dir: str = Field(description=""" 69 | Directory to store uploads in. 70 | Must match ``tus_upload_store`` setting in ``galaxy:`` section. 71 | """) 72 | hooks_http: str = Field(default="/api/upload/hooks", description=""" 73 | Value of tusd -hooks-httpd option 74 | 75 | the default of is suitable for using tusd for Galaxy uploads and should not be changed unless you are using tusd for 76 | other purposes such as Pulsar staging. 77 | 78 | The value of galaxy_infrastructure_url is automatically prepended if the option starts with a `/` 79 | """) 80 | hooks_enabled_events: str = Field(default="pre-create", description=""" 81 | Comma-separated string of enabled tusd hooks. 82 | 83 | Leave at the default value to require authorization at upload creation time. 84 | This means Galaxy's web process does not need to be running after creating the initial 85 | upload request. 86 | 87 | Set to empty string to disable all authorization. This means data can be uploaded (but not processed) 88 | without the Galaxy web process being available. 89 | 90 | You can find a list of available hooks at https://github.com/tus/tusd/blob/master/docs/hooks.md#list-of-available-hooks. 91 | """) 92 | extra_args: str = Field(default="", description="Extra arguments to pass to tusd command line.") 93 | umask: Optional[str] = Field(None, description="umask under which service should be executed") 94 | start_timeout: int = Field(10, description="Value of supervisor startsecs, systemd TimeoutStartSec") 95 | stop_timeout: int = Field(10, description="Value of supervisor stopwaitsecs, systemd TimeoutStopSec") 96 | memory_limit: Optional[int] = Field( 97 | None, 98 | description=""" 99 | Memory limit (in GB). If the service exceeds the limit, it will be killed. Default is no limit or the value of the 100 | ``memory_limit`` setting at the top level of the Gravity configuration, if set. Ignored if ``process_manager`` is 101 | ``supervisor``. 102 | """) 103 | environment: Dict[str, str] = Field( 104 | default={}, 105 | description=""" 106 | Extra environment variables and their values to set when running the service. A dictionary where keys are the variable 107 | names. 108 | """) 109 | 110 | 111 | class CelerySettings(BaseModel): 112 | enable: bool = Field(True, description="Enable Celery distributed task queue.") 113 | enable_beat: bool = Field(True, description="Enable Celery Beat periodic task runner.") 114 | concurrency: int = Field(2, ge=0, description="Number of Celery Workers to start.") 115 | loglevel: LogLevel = Field(LogLevel.debug, description="Log Level to use for Celery Worker.") 116 | queues: str = Field("celery,galaxy.internal,galaxy.external", description="Queues to join") 117 | pool: Pool = Field(Pool.threads, description="Pool implementation") 118 | extra_args: str = Field(default="", description="Extra arguments to pass to Celery command line.") 119 | umask: Optional[str] = Field(None, description="umask under which service should be executed") 120 | start_timeout: int = Field(10, description="Value of supervisor startsecs, systemd TimeoutStartSec") 121 | stop_timeout: int = Field(10, description="Value of supervisor stopwaitsecs, systemd TimeoutStopSec") 122 | memory_limit: Optional[int] = Field( 123 | None, 124 | description=""" 125 | Memory limit (in GB). If the service exceeds the limit, it will be killed. Default is no limit or the value of the 126 | ``memory_limit`` setting at the top level of the Gravity configuration, if set. Ignored if ``process_manager`` is 127 | ``supervisor``. 128 | """) 129 | environment: Dict[str, str] = Field( 130 | default={}, 131 | description=""" 132 | Extra environment variables and their values to set when running the service. A dictionary where keys are the variable 133 | names. 134 | """) 135 | 136 | class Config: 137 | use_enum_values = True 138 | 139 | 140 | class GunicornSettings(BaseModel): 141 | enable: bool = Field(True, description="Enable Galaxy gunicorn server.") 142 | bind: str = Field( 143 | default="localhost:8080", 144 | description="The socket to bind. A string of the form: ``HOST``, ``HOST:PORT``, ``unix:PATH``, ``fd://FD``. An IP is a valid HOST.", 145 | ) 146 | workers: int = Field( 147 | default=1, 148 | ge=1, 149 | description=""" 150 | Controls the number of Galaxy application processes Gunicorn will spawn. 151 | Increased web performance can be attained by increasing this value. 152 | If Gunicorn is the only application on the server, a good starting value is the number of CPUs * 2 + 1. 153 | 4-12 workers should be able to handle hundreds if not thousands of requests per second. 154 | """) 155 | timeout: int = Field( 156 | default=300, 157 | ge=0, 158 | description=""" 159 | Gunicorn workers silent for more than this many seconds are killed and restarted. 160 | Value is a positive number or 0. Setting it to 0 has the effect of infinite timeouts by disabling timeouts for all workers entirely. 161 | If you disable the ``preload`` option workers need to have finished booting within the timeout. 162 | """) 163 | extra_args: str = Field(default="", description="Extra arguments to pass to Gunicorn command line.") 164 | preload: Optional[bool] = Field( 165 | default=None, 166 | description=""" 167 | Use Gunicorn's --preload option to fork workers after loading the Galaxy Application. 168 | Consumes less memory when multiple processes are configured. Default is ``false`` if using unicornherder, else ``true``. 169 | """) 170 | umask: Optional[str] = Field(None, description="umask under which service should be executed") 171 | start_timeout: int = Field(15, description="Value of supervisor startsecs, systemd TimeoutStartSec") 172 | stop_timeout: int = Field(65, description="Value of supervisor stopwaitsecs, systemd TimeoutStopSec") 173 | restart_timeout: int = Field( 174 | default=300, 175 | description=""" 176 | Amount of time to wait for a server to become alive when performing rolling restarts. 177 | """) 178 | memory_limit: Optional[int] = Field( 179 | None, 180 | description=""" 181 | Memory limit (in GB). If the service exceeds the limit, it will be killed. Default is no limit or the value of the 182 | ``memory_limit`` setting at the top level of the Gravity configuration, if set. Ignored if ``process_manager`` is 183 | ``supervisor``. 184 | """) 185 | environment: Dict[str, str] = Field( 186 | default={}, 187 | description=""" 188 | Extra environment variables and their values to set when running the service. A dictionary where keys are the variable 189 | names. 190 | """) 191 | 192 | 193 | class ReportsSettings(BaseModel): 194 | enable: bool = Field(False, description="Enable Galaxy Reports server.") 195 | config_file: str = Field("reports.yml", description="Path to reports.yml, relative to galaxy.yml if not absolute") 196 | bind: str = Field( 197 | default="localhost:9001", 198 | description="The socket to bind. A string of the form: ``HOST``, ``HOST:PORT``, ``unix:PATH``, ``fd://FD``. An IP is a valid HOST.", 199 | ) 200 | workers: int = Field( 201 | default=1, 202 | ge=1, 203 | description=""" 204 | Controls the number of Galaxy Reports application processes Gunicorn will spawn. 205 | It is not generally necessary to increase this for the low-traffic Reports server. 206 | """) 207 | timeout: int = Field( 208 | default=300, 209 | ge=0, 210 | description=""" 211 | Gunicorn workers silent for more than this many seconds are killed and restarted. 212 | Value is a positive number or 0. Setting it to 0 has the effect of infinite timeouts by disabling timeouts for all workers entirely. 213 | """) 214 | url_prefix: Optional[str] = Field( 215 | default=None, 216 | description=""" 217 | URL prefix to serve from. 218 | The corresponding nginx configuration is (replace and with the values from these options): 219 | 220 | location // { 221 | proxy_pass http:///; 222 | } 223 | 224 | If is a unix socket, you will need a ``:`` after the socket path but before the trailing slash like so: 225 | proxy_pass http://unix:/run/reports.sock:/; 226 | """) 227 | extra_args: str = Field(default="", description="Extra arguments to pass to Gunicorn command line.") 228 | umask: Optional[str] = Field(None, description="umask under which service should be executed") 229 | start_timeout: int = Field(10, description="Value of supervisor startsecs, systemd TimeoutStartSec") 230 | stop_timeout: int = Field(10, description="Value of supervisor stopwaitsecs, systemd TimeoutStopSec") 231 | memory_limit: Optional[int] = Field( 232 | None, 233 | description=""" 234 | Memory limit (in GB). If the service exceeds the limit, it will be killed. Default is no limit or the value of the 235 | ``memory_limit`` setting at the top level of the Gravity configuration, if set. Ignored if ``process_manager`` is 236 | ``supervisor``. 237 | """) 238 | environment: Dict[str, str] = Field( 239 | default={}, 240 | description=""" 241 | Extra environment variables and their values to set when running the service. A dictionary where keys are the variable 242 | names. 243 | """) 244 | 245 | 246 | class GxItProxySettings(BaseModel): 247 | enable: bool = Field(default=False, description="Set to true to start gx-it-proxy") 248 | version: str = Field(default=f">={GX_IT_PROXY_MIN_VERSION}", description="gx-it-proxy version") 249 | ip: str = Field(default="localhost", description="Public-facing IP of the proxy") 250 | port: int = Field(default=4002, description="Public-facing port of the proxy") 251 | sessions: str = Field( 252 | default="database/interactivetools_map.sqlite", 253 | description=""" 254 | Database to monitor. 255 | Should be set to the same value as ``interactivetools_map`` (or ``interactivetoolsproxy_map``) in the ``galaxy:`` section. This is 256 | ignored if either ``interactivetools_map`` or ``interactivetoolsproxy_map`` are set. 257 | """) 258 | verbose: bool = Field(default=True, description="Include verbose messages in gx-it-proxy") 259 | forward_ip: Optional[str] = Field( 260 | default=None, 261 | description=""" 262 | Forward all requests to IP. 263 | This is an advanced option that is only needed when proxying to remote interactive tool container that cannot be reached through the local network. 264 | """) 265 | forward_port: Optional[int] = Field( 266 | default=None, 267 | description=""" 268 | Forward all requests to port. 269 | This is an advanced option that is only needed when proxying to remote interactive tool container that cannot be reached through the local network.""") 270 | reverse_proxy: Optional[bool] = Field( 271 | default=False, 272 | description=""" 273 | Rewrite location blocks with proxy port. 274 | This is an advanced option that is only needed when proxying to remote interactive tool container that cannot be reached through the local network. 275 | """) 276 | umask: Optional[str] = Field(None, description="umask under which service should be executed") 277 | start_timeout: int = Field(10, description="Value of supervisor startsecs, systemd TimeoutStartSec") 278 | stop_timeout: int = Field(10, description="Value of supervisor stopwaitsecs, systemd TimeoutStopSec") 279 | memory_limit: Optional[int] = Field( 280 | None, 281 | description=""" 282 | Memory limit (in GB). If the service exceeds the limit, it will be killed. Default is no limit or the value of the 283 | ``memory_limit`` setting at the top level of the Gravity configuration, if set. Ignored if ``process_manager`` is 284 | ``supervisor``. 285 | """) 286 | environment: Dict[str, str] = Field( 287 | default={}, 288 | description=""" 289 | Extra environment variables and their values to set when running the service. A dictionary where keys are the variable 290 | names. 291 | """) 292 | 293 | 294 | class Settings(BaseSettings): 295 | """ 296 | Configuration for Gravity process manager. 297 | ``uwsgi:`` section will be ignored if Galaxy is started via Gravity commands (e.g ``./run.sh``, ``galaxy`` or ``galaxyctl``). 298 | """ 299 | 300 | process_manager: Optional[ProcessManager] = Field( 301 | None, 302 | description=""" 303 | Process manager to use. 304 | ``supervisor`` is the default process manager when Gravity is invoked as a non-root user. 305 | ``systemd`` is the default when Gravity is invoked as root. 306 | ``multiprocessing`` is the default when Gravity is invoked as the foreground shortcut ``galaxy`` instead of ``galaxyctl`` 307 | """) 308 | 309 | service_command_style: ServiceCommandStyle = Field( 310 | ServiceCommandStyle.gravity, 311 | description=""" 312 | What command to write to the process manager configs 313 | `gravity` (`galaxyctl exec `) is the default 314 | `direct` (each service's actual command) is also supported. 315 | """) 316 | 317 | use_service_instances: bool = Field( 318 | True, 319 | description=""" 320 | Use the process manager's *service instance* functionality for services that can run multiple instances. 321 | Presently this includes services like gunicorn and Galaxy dynamic job handlers. Service instances are only supported if 322 | ``service_command_style`` is ``gravity``, and so this option is automatically set to ``false`` if 323 | ``service_command_style`` is set to ``direct``. 324 | """) 325 | 326 | umask: str = Field("022", description=""" 327 | umask under which services should be executed. Setting ``umask`` on an individual service overrides this value. 328 | """) 329 | 330 | memory_limit: Optional[int] = Field( 331 | None, 332 | description=""" 333 | Memory limit (in GB), processes exceeding the limit will be killed. Default is no limit. If set, this is default value 334 | for all services. Setting ``memory_limit`` on an individual service overrides this value. Ignored if ``process_manager`` 335 | is ``supervisor``. 336 | """) 337 | 338 | galaxy_config_file: Optional[str] = Field( 339 | None, 340 | description=""" 341 | Specify Galaxy config file (galaxy.yml), if the Gravity config is separate from the Galaxy config. Assumed to be the 342 | same file as the Gravity config if a ``galaxy`` key exists at the root level, otherwise, this option is required. 343 | """) 344 | galaxy_root: Optional[str] = Field( 345 | None, 346 | description=""" 347 | Specify Galaxy's root directory. 348 | Gravity will attempt to find the root directory, but you can set the directory explicitly with this option. 349 | """) 350 | galaxy_user: Optional[str] = Field( 351 | None, 352 | description=""" 353 | User to run Galaxy as, required when using the systemd process manager as root. 354 | Ignored if ``process_manager`` is ``supervisor`` or user-mode (non-root) ``systemd``. 355 | """) 356 | galaxy_group: Optional[str] = Field( 357 | None, 358 | description=""" 359 | Group to run Galaxy as, optional when using the systemd process manager as root. 360 | Ignored if ``process_manager`` is ``supervisor`` or user-mode (non-root) ``systemd``. 361 | """) 362 | log_dir: Optional[str] = Field( 363 | None, 364 | description=""" 365 | Set to a directory that should contain log files for the processes controlled by Gravity. 366 | If not specified defaults to ``/gravity/log``. 367 | """) 368 | virtualenv: Optional[str] = Field(None, description=""" 369 | Set to Galaxy's virtualenv directory. 370 | If not specified, Gravity assumes all processes are on PATH. This option is required in most circumstances when using 371 | the ``systemd`` process manager. 372 | """) 373 | app_server: AppServer = Field( 374 | AppServer.gunicorn, 375 | description=""" 376 | Select the application server. 377 | ``gunicorn`` is the default application server. 378 | ``unicornherder`` is a production-oriented manager for (G)unicorn servers that automates zero-downtime Galaxy server restarts, 379 | similar to uWSGI Zerg Mode used in the past. 380 | """) 381 | instance_name: str = Field(default=DEFAULT_INSTANCE_NAME, description="""Override the default instance name. 382 | this is hidden from you when running a single instance.""") 383 | gunicorn: Union[List[GunicornSettings], GunicornSettings] = Field(default={}, description=""" 384 | Configuration for Gunicorn. Can be a list to run multiple gunicorns for rolling restarts. 385 | """) 386 | celery: CelerySettings = Field(default={}, description="Configuration for Celery Processes.") 387 | gx_it_proxy: GxItProxySettings = Field(default={}, description="Configuration for gx-it-proxy.") 388 | # The default value for tusd is a little awkward, but is a convenient way to ensure that if 389 | # a user enables tusd that they most also set upload_dir, and yet have the default be valid. 390 | tusd: Union[List[TusdSettings], TusdSettings] = Field(default={'upload_dir': ''}, description=""" 391 | Configuration for tusd server (https://github.com/tus/tusd). 392 | The ``tusd`` binary must be installed manually and made available on PATH (e.g in galaxy's .venv/bin directory). 393 | """) 394 | reports: ReportsSettings = Field(default={}, description="Configuration for Galaxy Reports.") 395 | handlers: Dict[str, Dict[str, Any]] = Field( 396 | default={}, 397 | description=""" 398 | Configure dynamic handlers in this section. 399 | See https://docs.galaxyproject.org/en/latest/admin/scaling.html#dynamically-defined-handlers for details. 400 | """) 401 | 402 | # Use validators to turn None to default value 403 | _normalize_gunicorn = validator("gunicorn", allow_reuse=True, pre=True)(none_to_default) 404 | _normalize_gx_it_proxy = validator("gx_it_proxy", allow_reuse=True, pre=True)(none_to_default) 405 | _normalize_celery = validator("celery", allow_reuse=True, pre=True)(none_to_default) 406 | _normalize_tusd = validator("tusd", allow_reuse=True, pre=True)(none_to_default) 407 | _normalize_reports = validator("reports", allow_reuse=True, pre=True)(none_to_default) 408 | 409 | # Require galaxy_user if running as root 410 | @validator("galaxy_user") 411 | def _user_required_if_root(cls, v, values): 412 | if os.geteuid() == 0: 413 | is_systemd = values["process_manager"] == ProcessManager.systemd 414 | if is_systemd and not v: 415 | raise ValueError("galaxy_user is required when running as root") 416 | elif not is_systemd: 417 | raise ValueError("Gravity cannot be run as root unless using the systemd process manager") 418 | return v 419 | 420 | # automatically set process_manager to systemd if unset and running is root 421 | @validator("process_manager") 422 | def _process_manager_systemd_if_root(cls, v, values): 423 | if v is None: 424 | if os.geteuid() == 0: 425 | v = ProcessManager.systemd.value 426 | return v 427 | 428 | # disable service instances unless command style is gravity 429 | @validator("use_service_instances") 430 | def _disable_service_instances_if_direct(cls, v, values): 431 | if values["service_command_style"] != ServiceCommandStyle.gravity: 432 | v = False 433 | return v 434 | 435 | class Config: 436 | env_prefix = "gravity_" 437 | env_nested_delimiter = "." 438 | case_sensitive = False 439 | use_enum_values = True 440 | # Ignore extra fields so you can switch from gravity versions that recognize new fields 441 | # to an older version that does not specify the fields, without having to comment them out. 442 | extra = Extra.ignore 443 | -------------------------------------------------------------------------------- /gravity/util/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | """ 3 | import collections.abc 4 | import copy 5 | import os 6 | import sys 7 | 8 | import jsonref 9 | import requests 10 | import requests_unixsocket 11 | import yaml 12 | 13 | from gravity.settings import Settings 14 | 15 | 16 | def recursive_update(to_update, update_from): 17 | """ 18 | Update values in `to_update` with values in `update_from`. 19 | 20 | Does not mutate values in to_update, but returns a new dictionary. 21 | """ 22 | d = copy.deepcopy(to_update) 23 | for k, v in update_from.items(): 24 | if isinstance(v, collections.abc.Mapping): 25 | d[k] = recursive_update(d.get(k, {}), v) 26 | else: 27 | d[k] = v 28 | return d 29 | 30 | 31 | def which(file): 32 | # http://stackoverflow.com/questions/5226958/which-equivalent-function-in-python 33 | if os.path.exists(os.path.dirname(sys.executable) + "/" + file): 34 | return os.path.dirname(sys.executable) + "/" + file 35 | for path in os.environ["PATH"].split(":"): 36 | if os.path.exists(path + "/" + file): 37 | return path + "/" + file 38 | return None 39 | 40 | 41 | def settings_to_sample(): 42 | schema = Settings.schema_json() 43 | # expand schema for easier processing 44 | data = jsonref.loads(schema) 45 | strings = [process_property("gravity", data)] 46 | for key, value in data["properties"].items(): 47 | strings.append(process_property(key, value, 1)) 48 | concat = "\n".join(strings) 49 | return concat 50 | 51 | 52 | def process_property(key, value, depth=0): 53 | extra_white_space = " " * depth 54 | default = value.get("default", "") 55 | if isinstance(default, dict): 56 | # Little hack that prevents listing the default value for tusd in the sample config 57 | default = {} 58 | if default != "": 59 | # make values more yaml-like. 60 | default = yaml.dump(default) 61 | if default.endswith("\n...\n"): 62 | default = default[: -(len("\n...\n"))] 63 | default = default.strip() 64 | description = "\n".join(f"{extra_white_space}# {desc}".rstrip() for desc in value["description"].strip().split("\n")) 65 | combined = value.get("allOf", []) 66 | if not combined and value.get("anyOf"): 67 | # we've got a union 68 | combined = [c for c in value["anyOf"] if c["type"] == "object"] 69 | if combined and combined[0].get("properties"): 70 | # we've got a nested map, add key once 71 | description = f"{description}\n{extra_white_space}{key}:\n" 72 | has_child = False 73 | for item in combined: 74 | if "enum" in item: 75 | enum_items = [i for i in item["enum"] if not i.startswith("_")] 76 | description = f'{description}\n{extra_white_space}# Valid options are: {", ".join(enum_items)}' 77 | if "properties" in item: 78 | has_child = True 79 | for _key, _value in item["properties"].items(): 80 | description = f"{description}\n{process_property(_key, _value, depth=depth+1)}" 81 | if not has_child or key == "handlers": 82 | comment = "# " 83 | if key == "gravity": 84 | # gravity section should not be commented 85 | comment = "" 86 | if default == "": 87 | value_sep = "" 88 | else: 89 | value_sep = " " 90 | description = f"{description}\n{extra_white_space}{comment}{key}:{value_sep}{default}\n" 91 | return description 92 | 93 | 94 | def http_check(bind, path): 95 | if bind.startswith("unix:"): 96 | socket = requests.utils.quote(bind.split(":", 1)[1], safe="") 97 | session = requests_unixsocket.Session() 98 | response = session.get(f"http+unix://{socket}{path}") 99 | else: 100 | response = requests.get(f"http://{bind}{path}", timeout=30) 101 | response.raise_for_status() 102 | return response 103 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.black] 2 | line-length = 160 3 | 4 | [tool.pytest.ini_options] 5 | norecursedirs = "tests/galaxy.git/* tests/galaxy_venv" 6 | timeout = 300 7 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal = 0 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | import ast 4 | import os 5 | import re 6 | 7 | from setuptools import find_packages, setup 8 | 9 | with open("README.rst") as file: 10 | long_description = file.read() 11 | 12 | long_description += "\n\n" 13 | with open("HISTORY.rst") as file: 14 | long_description += file.read() 15 | 16 | with open(os.path.join("gravity", "__init__.py")) as f: 17 | init_contents = f.read() 18 | 19 | def get_var(var_name): 20 | pattern = re.compile(r"%s\s+=\s+(.*)" % var_name) 21 | match = pattern.search(init_contents).group(1) 22 | return str(ast.literal_eval(match)) 23 | 24 | version = get_var("__version__") 25 | 26 | setup( 27 | name="gravity", 28 | version=version, 29 | packages=find_packages(), 30 | description="Command-line utilities to assist in managing Galaxy servers", 31 | long_description=long_description, 32 | long_description_content_type="text/x-rst", 33 | url="https://github.com/galaxyproject/gravity", 34 | author="The Galaxy Team", 35 | author_email="team@galaxyproject.org", 36 | license="MIT", 37 | keywords="gravity galaxy", 38 | python_requires=">=3.7", 39 | install_requires=[ 40 | "Click", 41 | "supervisor", 42 | "pyyaml", 43 | "packaging", 44 | "pydantic<3", # pydantic.v1 import will be removed in v3 45 | "jsonref", 46 | "requests", 47 | "requests-unixsocket", 48 | ], 49 | entry_points={"console_scripts": [ 50 | "galaxy = gravity.cli:galaxy", 51 | "galaxyctl = gravity.cli:galaxyctl", 52 | ]}, 53 | classifiers=[ 54 | "Intended Audience :: System Administrators", 55 | "License :: OSI Approved :: MIT License", 56 | "Natural Language :: English", 57 | "Operating System :: POSIX", 58 | "Programming Language :: Python :: 3", 59 | "Programming Language :: Python :: 3.8", 60 | "Programming Language :: Python :: 3.9", 61 | "Programming Language :: Python :: 3.10", 62 | "Programming Language :: Python :: 3.11", 63 | ], 64 | zip_safe=False, 65 | ) 66 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import os 2 | import glob 3 | import signal 4 | import shutil 5 | import socket 6 | import subprocess 7 | import tempfile 8 | from pathlib import Path 9 | 10 | import pytest 11 | import yaml 12 | from gravity import config_manager 13 | 14 | GALAXY_BRANCH = os.environ.get("GRAVITY_TEST_GALAXY_BRANCH", "dev") 15 | TEST_DIR = Path(os.path.dirname(__file__)) 16 | GXIT_CONFIG = """ 17 | gravity: 18 | process_manager: {process_manager_name} 19 | service_command_style: direct 20 | instance_name: {instance_name} 21 | gunicorn: 22 | bind: 'localhost:{gx_port}' 23 | gx_it_proxy: 24 | enable: true 25 | port: {gxit_port} 26 | verbose: true 27 | galaxy: 28 | conda_auto_init: false 29 | interactivetools_enable: true 30 | interactivetools_map: database/interactivetools_map.sqlite 31 | galaxy_infrastructure_url: http://localhost:{gx_port} 32 | interactivetools_upstream_proxy: false 33 | interactivetools_proxy_host: localhost:{gxit_port} 34 | """ 35 | 36 | 37 | @pytest.fixture(scope='session') 38 | def galaxy_git_dir(): 39 | galaxy_dir = TEST_DIR / 'galaxy.git' 40 | if not galaxy_dir.exists(): 41 | subprocess.run(['git', 'clone', '--bare', '--depth=1', '--branch', GALAXY_BRANCH, 'https://github.com/galaxyproject/galaxy'], cwd=TEST_DIR) 42 | yield galaxy_dir 43 | 44 | 45 | @pytest.fixture(scope='session') 46 | def galaxy_root_dir(galaxy_git_dir, tmpdir_factory): 47 | tmpdir = tmpdir_factory.mktemp('galaxy-worktree') 48 | subprocess.run(['git', 'worktree', 'add', '-d', str(tmpdir)], cwd=str(galaxy_git_dir)) 49 | return tmpdir 50 | 51 | 52 | @pytest.fixture() 53 | def galaxy_yml(galaxy_root_dir): 54 | config = galaxy_root_dir / 'config' / 'galaxy123.yml' 55 | sample_config = galaxy_root_dir / 'config' / 'galaxy.yml.sample' 56 | sample_config.copy(config) 57 | try: 58 | yield config 59 | finally: 60 | config.remove() 61 | 62 | 63 | @pytest.fixture() 64 | def state_dir(monkeypatch): 65 | directory = tempfile.mkdtemp(prefix="gravity_test") 66 | unit_path = f"/run/user/{os.getuid()}/systemd/user" 67 | monkeypatch.setenv("GRAVITY_STATE_DIR", directory) 68 | monkeypatch.setenv("GRAVITY_SYSTEMD_UNIT_PATH", unit_path) 69 | try: 70 | yield Path(directory) 71 | finally: 72 | try: 73 | os.kill(int(open(os.path.join(directory, 'supervisor', 'supervisord.pid')).read()), signal.SIGTERM) 74 | except Exception: 75 | pass 76 | shutil.rmtree(directory) 77 | instance_name = os.path.basename(directory) 78 | unit_paths = glob.glob(os.path.join(unit_path, f"galaxy-{instance_name}*")) 79 | if unit_paths: 80 | units = list(map(os.path.basename, unit_paths)) 81 | try: 82 | subprocess.check_call(["systemctl", "--user", "stop", *units]) 83 | list(map(os.unlink, unit_paths)) 84 | subprocess.check_call(["systemctl", "--user", "daemon-reload"]) 85 | except Exception: 86 | subprocess.check_call(["systemctl", "--user", "list-units", "--all", "galaxy*"]) 87 | try: 88 | # unfortunately these aren't created in /run 89 | os.unlink(os.path.expanduser(f"~/.config/systemd/user/multi-user.target.wants/galaxy-{instance_name}.target")) 90 | except Exception: 91 | pass 92 | 93 | 94 | @pytest.fixture 95 | def default_config_manager(state_dir): 96 | with config_manager.config_manager(state_dir=state_dir) as cm: 97 | yield cm 98 | 99 | 100 | @pytest.fixture() 101 | def job_conf(request, galaxy_root_dir): 102 | conf = yaml.safe_load(request.param) 103 | ext = "xml" if isinstance(conf, str) else "yml" 104 | job_conf_path = galaxy_root_dir / 'config' / f'job_conf.{ext}' 105 | with open(job_conf_path, 'w') as jcfh: 106 | jcfh.write(request.param) 107 | yield job_conf_path 108 | os.unlink(job_conf_path) 109 | 110 | 111 | @pytest.fixture() 112 | def free_port(): 113 | # Inspired by https://gist.github.com/bertjwregeer/0be94ced48383a42e70c3d9fff1f4ad0 114 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 115 | s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 116 | s.bind(("localhost", 0)) 117 | portnum = s.getsockname()[1] 118 | s.close() 119 | return portnum 120 | 121 | 122 | another_free_port = free_port 123 | 124 | 125 | @pytest.fixture() 126 | def startup_config(state_dir, galaxy_virtualenv, free_port): 127 | return { 128 | 'gravity': { 129 | 'service_command_style': 'direct', 130 | 'virtualenv': galaxy_virtualenv, 131 | 'gunicorn': { 132 | 'bind': f'localhost:{free_port}'} 133 | }, 134 | 'galaxy': { 135 | 'conda_auto_init': False 136 | } 137 | } 138 | 139 | 140 | @pytest.fixture() 141 | def reports_config(galaxy_root_dir, galaxy_virtualenv, free_port): 142 | return { 143 | 'gravity': { 144 | 'service_command_style': 'direct', 145 | 'virtualenv': galaxy_virtualenv, 146 | 'gunicorn': {'enable': False}, 147 | 'celery': { 148 | 'enable': False, 149 | 'enable_beat': False, 150 | }, 151 | 'reports': { 152 | 'enable': True, 153 | 'bind': f'localhost:{free_port}', 154 | 'config_file': str(galaxy_root_dir / "config" / "reports.yml.sample"), 155 | } 156 | } 157 | } 158 | 159 | 160 | @pytest.fixture() 161 | def non_default_config(): 162 | return { 163 | 'galaxy': None, 164 | 'gravity': { 165 | 'service_command_style': 'direct', 166 | 'gunicorn': { 167 | 'bind': 'localhost:8081', 168 | 'environment': {'FOO': 'foo'} 169 | }, 170 | 'celery': { 171 | 'concurrency': 4 172 | } 173 | } 174 | } 175 | 176 | 177 | @pytest.fixture 178 | def gxit_config(state_dir, free_port, another_free_port, process_manager_name): 179 | config_yaml = GXIT_CONFIG.format( 180 | gxit_port=another_free_port, 181 | gx_port=free_port, 182 | process_manager_name=process_manager_name, 183 | instance_name=os.path.basename(state_dir), 184 | ) 185 | return yaml.safe_load(config_yaml) 186 | 187 | 188 | @pytest.fixture 189 | def tusd_config(state_dir, startup_config, free_port, another_free_port, process_manager_name): 190 | startup_config["gravity"] = { 191 | "process_manager": process_manager_name, 192 | "service_command_style": "direct", 193 | "instance_name": os.path.basename(state_dir), 194 | "tusd": {"enable": True, "port": another_free_port, "upload_dir": "/tmp"}} 195 | startup_config["galaxy"]["galaxy_infrastructure_url"] = f"http://localhost:{free_port}" 196 | return startup_config 197 | 198 | 199 | @pytest.fixture 200 | def gxit_startup_config(galaxy_virtualenv, gxit_config): 201 | gxit_config['gravity']['virtualenv'] = galaxy_virtualenv 202 | return gxit_config 203 | 204 | 205 | @pytest.fixture 206 | def tusd_startup_config(galaxy_virtualenv, tusd_config, free_port): 207 | tusd_config['gravity']['gunicorn'] = {'bind': f'localhost:{free_port}'} 208 | tusd_config['gravity']['virtualenv'] = galaxy_virtualenv 209 | return tusd_config 210 | 211 | 212 | @pytest.fixture(scope="session") 213 | def galaxy_virtualenv(galaxy_root_dir): 214 | virtual_env_dir = str(TEST_DIR / "galaxy_venv") 215 | os.environ['GALAXY_VIRTUAL_ENV'] = virtual_env_dir 216 | subprocess.run( 217 | str(galaxy_root_dir / "scripts/common_startup.sh"), 218 | env={ 219 | "GALAXY_SKIP_CLIENT_BUILD": "1", 220 | "GALAXY_VIRTUAL_ENV": virtual_env_dir, 221 | "PATH": os.getenv("PATH"), 222 | }, 223 | cwd=str(galaxy_root_dir) 224 | ) 225 | return virtual_env_dir 226 | -------------------------------------------------------------------------------- /tests/test_config_manager.py: -------------------------------------------------------------------------------- 1 | import json 2 | from pathlib import Path 3 | 4 | import pytest 5 | 6 | from gravity import config_manager 7 | from gravity.settings import Settings 8 | from gravity.state import GracefulMethod 9 | 10 | 11 | def test_load_defaults(galaxy_yml, galaxy_root_dir, state_dir, default_config_manager): 12 | default_config_manager.load_config_file(str(galaxy_yml)) 13 | config = default_config_manager.get_config() 14 | default_settings = Settings() 15 | assert config.process_manager == 'supervisor' 16 | assert config.instance_name == default_settings.instance_name 17 | assert config.services != [] 18 | assert config.app_server == 'gunicorn' 19 | assert Path(config.log_dir) == Path(state_dir) / 'log' 20 | assert Path(config.galaxy_root) == galaxy_root_dir 21 | gunicorn_settings = config.get_service('gunicorn').settings 22 | assert gunicorn_settings['bind'] == default_settings.gunicorn.bind 23 | assert gunicorn_settings['workers'] == default_settings.gunicorn.workers 24 | assert gunicorn_settings['timeout'] == default_settings.gunicorn.timeout 25 | assert gunicorn_settings['extra_args'] == default_settings.gunicorn.extra_args 26 | assert gunicorn_settings['preload'] is True 27 | celery_settings = config.get_service('celery').settings 28 | assert celery_settings == default_settings.celery.dict() 29 | with pytest.raises(IndexError): 30 | config.get_service('tusd') 31 | 32 | 33 | def test_preload_default(galaxy_yml, default_config_manager): 34 | app_server = 'unicornherder' 35 | galaxy_yml.write(json.dumps({ 36 | 'galaxy': None, 37 | 'gravity': { 38 | 'app_server': app_server 39 | } 40 | })) 41 | default_config_manager.load_config_file(str(galaxy_yml)) 42 | config = default_config_manager.get_config() 43 | unicornherder_settings = config.get_service('unicornherder').settings 44 | assert unicornherder_settings['preload'] is False 45 | 46 | 47 | def test_load_non_default(galaxy_yml, default_config_manager, non_default_config): 48 | if default_config_manager.instance_count == 0: 49 | galaxy_yml.write(json.dumps(non_default_config)) 50 | default_config_manager.load_config_file(str(galaxy_yml)) 51 | config = default_config_manager.get_config() 52 | gunicorn_settings = config.get_service('gunicorn').settings 53 | assert gunicorn_settings['bind'] == non_default_config['gravity']['gunicorn']['bind'] 54 | assert gunicorn_settings['environment'] == non_default_config['gravity']['gunicorn']['environment'] 55 | default_settings = Settings() 56 | assert gunicorn_settings['workers'] == default_settings.gunicorn.workers 57 | celery_settings = config.get_service('celery').settings 58 | assert celery_settings['concurrency'] == non_default_config['gravity']['celery']['concurrency'] 59 | 60 | 61 | def test_split_config(galaxy_yml, galaxy_root_dir, default_config_manager, non_default_config): 62 | default_config_file = str(galaxy_root_dir / "config" / "galaxy.yml.sample") 63 | non_default_config['gravity']['galaxy_config_file'] = default_config_file 64 | del non_default_config['galaxy'] 65 | galaxy_yml.write(json.dumps(non_default_config)) 66 | default_config_manager.load_config_file(str(galaxy_yml)) 67 | test_load_non_default(galaxy_yml, default_config_manager, non_default_config) 68 | config = default_config_manager.get_config() 69 | assert config.gravity_config_file == str(galaxy_yml) 70 | assert config.galaxy_config_file == default_config_file 71 | 72 | 73 | def test_auto_load_env_var(galaxy_yml, default_config_manager, monkeypatch): 74 | monkeypatch.setenv("GALAXY_CONFIG_FILE", str(galaxy_yml)) 75 | assert default_config_manager.instance_count == 0 76 | default_config_manager.auto_load() 77 | assert default_config_manager.is_loaded(galaxy_yml) 78 | 79 | 80 | def test_auto_load_root_dir(galaxy_root_dir, monkeypatch): 81 | monkeypatch.chdir(galaxy_root_dir) 82 | galaxy_yml_sample = galaxy_root_dir / "config" / "galaxy.yml.sample" 83 | with config_manager.config_manager() as cm: 84 | assert cm.instance_count == 1 85 | assert cm.is_loaded(galaxy_yml_sample) 86 | galaxy_yml = galaxy_root_dir / "config" / "galaxy.yml" 87 | galaxy_yml_sample.copy(galaxy_yml) 88 | with config_manager.config_manager() as cm: 89 | assert cm.instance_count == 1 90 | assert cm.is_loaded(galaxy_yml) 91 | galaxy_yml.remove() 92 | 93 | 94 | def test_gunicorn_graceful_method_preload(galaxy_yml, default_config_manager): 95 | default_config_manager.load_config_file(str(galaxy_yml)) 96 | config = default_config_manager.get_config() 97 | graceful_method = config.get_service('gunicorn').graceful_method 98 | assert graceful_method == GracefulMethod.DEFAULT 99 | 100 | 101 | def test_gunicorn_graceful_method_no_preload(galaxy_yml, default_config_manager): 102 | galaxy_yml.write(json.dumps( 103 | {'galaxy': None, 'gravity': { 104 | 'gunicorn': {'preload': False}}} 105 | )) 106 | default_config_manager.load_config_file(str(galaxy_yml)) 107 | config = default_config_manager.get_config() 108 | graceful_method = config.get_service('gunicorn').graceful_method 109 | assert graceful_method == GracefulMethod.SIGHUP 110 | 111 | 112 | # TODO: tests for switching process managers between supervisor and systemd 113 | -------------------------------------------------------------------------------- /tests/test_operations.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import re 4 | import subprocess 5 | import time 6 | 7 | import pytest 8 | import requests 9 | from click.testing import CliRunner 10 | from yaml import safe_load 11 | 12 | from gravity.cli import galaxyctl 13 | from gravity.state import CELERY_BEAT_DB_FILENAME 14 | 15 | STARTUP_TIMEOUT = 30 16 | CELERY_BEAT_TIMEOUT = 10 17 | # celery.beat.PersistentScheduler uses shelve, which can append a suffix based on which db backend is used 18 | CELERY_BEAT_DB_FILENAMES = list(map(lambda ext: CELERY_BEAT_DB_FILENAME + ext, ('', '.db', '.dat', '.bak', '.dir'))) 19 | 20 | 21 | def log_for_service(state_dir, process_manager_name, start_time, service_name, instance_name=None): 22 | if process_manager_name == "systemd": 23 | # instance_name should never be none in the systemd case 24 | log_name = f"galaxy-{instance_name}-{service_name}" 25 | cmd = f"journalctl --user --no-pager --since=@{start_time} --unit={log_name}.service".split() 26 | return subprocess.check_output(cmd, text=True) 27 | else: 28 | # could probably just glob here 29 | if instance_name is not None: 30 | log_name = f"{instance_name}_{service_name}_{service_name}.log" 31 | else: 32 | log_name = f"{service_name}.log" 33 | path = state_dir / "log" / log_name 34 | with open(path) as fh: 35 | return fh.read() 36 | 37 | 38 | def wait_for_startup(state_dir, free_port, prefix="/", path="/api/version", service_name="gunicorn", 39 | process_manager_name="supervisor", start_time=None, instance_name=None): 40 | for _ in range(STARTUP_TIMEOUT * 4): 41 | try: 42 | requests.get(f"http://localhost:{free_port}{prefix.rstrip('/')}{path}").raise_for_status() 43 | return True, "" 44 | except Exception: 45 | time.sleep(0.25) 46 | return False, log_for_service(state_dir, process_manager_name, start_time, service_name, instance_name=instance_name) 47 | 48 | 49 | def wait_for_gxit_proxy(state_dir, process_manager_name, start_time): 50 | instance_name = os.path.basename(state_dir) 51 | for _ in range(STARTUP_TIMEOUT * 4): 52 | startup_logs = log_for_service(state_dir, process_manager_name, start_time, service_name="gx-it-proxy", instance_name=instance_name) 53 | if 'Watching path' in startup_logs: 54 | return True, "" 55 | time.sleep(0.25) 56 | return False, startup_logs 57 | 58 | 59 | def wait_for_any_path(paths, timeout): 60 | for _ in range(timeout * 4): 61 | try: 62 | assert any(map(lambda x: x.exists(), paths)) 63 | return True 64 | except AssertionError: 65 | time.sleep(0.25) 66 | return False 67 | 68 | 69 | def start_instance(state_dir, galaxy_yml, free_port, process_manager_name="supervisor", instance_name=None): 70 | runner = CliRunner() 71 | start_time = time.time() 72 | result = runner.invoke(galaxyctl, ['--config-file', str(galaxy_yml), 'start']) 73 | assert result.exit_code == 0, result.output 74 | if process_manager_name == "systemd": 75 | gunicorn_name = f"galaxy-{instance_name}-gunicorn" 76 | output = subprocess.check_output(f"systemctl --user status {gunicorn_name}.service".split(), text=True) 77 | assert f"● {gunicorn_name}.service" in output 78 | else: 79 | assert re.search(r"gunicorn\s*STARTING", result.output) 80 | startup_done, startup_logs = wait_for_startup(state_dir, free_port, process_manager_name=process_manager_name, 81 | start_time=start_time, instance_name=instance_name) 82 | assert startup_done is True, f"Startup failed. Application startup logs:\n {startup_logs}" 83 | 84 | 85 | def supervisor_service_pids(runner, galaxy_yml, instance_name): 86 | result = runner.invoke(galaxyctl, ['--config-file', str(galaxy_yml), 'status']) 87 | assert result.exit_code == 0, result.output 88 | start_time = time.time() 89 | while 'STARTING' in result.output: 90 | assert (time.time() - start_time) < STARTUP_TIMEOUT, result.output 91 | time.sleep(1) 92 | result = runner.invoke(galaxyctl, ['--config-file', str(galaxy_yml), 'status']) 93 | assert result.exit_code == 0, result.output 94 | pids = {} 95 | for line in result.output.splitlines(): 96 | line_a = line.split() 97 | service = line_a[0].split(":")[-1] 98 | pids[service] = line_a[3].rstrip(",") 99 | return pids 100 | 101 | 102 | def systemd_service_pids(runner, galaxy_yml, instance_name): 103 | pids = {} 104 | units = subprocess.check_output(f"systemctl --user list-units --plain --no-legend galaxy-{instance_name}-*".split(), text=True) 105 | for unit_line in units.splitlines(): 106 | assert 'active' in unit_line, unit_line 107 | assert 'running' in unit_line, unit_line 108 | unit = unit_line.split()[0] 109 | output = subprocess.check_output(f"systemctl --user show --property=MainPID {unit}".split(), text=True) 110 | assert 'MainPID=' in output, output 111 | pid = output.split("=")[-1] 112 | assert pid != "0", output 113 | service = unit.replace(f"galaxy-{instance_name}-", "").replace(".service", "") 114 | pids[service] = pid 115 | return pids 116 | 117 | 118 | @pytest.mark.parametrize('process_manager_name', ['supervisor', 'systemd']) 119 | def test_cmd_start(state_dir, galaxy_yml, startup_config, free_port, process_manager_name): 120 | # TODO: test service_command_style = gravity, doesn't work when you're using CliRunner, which just imports the cli 121 | # rather than the entry point existing on the filesystem somewhere. 122 | instance_name = os.path.basename(state_dir) 123 | startup_config["gravity"]["process_manager"] = process_manager_name 124 | startup_config["gravity"]["instance_name"] = instance_name 125 | galaxy_yml.write(json.dumps(startup_config)) 126 | runner = CliRunner() 127 | result = runner.invoke(galaxyctl, ['--config-file', str(galaxy_yml), 'update']) 128 | assert result.exit_code == 0, result.output 129 | start_instance(state_dir, galaxy_yml, free_port, process_manager_name, instance_name=instance_name) 130 | result = runner.invoke(galaxyctl, ['--config-file', str(galaxy_yml), 'status']) 131 | celery_beat_db_paths = list(map(lambda f: state_dir / f, CELERY_BEAT_DB_FILENAMES)) 132 | celery_beat_db_exists = wait_for_any_path(celery_beat_db_paths, CELERY_BEAT_TIMEOUT) 133 | assert celery_beat_db_exists is True, "celery-beat failed to write db. State dir contents:\n" \ 134 | f"{os.listdir(state_dir)}" 135 | result = runner.invoke(galaxyctl, ['--config-file', str(galaxy_yml), 'stop']) 136 | assert result.exit_code == 0, result.output 137 | if process_manager_name == "supervisor": 138 | assert "All processes stopped, supervisord will exit" in result.output 139 | else: 140 | assert "" == result.output 141 | 142 | 143 | def test_cmd_start_reports(state_dir, galaxy_yml, reports_config, free_port): 144 | galaxy_yml.write(json.dumps(reports_config)) 145 | runner = CliRunner() 146 | result = runner.invoke(galaxyctl, ['--config-file', str(galaxy_yml), 'update']) 147 | assert result.exit_code == 0, result.output 148 | result = runner.invoke(galaxyctl, ['--config-file', str(galaxy_yml), 'start']) 149 | assert re.search(r"reports\s*STARTING", result.output) 150 | assert result.exit_code == 0, result.output 151 | startup_done, startup_logs = wait_for_startup(state_dir, free_port, path="/", service_name="reports") 152 | assert startup_done is True, f"Startup failed. Application startup logs:\n {startup_logs}" 153 | result = runner.invoke(galaxyctl, ['--config-file', str(galaxy_yml), 'stop']) 154 | assert result.exit_code == 0, result.output 155 | assert "All processes stopped, supervisord will exit" in result.output 156 | 157 | 158 | @pytest.mark.parametrize('process_manager_name', ['supervisor', 'systemd']) 159 | def test_cmd_start_with_gxit(state_dir, galaxy_yml, gxit_startup_config, free_port, process_manager_name): 160 | instance_name = gxit_startup_config["gravity"]["instance_name"] 161 | galaxy_yml.write(json.dumps(gxit_startup_config)) 162 | runner = CliRunner() 163 | result = runner.invoke(galaxyctl, ['--config-file', str(galaxy_yml), 'update']) 164 | assert result.exit_code == 0, result.output 165 | start_time = time.time() 166 | start_instance(state_dir, galaxy_yml, free_port, process_manager_name, instance_name=instance_name) 167 | result = runner.invoke(galaxyctl, ['--config-file', str(galaxy_yml), 'status']) 168 | assert result.exit_code == 0, result.output 169 | startup_done, startup_logs = wait_for_gxit_proxy(state_dir, process_manager_name, start_time) 170 | assert startup_done is True, f"gx-it-proxy startup failed. gx-it-proxy startup logs:\n {startup_logs}" 171 | 172 | 173 | @pytest.mark.parametrize('process_manager_name', ['supervisor', 'systemd']) 174 | def test_cmd_graceful(state_dir, galaxy_yml, tusd_startup_config, free_port, process_manager_name): 175 | service_pid_func = globals()[process_manager_name + "_service_pids"] 176 | instance_name = tusd_startup_config["gravity"]["instance_name"] 177 | # disable preload, causes graceful to HUP 178 | tusd_startup_config["gravity"]["gunicorn"]["preload"] = False 179 | # make a fake tusd 180 | tusd_path = state_dir / "tusd" 181 | tusd_path.write_text("#!/bin/sh\nsleep 60\n") 182 | tusd_path.chmod(0o755) 183 | tusd_startup_config["gravity"]["tusd"]["tusd_path"] = str(tusd_path) 184 | galaxy_yml.write(json.dumps(tusd_startup_config)) 185 | runner = CliRunner() 186 | result = runner.invoke(galaxyctl, ['--config-file', str(galaxy_yml), 'update']) 187 | assert result.exit_code == 0, result.output 188 | start_instance(state_dir, galaxy_yml, free_port, process_manager_name, instance_name=instance_name) 189 | before_pids = service_pid_func(runner, galaxy_yml, instance_name) 190 | result = runner.invoke(galaxyctl, ['--config-file', str(galaxy_yml), 'graceful']) 191 | assert result.exit_code == 0, result.output 192 | after_pids = service_pid_func(runner, galaxy_yml, instance_name) 193 | assert before_pids['gunicorn'] == after_pids['gunicorn'], f"{before_pids}; {after_pids}" 194 | assert before_pids['celery'] != after_pids['celery'], f"{before_pids}; {after_pids}" 195 | assert before_pids['celery-beat'] != after_pids['celery-beat'], f"{before_pids}; {after_pids}" 196 | assert before_pids['tusd'] == after_pids['tusd'], f"{before_pids}; {after_pids}" 197 | 198 | 199 | def test_cmd_restart_with_update(state_dir, galaxy_yml, startup_config, free_port): 200 | galaxy_yml.write(json.dumps(startup_config)) 201 | runner = CliRunner() 202 | result = runner.invoke(galaxyctl, ['--config-file', str(galaxy_yml), 'update']) 203 | assert result.exit_code == 0, result.output 204 | start_instance(state_dir, galaxy_yml, free_port) 205 | # change prefix 206 | prefix = '/galaxypf' 207 | startup_config['galaxy']['galaxy_url_prefix'] = prefix 208 | galaxy_yml.write(json.dumps(startup_config)) 209 | result = runner.invoke(galaxyctl, ['--config-file', str(galaxy_yml), 'restart']) 210 | assert result.exit_code == 0, result.output 211 | startup_done, startup_logs = wait_for_startup(state_dir=state_dir, free_port=free_port, prefix=prefix) 212 | assert startup_done is True, f"Startup failed. Application startup logs:\n {startup_logs}" 213 | 214 | 215 | def test_cmd_show(state_dir, galaxy_yml): 216 | runner = CliRunner() 217 | result = runner.invoke(galaxyctl, ['--config-file', str(galaxy_yml), 'show']) 218 | assert result.exit_code == 0, result.output 219 | details = safe_load(result.output) 220 | assert details['galaxy_config_file'] == str(galaxy_yml) 221 | assert details['instance_name'] == '_default_' 222 | 223 | 224 | def test_cmd_list(state_dir, galaxy_yml): 225 | runner = CliRunner() 226 | result = runner.invoke(galaxyctl, ['--config-file', str(galaxy_yml), 'list']) 227 | assert result.exit_code == 0, result.output 228 | assert result.output.startswith("INSTANCE NAME") 229 | assert str(galaxy_yml) in result.output 230 | -------------------------------------------------------------------------------- /tests/test_settings.py: -------------------------------------------------------------------------------- 1 | from io import StringIO 2 | import json 3 | 4 | from gravity.settings import Settings 5 | from gravity.util import settings_to_sample 6 | from yaml import safe_load 7 | 8 | 9 | def test_schema_json(): 10 | schema = Settings.schema_json(indent=2) 11 | assert "Configuration for Gravity process manager" in json.loads(schema)["description"] 12 | 13 | 14 | def test_extra_fields_allowed(): 15 | s = Settings(extra=1) 16 | assert not hasattr(s, "extra") 17 | 18 | 19 | def test_defaults_loaded(): 20 | settings = Settings() 21 | assert settings.gunicorn.bind == "localhost:8080" 22 | 23 | 24 | def test_defaults_override_constructor(): 25 | settings = Settings(**{"gunicorn": {"bind": "localhost:8081"}}) 26 | assert settings.gunicorn.bind == "localhost:8081" 27 | 28 | 29 | def test_defaults_override_env_var(monkeypatch): 30 | monkeypatch.setenv("GRAVITY_GUNICORN.BIND", "localhost:8081") 31 | settings = Settings() 32 | assert settings.gunicorn.bind == "localhost:8081" 33 | 34 | 35 | def test_schema_to_sample(): 36 | sample = settings_to_sample() 37 | settings = Settings(**safe_load(StringIO(sample))["gravity"]) 38 | default_settings = Settings() 39 | assert settings.dict() == default_settings.dict() 40 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | source_dir = gravity 3 | test_dir = tests 4 | 5 | [flake8] 6 | max-line-length = 160 7 | exclude = .venv,.git,tests/galaxy.git,.tox,tests/galaxy_venv 8 | 9 | [testenv] 10 | commands = 11 | lint: flake8 12 | test: coverage run -m pytest {posargs:-vv} 13 | test: coverage xml 14 | deps = 15 | lint: flake8 16 | test: pytest 17 | test: pytest-timeout 18 | test: coverage 19 | test: requests 20 | passenv = 21 | GRAVITY_TEST_GALAXY_BRANCH 22 | GRAVITY_SYSTEMCTL_EXTRA_ARGS 23 | DBUS_SESSION_BUS_ADDRESS 24 | XDG_RUNTIME_DIR 25 | --------------------------------------------------------------------------------