├── tests ├── __init__.py ├── cli │ ├── __init__.py │ ├── projects │ │ ├── pyproject │ │ │ ├── mysrc │ │ │ │ ├── __init__.py │ │ │ │ ├── subpkg │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── plugins.py │ │ │ │ └── plugins.py │ │ │ └── pyproject.toml │ │ ├── setupcfg │ │ │ ├── mysrc │ │ │ │ ├── __init__.py │ │ │ │ ├── subpkg │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── plugins.py │ │ │ │ └── plugins.py │ │ │ ├── setup.py │ │ │ └── setup.cfg │ │ └── manual_build_mode │ │ │ ├── mysrc │ │ │ ├── __init__.py │ │ │ ├── subpkg │ │ │ │ ├── __init__.py │ │ │ │ └── plugins.py │ │ │ └── plugins.py │ │ │ ├── plux.ini │ │ │ └── pyproject.toml │ ├── test_discover.py │ └── test_entrypoints.py ├── plugins │ ├── __init__.py │ ├── invalid_module.py │ └── sample_plugins.py ├── runtime │ ├── __init__.py │ ├── test_cache.py │ └── test_resolve.py ├── conftest.py ├── test_metadata.py ├── test_entrypoint.py ├── test_discovery.py ├── test_function_plugin.py ├── test_listener.py └── test_manager.py ├── plux ├── cli │ ├── __init__.py │ └── cli.py ├── core │ ├── __init__.py │ ├── entrypoint.py │ └── plugin.py ├── build │ ├── hatchling.py │ ├── __init__.py │ ├── project.py │ ├── index.py │ ├── config.py │ ├── discovery.py │ └── setuptools.py ├── __main__.py ├── runtime │ ├── __init__.py │ ├── resolve.py │ ├── filter.py │ ├── metadata.py │ ├── cache.py │ └── manager.py └── __init__.py ├── CODEOWNERS ├── docs └── plux-architecture.png ├── plugin ├── discovery.py ├── metadata.py ├── setuptools.py ├── entrypoint.py ├── core.py └── __init__.py ├── .github └── workflows │ └── build.yml ├── Makefile ├── pyproject.toml ├── .gitignore ├── LICENSE └── README.md /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /plux/cli/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/cli/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/plugins/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/runtime/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/cli/projects/pyproject/mysrc/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/cli/projects/setupcfg/mysrc/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/cli/projects/pyproject/mysrc/subpkg/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/cli/projects/setupcfg/mysrc/subpkg/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/cli/projects/manual_build_mode/mysrc/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /plux/core/__init__.py: -------------------------------------------------------------------------------- 1 | """Core concepts and API of Plux.""" 2 | -------------------------------------------------------------------------------- /tests/cli/projects/manual_build_mode/mysrc/subpkg/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | # by default, add all repo maintainers as reviewers 2 | * @thrau 3 | -------------------------------------------------------------------------------- /docs/plux-architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/localstack/plux/HEAD/docs/plux-architecture.png -------------------------------------------------------------------------------- /tests/cli/projects/setupcfg/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from setuptools import setup 3 | 4 | setup() 5 | -------------------------------------------------------------------------------- /plux/build/hatchling.py: -------------------------------------------------------------------------------- 1 | from plux.build.project import Project 2 | 3 | 4 | class HatchlingProject(Project): 5 | # TODO: implement me 6 | pass 7 | -------------------------------------------------------------------------------- /tests/cli/projects/manual_build_mode/plux.ini: -------------------------------------------------------------------------------- 1 | [plux.test.plugins] 2 | mynestedplugin = mysrc.subpkg.plugins:MyNestedPlugin 3 | myplugin = mysrc.plugins:MyPlugin 4 | 5 | -------------------------------------------------------------------------------- /tests/cli/projects/setupcfg/mysrc/plugins.py: -------------------------------------------------------------------------------- 1 | from plugin import Plugin 2 | 3 | 4 | class MyPlugin(Plugin): 5 | namespace = "plux.test.plugins" 6 | name = "myplugin" 7 | -------------------------------------------------------------------------------- /tests/cli/projects/pyproject/mysrc/plugins.py: -------------------------------------------------------------------------------- 1 | from plugin import Plugin 2 | 3 | 4 | class MyPlugin(Plugin): 5 | namespace = "plux.test.plugins" 6 | name = "myplugin" 7 | -------------------------------------------------------------------------------- /tests/cli/projects/manual_build_mode/mysrc/plugins.py: -------------------------------------------------------------------------------- 1 | from plugin import Plugin 2 | 3 | 4 | class MyPlugin(Plugin): 5 | namespace = "plux.test.plugins" 6 | name = "myplugin" 7 | -------------------------------------------------------------------------------- /plux/build/__init__.py: -------------------------------------------------------------------------------- 1 | """Code to hook plux into the build phase of various build tools. Currently, only setuptools is supported, 2 | but we may add more in the future (poetry, ...).""" 3 | -------------------------------------------------------------------------------- /tests/cli/projects/pyproject/mysrc/subpkg/plugins.py: -------------------------------------------------------------------------------- 1 | from plugin import Plugin 2 | 3 | 4 | class MyNestedPlugin(Plugin): 5 | namespace = "plux.test.plugins" 6 | name = "mynestedplugin" 7 | -------------------------------------------------------------------------------- /tests/cli/projects/setupcfg/mysrc/subpkg/plugins.py: -------------------------------------------------------------------------------- 1 | from plugin import Plugin 2 | 3 | 4 | class MyNestedPlugin(Plugin): 5 | namespace = "plux.test.plugins" 6 | name = "mynestedplugin" 7 | -------------------------------------------------------------------------------- /tests/cli/projects/manual_build_mode/mysrc/subpkg/plugins.py: -------------------------------------------------------------------------------- 1 | from plugin import Plugin 2 | 3 | 4 | class MyNestedPlugin(Plugin): 5 | namespace = "plux.test.plugins" 6 | name = "mynestedplugin" 7 | -------------------------------------------------------------------------------- /plux/__main__.py: -------------------------------------------------------------------------------- 1 | """ 2 | A plux frontend. 3 | """ 4 | 5 | from plux.cli import cli 6 | 7 | 8 | def main(argv=None): 9 | cli.main(argv) 10 | 11 | 12 | if __name__ == "__main__": 13 | main() 14 | -------------------------------------------------------------------------------- /plux/runtime/__init__.py: -------------------------------------------------------------------------------- 1 | """This module contains the API and machinery for resolving and loading Plugins from entrypoints at runtime. It is 2 | build-tool independent and only relies on importlib and plux internals.""" 3 | -------------------------------------------------------------------------------- /plugin/discovery.py: -------------------------------------------------------------------------------- 1 | from plux.build.discovery import ModuleScanningPluginFinder 2 | from plux.build.setuptools import PackagePathPluginFinder 3 | 4 | __all__ = [ 5 | "PackagePathPluginFinder", 6 | "ModuleScanningPluginFinder", 7 | ] 8 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from plux.build.discovery import ModuleScanningPluginFinder 4 | from tests.plugins import sample_plugins 5 | 6 | 7 | @pytest.fixture 8 | def sample_plugin_finder(): 9 | finder = ModuleScanningPluginFinder(modules=[sample_plugins]) 10 | yield finder 11 | -------------------------------------------------------------------------------- /plugin/metadata.py: -------------------------------------------------------------------------------- 1 | from plux.runtime.metadata import ( 2 | Distribution, 3 | metadata_packages_distributions, 4 | packages_distributions, 5 | resolve_distribution_information, 6 | ) 7 | 8 | __all__ = [ 9 | "resolve_distribution_information", 10 | "metadata_packages_distributions", 11 | "packages_distributions", 12 | "Distribution", 13 | ] 14 | -------------------------------------------------------------------------------- /tests/plugins/invalid_module.py: -------------------------------------------------------------------------------- 1 | # this module fails when importing to test the fault tolerance of the plugin discovery mechanism 2 | from plux import Plugin 3 | 4 | 5 | def fail(): 6 | raise ValueError("this is an expected exception") 7 | 8 | 9 | class CannotBeLoadedPlugin(Plugin): 10 | namespace = "namespace_2" 11 | name = "cannot-be-loaded" 12 | 13 | 14 | fail() 15 | -------------------------------------------------------------------------------- /plugin/setuptools.py: -------------------------------------------------------------------------------- 1 | from plux.build.setuptools import ( 2 | entry_points_from_egg_info, 3 | get_plux_json_path, 4 | load_plux_entrypoints, 5 | plugins, 6 | update_entrypoints, 7 | ) 8 | 9 | __all__ = [ 10 | "plugins", 11 | "load_plux_entrypoints", 12 | "get_plux_json_path", 13 | "update_entrypoints", 14 | "entry_points_from_egg_info", 15 | ] 16 | -------------------------------------------------------------------------------- /tests/test_metadata.py: -------------------------------------------------------------------------------- 1 | from plux import PluginSpec 2 | from plux.runtime.metadata import resolve_distribution_information 3 | 4 | 5 | def test_resolve_distribution_information(): 6 | import pytest 7 | 8 | # fake a plugin spec and use pytest as test object 9 | fake_plugin_spec = PluginSpec("foo", "bar", pytest.fixture) 10 | dist = resolve_distribution_information(fake_plugin_spec) 11 | assert dist.metadata["Name"] == "pytest" 12 | assert dist.metadata["License"] == "MIT" 13 | -------------------------------------------------------------------------------- /plugin/entrypoint.py: -------------------------------------------------------------------------------- 1 | """ 2 | Deprecated bindings, use plux imports instead, but you shouldn't use the internals in the first place. 3 | """ 4 | 5 | from plux.build.setuptools import find_plugins 6 | from plux.core.entrypoint import ( 7 | EntryPoint, 8 | EntryPointDict, 9 | spec_to_entry_point, 10 | to_entry_point_dict, 11 | ) 12 | 13 | __all__ = [ 14 | "find_plugins", 15 | "spec_to_entry_point", 16 | "EntryPointDict", 17 | "EntryPoint", 18 | "to_entry_point_dict", 19 | ] 20 | -------------------------------------------------------------------------------- /tests/cli/projects/setupcfg/setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = test-project 3 | version = 0.1.0 4 | author = LocalStack Team 5 | author_email = info@localstack.cloud 6 | description = A test project to test plux with setup.cfg projects 7 | license = Proprietary 8 | classifiers = 9 | Programming Language :: Python :: 3 10 | 11 | [options] 12 | python_requires = >=3.8 13 | zip_safe = False 14 | packages = find: 15 | 16 | # Only contains dependencies necessary to use the CLI 17 | install_requires = 18 | plux 19 | 20 | [options.packages.find] 21 | exclude = 22 | tests* 23 | -------------------------------------------------------------------------------- /plugin/core.py: -------------------------------------------------------------------------------- 1 | from plux.core.plugin import ( 2 | FunctionPlugin, 3 | Plugin, 4 | PluginDisabled, 5 | PluginException, 6 | PluginFactory, 7 | PluginFinder, 8 | PluginLifecycleListener, 9 | PluginSpec, 10 | PluginSpecResolver, 11 | PluginType, 12 | plugin, 13 | ) 14 | 15 | __all__ = [ 16 | "FunctionPlugin", 17 | "Plugin", 18 | "PluginDisabled", 19 | "PluginException", 20 | "PluginFactory", 21 | "PluginFinder", 22 | "PluginLifecycleListener", 23 | "PluginSpec", 24 | "PluginSpecResolver", 25 | "PluginType", 26 | "plugin", 27 | ] 28 | -------------------------------------------------------------------------------- /tests/cli/projects/pyproject/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools", "wheel", "plux"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "test-project" 7 | authors = [ 8 | { name = "LocalStack Contributors", email = "info@localstack.cloud" } 9 | ] 10 | version = "0.1.0" 11 | description = "A test project to test plux with pyproject.toml projects" 12 | dependencies = [ 13 | "plux", 14 | ] 15 | requires-python = ">=3.8" 16 | classifiers = [ 17 | "Programming Language :: Python :: 3", 18 | ] 19 | 20 | [tool.setuptools] 21 | include-package-data = false 22 | 23 | [tool.setuptools.packages.find] 24 | include = ["mysrc*"] 25 | -------------------------------------------------------------------------------- /plugin/__init__.py: -------------------------------------------------------------------------------- 1 | from plux import __version__ as _version 2 | from plux.core.plugin import ( 3 | FunctionPlugin, 4 | Plugin, 5 | PluginDisabled, 6 | PluginException, 7 | PluginFinder, 8 | PluginLifecycleListener, 9 | PluginSpec, 10 | PluginSpecResolver, 11 | PluginType, 12 | plugin, 13 | ) 14 | from plux.runtime.manager import PluginManager 15 | 16 | name = "plugin" 17 | 18 | __version__ = _version 19 | 20 | __all__ = [ 21 | "__version__", 22 | "FunctionPlugin", 23 | "Plugin", 24 | "PluginSpec", 25 | "PluginType", 26 | "PluginLifecycleListener", 27 | "PluginFinder", 28 | "PluginManager", 29 | "PluginSpecResolver", 30 | "PluginException", 31 | "PluginDisabled", 32 | "plugin", 33 | ] 34 | -------------------------------------------------------------------------------- /tests/runtime/test_cache.py: -------------------------------------------------------------------------------- 1 | from plux.runtime.cache import EntryPointsCache 2 | from plux.runtime.metadata import build_entry_point_index, parse_entry_points_text 3 | 4 | 5 | def test_cache_get_entry_points(): 6 | result = EntryPointsCache.instance().get_entry_points() 7 | 8 | assert result 9 | assert result == EntryPointsCache.instance().get_entry_points() 10 | 11 | 12 | def test_cache_get_entry_points_creates_parseable_file(tmp_path): 13 | # a white box test for caching 14 | index = EntryPointsCache.instance()._build_and_store_index(tmp_path) 15 | assert index 16 | 17 | files = list(tmp_path.glob("*.entry_points.txt")) 18 | assert files 19 | assert index == build_entry_point_index(parse_entry_points_text(files[0].read_text())) 20 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | paths-ignore: 6 | - 'README.md' 7 | branches: 8 | - main 9 | pull_request: 10 | branches: 11 | - main 12 | 13 | jobs: 14 | test: 15 | runs-on: ubuntu-latest 16 | strategy: 17 | fail-fast: false 18 | matrix: 19 | python-version: [ "3.10", "3.11", "3.12", "3.13" ] 20 | steps: 21 | - name: Checkout 22 | uses: actions/checkout@v3 23 | 24 | - name: Set up Python 25 | id: setup-python 26 | uses: actions/setup-python@v4 27 | with: 28 | python-version: ${{ matrix.python-version }} 29 | 30 | - name: Run linting 31 | run: | 32 | make lint 33 | 34 | - name: Run tests 35 | run: | 36 | make test 37 | -------------------------------------------------------------------------------- /tests/cli/projects/manual_build_mode/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools", "wheel"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "test-project" 7 | authors = [ 8 | { name = "LocalStack Contributors", email = "info@localstack.cloud" } 9 | ] 10 | version = "0.1.0" 11 | description = "A test project to test plux with pyproject.toml projects and manual build mode" 12 | dependencies = [ 13 | "plux", 14 | "build", 15 | ] 16 | requires-python = ">=3.8" 17 | classifiers = [ 18 | "Programming Language :: Python :: 3", 19 | ] 20 | dynamic = [ 21 | "entry-points", 22 | ] 23 | 24 | [tool.setuptools] 25 | include-package-data = false 26 | 27 | [tool.setuptools.dynamic] 28 | entry-points = { file = ["plux.ini"] } 29 | 30 | [tool.setuptools.packages.find] 31 | include = ["mysrc*"] 32 | 33 | [tool.plux] 34 | entrypoint_build_mode = "manual" 35 | -------------------------------------------------------------------------------- /plux/__init__.py: -------------------------------------------------------------------------------- 1 | # TODO: should start to migrate things from the plugin package to plux 2 | from plux.core.plugin import ( 3 | CompositePluginLifecycleListener, 4 | FunctionPlugin, 5 | Plugin, 6 | PluginDisabled, 7 | PluginException, 8 | PluginFactory, 9 | PluginFinder, 10 | PluginLifecycleListener, 11 | PluginSpec, 12 | PluginSpecResolver, 13 | PluginType, 14 | plugin, 15 | ) 16 | from plux.runtime.manager import PluginContainer, PluginManager 17 | 18 | name = "plux" 19 | 20 | __version__ = "1.14.0" 21 | 22 | __all__ = [ 23 | "FunctionPlugin", 24 | "Plugin", 25 | "PluginDisabled", 26 | "PluginException", 27 | "PluginFactory", 28 | "PluginFinder", 29 | "PluginLifecycleListener", 30 | "CompositePluginLifecycleListener", 31 | "PluginManager", 32 | "PluginContainer", 33 | "PluginSpec", 34 | "PluginSpecResolver", 35 | "PluginType", 36 | "plugin", 37 | ] 38 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | VENV_BIN = python3 -m venv 2 | VENV_DIR ?= .venv 3 | 4 | VENV_ACTIVATE = . $(VENV_DIR)/bin/activate 5 | 6 | 7 | venv: $(VENV_DIR)/bin/activate 8 | 9 | $(VENV_DIR)/bin/activate: pyproject.toml 10 | test -d $(VENV_DIR) || $(VENV_BIN) $(VENV_DIR) 11 | $(VENV_ACTIVATE); pip install -e ".[dev]" 12 | touch $(VENV_DIR)/bin/activate 13 | 14 | clean: 15 | rm -rf build/ 16 | rm -rf .eggs/ 17 | rm -rf *.egg-info/ 18 | rm -rf .venv 19 | 20 | clean-dist: clean 21 | rm -rf dist/ 22 | 23 | lint: venv 24 | $(VENV_ACTIVATE); python -m ruff check . 25 | 26 | format: venv 27 | $(VENV_ACTIVATE); python -m ruff format . && python -m ruff check . --fix 28 | 29 | test: venv 30 | $(VENV_ACTIVATE); python -m pytest 31 | 32 | dist: venv 33 | $(VENV_ACTIVATE); python -m build 34 | 35 | install: venv 36 | $(VENV_ACTIVATE); pip install -e . 37 | 38 | upload: venv test dist 39 | $(VENV_ACTIVATE); pip install --upgrade twine; twine upload dist/* 40 | 41 | .PHONY: clean clean-dist format 42 | -------------------------------------------------------------------------------- /tests/plugins/sample_plugins.py: -------------------------------------------------------------------------------- 1 | from plugin import Plugin, PluginSpec, plugin 2 | 3 | 4 | # this is not a discoverable plugin (no name and namespace specified) 5 | class AbstractSamplePlugin(Plugin): 6 | pass 7 | 8 | 9 | # this plugin is discoverable since it can be resolved into a PluginSpec 10 | class SimplePlugin(AbstractSamplePlugin): 11 | namespace = "namespace_2" 12 | name = "simple" 13 | 14 | invoked: bool = False 15 | 16 | def load(self): 17 | self.invoked = True 18 | 19 | 20 | plugin_spec_1 = PluginSpec("namespace_1", "plugin_1", AbstractSamplePlugin) 21 | plugin_spec_2 = PluginSpec("namespace_1", "plugin_2", AbstractSamplePlugin) 22 | 23 | some_member = "this string should not be interpreted as a plugin" 24 | 25 | 26 | @plugin(namespace="namespace_3") 27 | def plugin_3(): 28 | # this plugin is discoverable via the FunctionPlugin decorator 29 | return "foobar" 30 | 31 | 32 | @plugin(name="plugin_4", namespace="namespace_3") 33 | def functional_plugin(): 34 | return "another" 35 | 36 | 37 | def load_condition(): 38 | return False 39 | 40 | 41 | @plugin(name="plugin_5", namespace="namespace_4", should_load=lambda: load_condition()) 42 | def functional_plugin_with_load_condition_function(): 43 | return "not loading this one" 44 | 45 | 46 | @plugin(name="plugin_6", namespace="namespace_4", should_load=False) 47 | def functional_plugin_with_load_condition_bool(): 48 | return "not loading this one either" 49 | -------------------------------------------------------------------------------- /tests/test_entrypoint.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from plux import PluginSpec 4 | from plux.core.entrypoint import EntryPoint, spec_to_entry_point, to_entry_point_dict 5 | 6 | from .plugins import sample_plugins 7 | 8 | 9 | def test_spec_to_entry_point(): 10 | spec = PluginSpec("my.namespace", "zeplugin", sample_plugins.AbstractSamplePlugin) 11 | ep = spec_to_entry_point(spec) 12 | 13 | assert ep.name == "zeplugin" 14 | assert ep.group == "my.namespace" 15 | assert ep.value == "tests.plugins.sample_plugins:AbstractSamplePlugin" 16 | 17 | 18 | def test_to_entry_point_dict(): 19 | eps = [ 20 | EntryPoint("foo", "MyFooPlugin1", "group.a"), 21 | EntryPoint("bar", "MyBarPlugin", "group.a"), 22 | EntryPoint("foo", "MyFooPlugin3", "group.b"), 23 | ] 24 | 25 | ep_dict = to_entry_point_dict(eps) 26 | 27 | assert "group.a" in ep_dict 28 | assert "group.b" in ep_dict 29 | 30 | assert "foo=MyFooPlugin1" in ep_dict["group.a"] 31 | assert "bar=MyBarPlugin" in ep_dict["group.a"] 32 | assert len(ep_dict["group.a"]) == 2 33 | 34 | assert "foo=MyFooPlugin3" in ep_dict["group.b"] 35 | assert len(ep_dict["group.b"]) == 1 36 | 37 | 38 | def test_to_entry_point_dict_duplicates(): 39 | eps = [ 40 | EntryPoint("foo", "MyFooPlugin1", "group_a"), 41 | EntryPoint("foo", "MyFooPlugin2", "group_a"), 42 | ] 43 | 44 | with pytest.raises(ValueError) as ex: 45 | to_entry_point_dict(eps) 46 | 47 | ex.match("Duplicate entry point group_a foo") 48 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ['hatchling'] 3 | build-backend = "hatchling.build" 4 | 5 | [project] 6 | name = "plux" 7 | authors = [ 8 | { name = "Thomas Rausch", email = "info@localstack.cloud" } 9 | ] 10 | description = "A dynamic code loading framework for building pluggable Python distributions" 11 | readme = "README.md" 12 | license = "Apache-2.0" 13 | classifiers = [ 14 | "Development Status :: 5 - Production/Stable", 15 | "Operating System :: OS Independent", 16 | "Programming Language :: Python :: 3", 17 | "Programming Language :: Python :: 3.10", 18 | "Programming Language :: Python :: 3.11", 19 | "Programming Language :: Python :: 3.12", 20 | "Programming Language :: Python :: 3.13", 21 | "Topic :: Software Development :: Libraries", 22 | "Topic :: Utilities" 23 | ] 24 | requires-python = ">=3.10" 25 | dynamic = ["version"] 26 | 27 | [project.urls] 28 | Repository = "https://github.com/localstack/plux" 29 | 30 | [project.optional-dependencies] 31 | dev = [ 32 | "build", 33 | "setuptools", 34 | "pytest==8.4.1", 35 | "ruff==0.9.1", 36 | ] 37 | 38 | [tool.hatch.version] 39 | path = "plux/__init__.py" 40 | 41 | [tool.ruff] 42 | line-length = 110 43 | target-version = "py310" 44 | 45 | # integrations with setuptools 46 | 47 | [project.entry-points."distutils.commands"] 48 | plugins = "plux.build.setuptools:plugins" 49 | 50 | [project.entry-points."egg_info.writers"] 51 | # this is actually not a writer, it's a reader :-) 52 | "plux.json" = "plux.build.setuptools:load_plux_entrypoints" 53 | 54 | -------------------------------------------------------------------------------- /tests/test_discovery.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from plux.build.discovery import ModuleScanningPluginFinder 4 | from plux.build.setuptools import PackagePathPluginFinder 5 | 6 | from .plugins import sample_plugins 7 | 8 | 9 | class TestModuleScanningPluginFinder: 10 | def test_find_plugins(self): 11 | finder = ModuleScanningPluginFinder(modules=[sample_plugins]) 12 | 13 | plugins = finder.find_plugins() 14 | assert len(plugins) == 7 15 | 16 | plugins = [(spec.namespace, spec.name) for spec in plugins] 17 | 18 | # update when adding plugins to sample_plugins 19 | assert ("namespace_2", "simple") in plugins 20 | assert ("namespace_1", "plugin_1") in plugins 21 | assert ("namespace_1", "plugin_2") in plugins 22 | assert ("namespace_3", "plugin_3") in plugins 23 | assert ("namespace_3", "plugin_4") in plugins 24 | assert ("namespace_4", "plugin_5") in plugins 25 | assert ("namespace_4", "plugin_6") in plugins 26 | 27 | 28 | class TestPackagePathPluginFinder: 29 | def test_find_plugins(self): 30 | where = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) 31 | 32 | finder = PackagePathPluginFinder(where=where, include=("tests.plugins",)) 33 | 34 | plugins = finder.find_plugins() 35 | assert len(plugins) == 7 36 | 37 | plugins = [(spec.namespace, spec.name) for spec in plugins] 38 | 39 | # update when adding plugins to sample_plugins 40 | assert ("namespace_2", "simple") in plugins 41 | assert ("namespace_1", "plugin_1") in plugins 42 | assert ("namespace_1", "plugin_2") in plugins 43 | assert ("namespace_3", "plugin_3") in plugins 44 | assert ("namespace_3", "plugin_4") in plugins 45 | assert ("namespace_4", "plugin_5") in plugins 46 | assert ("namespace_4", "plugin_6") in plugins 47 | -------------------------------------------------------------------------------- /plux/core/entrypoint.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | import typing as t 3 | from collections import defaultdict 4 | 5 | from .plugin import PluginFinder, PluginSpec 6 | 7 | 8 | class EntryPoint(t.NamedTuple): 9 | name: str 10 | value: str 11 | group: str 12 | 13 | 14 | EntryPointDict = dict[str, list[str]] 15 | 16 | 17 | def discover_entry_points(finder: PluginFinder) -> EntryPointDict: 18 | """ 19 | Creates a dictionary for the entry_points attribute of setuptools' setup(), where keys are 20 | stevedore plugin namespaces, and values are lists of "name = module:object" pairs. 21 | 22 | :return: an entry_point dictionary 23 | """ 24 | return to_entry_point_dict([spec_to_entry_point(spec) for spec in finder.find_plugins()]) 25 | 26 | 27 | def to_entry_point_dict(eps: list[EntryPoint]) -> EntryPointDict: 28 | """ 29 | Convert the list of EntryPoint objects to a dictionary that maps entry point groups to their respective list of 30 | ``name=value`` entry points. Each pair is represented as a string. 31 | 32 | :param eps: List of entrypoints to convert 33 | :return: an entry point dictionary 34 | :raises ValueError: if there are duplicate entry points in the same group 35 | """ 36 | result = defaultdict(list) 37 | names = defaultdict(set) # book-keeping to check duplicates 38 | 39 | for ep in eps: 40 | if ep.name in names[ep.group]: 41 | raise ValueError("Duplicate entry point %s %s" % (ep.group, ep.name)) 42 | 43 | result[ep.group].append("%s=%s" % (ep.name, ep.value)) 44 | names[ep.group].add(ep.name) 45 | 46 | return result 47 | 48 | 49 | def spec_to_entry_point(spec: PluginSpec) -> EntryPoint: 50 | module = inspect.getmodule(spec.factory).__name__ 51 | name = spec.factory.__name__ 52 | path = f"{module}:{name}" 53 | return EntryPoint(group=spec.namespace, name=spec.name, value=path) 54 | -------------------------------------------------------------------------------- /tests/test_function_plugin.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from plux import PluginDisabled, PluginManager 4 | from tests.plugins import sample_plugins 5 | 6 | 7 | def test_load_functional_plugins(sample_plugin_finder): 8 | manager = PluginManager("namespace_3", finder=sample_plugin_finder) 9 | 10 | plugins = manager.load_all() 11 | assert len(plugins) == 2 12 | 13 | assert plugins[0].name in ["plugin_3", "plugin_4"] 14 | assert plugins[1].name in ["plugin_3", "plugin_4"] 15 | 16 | if plugins[0].name == "plugin_3": 17 | plugin_3 = plugins[0] 18 | plugin_4 = plugins[1] 19 | else: 20 | plugin_3 = plugins[1] 21 | plugin_4 = plugins[0] 22 | 23 | # function plugins are callable directly 24 | assert plugin_3() == "foobar" 25 | assert plugin_4() == "another" 26 | 27 | 28 | def test_load_functional_plugins_with_load_condition_false(sample_plugin_finder): 29 | manager = PluginManager("namespace_4", finder=sample_plugin_finder) 30 | 31 | plugins = manager.load_all() 32 | assert len(plugins) == 0 33 | 34 | with pytest.raises(PluginDisabled) as e: 35 | manager.load("plugin_5") 36 | e.match("plugin namespace_4:plugin_5 is disabled, reason: Load condition for plugin was false") 37 | 38 | with pytest.raises(PluginDisabled) as e: 39 | manager.load("plugin_6") 40 | e.match("plugin namespace_4:plugin_6 is disabled, reason: Load condition for plugin was false") 41 | 42 | 43 | def test_load_functional_plugins_with_load_condition_patchd(sample_plugin_finder, monkeypatch): 44 | monkeypatch.setattr(sample_plugins, "load_condition", lambda: True) 45 | 46 | manager = PluginManager("namespace_4", finder=sample_plugin_finder) 47 | 48 | plugins = manager.load_all() 49 | assert len(plugins) == 1 50 | 51 | manager.load("plugin_5") 52 | 53 | with pytest.raises(PluginDisabled) as e: 54 | manager.load("plugin_6") 55 | e.match("plugin namespace_4:plugin_6 is disabled, reason: Load condition for plugin was false") 56 | -------------------------------------------------------------------------------- /plux/runtime/resolve.py: -------------------------------------------------------------------------------- 1 | """Tools to resolve PluginSpec instances from entry points at runtime. The primary mechanism we use to do 2 | that is ``importlib.metadata`` and a bit of caching.""" 3 | 4 | import logging 5 | import typing as t 6 | from importlib.metadata import EntryPoint 7 | 8 | from plux.core.plugin import PluginFinder, PluginSpec, PluginSpecResolver 9 | 10 | from .cache import EntryPointsCache 11 | from .metadata import EntryPointsResolver 12 | 13 | LOG = logging.getLogger(__name__) 14 | 15 | 16 | class MetadataPluginFinder(PluginFinder): 17 | """ 18 | This is a simple implementation of a PluginFinder that uses by default the ``EntryPointsCache`` singleton. 19 | """ 20 | 21 | def __init__( 22 | self, 23 | namespace: str, 24 | on_resolve_exception_callback: t.Callable[[str, t.Any, Exception], None] = None, 25 | spec_resolver: PluginSpecResolver = None, 26 | entry_points_resolver: EntryPointsResolver = None, 27 | ) -> None: 28 | super().__init__() 29 | self.namespace = namespace 30 | self.on_resolve_exception_callback = on_resolve_exception_callback 31 | self.spec_resolver = spec_resolver or PluginSpecResolver() 32 | self.entry_points_resolver = entry_points_resolver or EntryPointsCache.instance() 33 | 34 | def find_plugins(self) -> list[PluginSpec]: 35 | specs = [] 36 | finds = self.entry_points_resolver.get_entry_points().get(self.namespace, []) 37 | for ep in finds: 38 | spec = self.to_plugin_spec(ep) 39 | if spec: 40 | specs.append(spec) 41 | return specs 42 | 43 | def to_plugin_spec(self, entry_point: EntryPoint) -> PluginSpec: 44 | """ 45 | Convert a stevedore extension into a PluginSpec by using a spec_resolver. 46 | """ 47 | try: 48 | source = entry_point.load() 49 | return self.spec_resolver.resolve(source) 50 | except Exception as e: 51 | if LOG.isEnabledFor(logging.DEBUG): 52 | LOG.exception("error resolving PluginSpec for plugin %s.%s", self.namespace, entry_point.name) 53 | 54 | if self.on_resolve_exception_callback: 55 | self.on_resolve_exception_callback(self.namespace, entry_point, e) 56 | -------------------------------------------------------------------------------- /tests/runtime/test_resolve.py: -------------------------------------------------------------------------------- 1 | from importlib import metadata 2 | from importlib.metadata import EntryPoint 3 | from unittest.mock import MagicMock 4 | 5 | import pytest 6 | 7 | from plux.runtime.metadata import EntryPointsResolver, build_entry_point_index 8 | from plux.runtime.resolve import MetadataPluginFinder 9 | 10 | 11 | class DummyEntryPointsResolver(EntryPointsResolver): 12 | entry_points: dict[str, list[metadata.EntryPoint]] 13 | 14 | def __init__(self, entry_points: list[metadata.EntryPoint]): 15 | self.entry_points = build_entry_point_index(entry_points) 16 | 17 | def get_entry_points(self) -> dict[str, list[metadata.EntryPoint]]: 18 | return self.entry_points 19 | 20 | 21 | @pytest.fixture 22 | def dummy_entry_point_resolver(): 23 | return DummyEntryPointsResolver( 24 | [ 25 | EntryPoint( 26 | group="namespace_1", 27 | name="plugin_1", 28 | value="tests.plugins.sample_plugins:plugin_spec_1", 29 | ), 30 | EntryPoint( 31 | group="namespace_1", 32 | name="plugin_2", 33 | value="tests.plugins.sample_plugins:plugin_spec_2", 34 | ), 35 | EntryPoint( 36 | group="namespace_2", 37 | name="cannot-be-loaded", 38 | value="tests.plugins.invalid_module:CannotBeLoadedPlugin", 39 | ), 40 | EntryPoint( 41 | group="namespace_2", 42 | name="simple", 43 | value="tests.plugins.sample_plugins:SimplePlugin", 44 | ), 45 | ] 46 | ) 47 | 48 | 49 | def test_resolve_error(dummy_entry_point_resolver): 50 | mock = MagicMock() 51 | finder = MetadataPluginFinder( 52 | "namespace_2", 53 | on_resolve_exception_callback=mock, 54 | entry_points_resolver=dummy_entry_point_resolver, 55 | ) 56 | plugins = finder.find_plugins() 57 | assert len(plugins) == 1 58 | assert plugins[0].name == "simple" 59 | assert mock.call_count == 1 60 | assert mock.call_args[0][0] == "namespace_2" 61 | assert mock.call_args[0][1] == EntryPoint( 62 | "cannot-be-loaded", 63 | "tests.plugins.invalid_module:CannotBeLoadedPlugin", 64 | "namespace_2", 65 | ) 66 | assert str(mock.call_args[0][2]) == "this is an expected exception" 67 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | *.iml 3 | *~ 4 | 5 | # General 6 | .DS_Store 7 | .AppleDouble 8 | .LSOverride 9 | 10 | # Icon must end with two \r 11 | Icon 12 | 13 | # Thumbnails 14 | ._* 15 | 16 | # Files that might appear in the root of a volume 17 | .DocumentRevisions-V100 18 | .fseventsd 19 | .Spotlight-V100 20 | .TemporaryItems 21 | .Trashes 22 | .VolumeIcon.icns 23 | .com.apple.timemachine.donotpresent 24 | 25 | # Directories potentially created on remote AFP share 26 | .AppleDB 27 | .AppleDesktop 28 | Network Trash Folder 29 | Temporary Items 30 | .apdisk 31 | 32 | # Byte-compiled / optimized / DLL files 33 | __pycache__/ 34 | *.py[cod] 35 | *$py.class 36 | 37 | # C extensions 38 | *.so 39 | 40 | # Distribution / packaging 41 | .Python 42 | build/ 43 | develop-eggs/ 44 | dist/ 45 | downloads/ 46 | eggs/ 47 | .eggs/ 48 | lib/ 49 | lib64/ 50 | parts/ 51 | sdist/ 52 | var/ 53 | wheels/ 54 | *.egg-info/ 55 | .installed.cfg 56 | *.egg 57 | MANIFEST 58 | 59 | # PyInstaller 60 | # Usually these files are written by a python script from a template 61 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 62 | *.manifest 63 | *.spec 64 | 65 | # Installer logs 66 | pip-log.txt 67 | pip-delete-this-directory.txt 68 | 69 | # Unit test / coverage reports 70 | htmlcov/ 71 | .tox/ 72 | .coverage 73 | .coverage.* 74 | .cache 75 | nosetests.xml 76 | coverage.xml 77 | *.cover 78 | .hypothesis/ 79 | 80 | # Translations 81 | *.mo 82 | *.pot 83 | 84 | # Django stuff: 85 | *.log 86 | .static_storage/ 87 | .media/ 88 | local_settings.py 89 | 90 | # Flask stuff: 91 | instance/ 92 | .webassets-cache 93 | 94 | # Scrapy stuff: 95 | .scrapy 96 | 97 | # Sphinx documentation 98 | docs/_build/ 99 | 100 | # PyBuilder 101 | target/ 102 | 103 | # Jupyter Notebook 104 | .ipynb_checkpoints 105 | 106 | # pyenv 107 | .python-version 108 | 109 | # celery beat schedule file 110 | celerybeat-schedule 111 | 112 | # SageMath parsed files 113 | *.sage.py 114 | 115 | # Environments 116 | .venv 117 | env/ 118 | venv/ 119 | ENV/ 120 | env.bak/ 121 | venv.bak/ 122 | 123 | # Spyder project settings 124 | .spyderproject 125 | .spyproject 126 | 127 | # Rope project settings 128 | .ropeproject 129 | 130 | # mkdocs documentation 131 | /site 132 | 133 | # mypy 134 | .mypy_cache/ 135 | 136 | # don't ignore build package 137 | !plux/build 138 | -------------------------------------------------------------------------------- /plux/runtime/filter.py: -------------------------------------------------------------------------------- 1 | import fnmatch 2 | import logging 3 | import typing as t 4 | 5 | from plux.core.entrypoint import spec_to_entry_point 6 | from plux.core.plugin import PluginSpec 7 | 8 | LOG = logging.getLogger(__name__) 9 | 10 | 11 | class PluginFilter(t.Protocol): 12 | def __call__(self, spec: PluginSpec) -> bool: 13 | """ 14 | Returns True if the plugin should be filtered (disabled), or False otherwise. 15 | 16 | :param spec: the spec to check 17 | :return: True if the plugin should be disabled 18 | """ 19 | ... 20 | 21 | 22 | class PluginSpecMatcher: 23 | namespace: str = None 24 | name: str = None 25 | value: str = None 26 | 27 | def __init__(self, *, namespace: str = None, name: str = None, value: str = None): 28 | self.namespace = namespace 29 | self.name = name 30 | self.value = value 31 | 32 | def matches(self, spec: PluginSpec) -> bool: 33 | if self.namespace: 34 | if not fnmatch.fnmatch(spec.namespace, self.namespace): 35 | return False 36 | 37 | if self.name: 38 | if not fnmatch.fnmatch(spec.name, self.name): 39 | return False 40 | 41 | if self.value: 42 | ep = spec_to_entry_point(spec) 43 | if not fnmatch.fnmatch(ep.value, self.value): 44 | return False 45 | 46 | return True 47 | 48 | def __str__(self): 49 | return f"PluginSpecMatcher({self.__dict__})" 50 | 51 | 52 | class MatchingPluginFilter: 53 | """ 54 | A MatchingPluginFilter can be used to exclude specific plugins from loading. 55 | """ 56 | 57 | exclusions: list[PluginSpecMatcher] 58 | 59 | def __init__(self): 60 | self.exclusions = [] 61 | 62 | def add_exclusion(self, *, namespace: str = None, name: str = None, value: str = None): 63 | """ 64 | Adds a pattern of plugins that should be excluded. The patterns use ``fnmatch``. For example:: 65 | 66 | # filters all plugins with a namespace "some.namespace.a", "some.namespace.b", ... 67 | add_exclusion(namespace = "some.namespace.*") 68 | 69 | # filters all plugins that come from a specific package tree 70 | add_exclusion(value = "my.package.*") 71 | 72 | Combining parameters will create an `AND` clause. 73 | 74 | :param namespace: 75 | :param name: 76 | :param value: 77 | :return: 78 | """ 79 | self.exclusions.append(PluginSpecMatcher(namespace=namespace, name=name, value=value)) 80 | 81 | def __call__(self, spec: PluginSpec) -> bool: 82 | """ 83 | Checks whether a given ``PluginSpec`` matches the filter criteria. If the matcher returns True, 84 | the plugin should be disabled. 85 | 86 | :param spec: the plugin spec 87 | :return: True if the plugin should be disabled 88 | """ 89 | for matcher in self.exclusions: 90 | if matcher.matches(spec): 91 | # do not load the plugin if it matches the spec 92 | LOG.debug("Filter rule %s matched %s", matcher, spec) 93 | return True 94 | return False 95 | 96 | 97 | global_plugin_filter = MatchingPluginFilter() 98 | -------------------------------------------------------------------------------- /plux/build/project.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pathlib import Path 3 | 4 | from plux.build.config import PluxConfiguration, read_plux_config_from_workdir 5 | from plux.build.discovery import PackageFinder, PluginFromPackageFinder 6 | from plux.build.index import PluginIndexBuilder 7 | 8 | 9 | class Project: 10 | """ 11 | Abstraction for a python project to hide the details of a build tool from the CLI. A Project provides access to 12 | the project's configuration, package discovery, and the entrypoint build mechanism. 13 | """ 14 | 15 | workdir: Path 16 | config: PluxConfiguration 17 | 18 | def __init__(self, workdir: str = None): 19 | self.workdir = Path(workdir or os.curdir) 20 | self.config = self.read_static_plux_config() 21 | 22 | def find_entry_point_file(self) -> Path: 23 | """ 24 | Finds the entry_point.txt file of the current project. In case of setuptools, this may be in the 25 | ``.egg-info`` directory, in case of hatch, where ``pip install -e .`` has become the standard, the 26 | entrypoints file lives in the ``.dist-info`` directory of the venv. 27 | 28 | :return: A path pointing to the entrypoints file. The file might not exist. 29 | """ 30 | raise NotImplementedError 31 | 32 | def find_plux_index_file(self) -> Path: 33 | """ 34 | Returns the plux index file location. This may depend on the build tool, for similar reasons described in 35 | ``find_entry_point_file``. For example, in setuptools, the plux index file by default is in 36 | ``.egg-info/plux.json``. 37 | 38 | :return: A path pointing to the plux index file. 39 | """ 40 | raise NotImplementedError 41 | 42 | def create_package_finder(self) -> PackageFinder: 43 | """ 44 | Returns a build tool-specific PackageFinder instance that can be used to discover packages to scan for plugins. 45 | 46 | :return: A PackageFinder instance 47 | """ 48 | raise NotImplementedError 49 | 50 | def build_entrypoints(self): 51 | """ 52 | Routine to build the entrypoints file using ``EntryPointBuildMode.BUILD_HOOK``. This is called by the CLI 53 | frontend. It's build tool-specific since we need to hook into the build process that generates the 54 | ``entry_points.txt``. 55 | """ 56 | raise NotImplementedError 57 | 58 | def create_plugin_index_builder(self) -> PluginIndexBuilder: 59 | """ 60 | Returns a PluginIndexBuilder instance that can be used to build the plugin index. 61 | 62 | The default implementation creates a PluginFromPackageFinder instance using ``create_package_finder``. 63 | 64 | :return: A PluginIndexBuilder instance. 65 | """ 66 | plugin_finder = PluginFromPackageFinder(self.create_package_finder()) 67 | return PluginIndexBuilder(plugin_finder) 68 | 69 | def read_static_plux_config(self) -> PluxConfiguration: 70 | """ 71 | Reads the static configuration (``pyproject.toml``) from the Project's working directory using 72 | ``read_read_plux_config_from_workdir``. 73 | 74 | :return: A PluxConfiguration object 75 | """ 76 | return read_plux_config_from_workdir(str(self.workdir)) 77 | -------------------------------------------------------------------------------- /tests/cli/test_discover.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import sys 3 | 4 | import pytest 5 | 6 | 7 | @pytest.mark.parametrize("project_name", ["pyproject", "setupcfg"]) 8 | def test_discover_with_ini_output(project_name, tmp_path): 9 | from plux.__main__ import main 10 | 11 | project = os.path.join(os.path.dirname(__file__), "projects", project_name) 12 | os.chdir(project) 13 | 14 | out_path = tmp_path / "plux.ini" 15 | 16 | sys.path.append(project) 17 | try: 18 | try: 19 | main(["--workdir", project, "discover", "--format", "ini", "--output", str(out_path)]) 20 | except SystemExit: 21 | pass 22 | finally: 23 | sys.path.remove(project) 24 | 25 | lines = out_path.read_text().strip().splitlines() 26 | assert lines == [ 27 | "[plux.test.plugins]", 28 | "mynestedplugin = mysrc.subpkg.plugins:MyNestedPlugin", 29 | "myplugin = mysrc.plugins:MyPlugin", 30 | ] 31 | 32 | 33 | @pytest.mark.parametrize("project_name", ["pyproject", "setupcfg"]) 34 | def test_discover_with_ini_output_exclude(project_name, tmp_path): 35 | from plux.__main__ import main 36 | 37 | project = os.path.join(os.path.dirname(__file__), "projects", project_name) 38 | os.chdir(project) 39 | 40 | out_path = tmp_path / "plux.ini" 41 | 42 | sys.path.append(project) 43 | try: 44 | try: 45 | main( 46 | [ 47 | "--workdir", 48 | project, 49 | "discover", 50 | "--format", 51 | "ini", 52 | "--output", 53 | str(out_path), 54 | "--exclude", 55 | "*/subpkg*", 56 | ] 57 | ) 58 | except SystemExit: 59 | pass 60 | finally: 61 | sys.path.remove(project) 62 | 63 | lines = out_path.read_text().strip().splitlines() 64 | assert lines == [ 65 | "[plux.test.plugins]", 66 | "myplugin = mysrc.plugins:MyPlugin", 67 | ] 68 | 69 | 70 | @pytest.mark.parametrize("project_name", ["pyproject", "setupcfg"]) 71 | def test_discover_with_ini_output_include(project_name, tmp_path): 72 | from plux.__main__ import main 73 | 74 | project = os.path.join(os.path.dirname(__file__), "projects", project_name) 75 | os.chdir(project) 76 | 77 | out_path = tmp_path / "plux.ini" 78 | 79 | sys.path.append(project) 80 | try: 81 | try: 82 | main( 83 | [ 84 | "--workdir", 85 | project, 86 | "discover", 87 | "--format", 88 | "ini", 89 | "--output", 90 | str(out_path), 91 | "--include", 92 | "*/subpkg*", 93 | ] 94 | ) 95 | except SystemExit: 96 | pass 97 | finally: 98 | sys.path.remove(project) 99 | 100 | lines = out_path.read_text().strip().splitlines() 101 | assert lines == [ 102 | "[plux.test.plugins]", 103 | "mynestedplugin = mysrc.subpkg.plugins:MyNestedPlugin", 104 | ] 105 | -------------------------------------------------------------------------------- /plux/build/index.py: -------------------------------------------------------------------------------- 1 | """ 2 | Code to manage the plux plugin index file. The index file contains all discovered plugins, which later is used to 3 | generate entry points. 4 | """ 5 | 6 | import configparser 7 | import json 8 | import sys 9 | import typing as t 10 | 11 | from plux.core.plugin import PluginFinder 12 | from plux.core.entrypoint import EntryPointDict, discover_entry_points 13 | 14 | if t.TYPE_CHECKING: 15 | from _typeshed import SupportsWrite 16 | 17 | 18 | class PluginIndexBuilder: 19 | """ 20 | Builds an index file containing all discovered plugins. The index file can be written to stdout, or to a file. 21 | The writer supports two formats: json and ini. 22 | """ 23 | 24 | def __init__( 25 | self, 26 | plugin_finder: PluginFinder, 27 | ): 28 | self.plugin_finder = plugin_finder 29 | 30 | def write( 31 | self, 32 | fp: "SupportsWrite[str]" = sys.stdout, 33 | output_format: t.Literal["json", "ini"] = "json", 34 | ) -> EntryPointDict: 35 | """ 36 | Discover entry points using the configured ``PluginFinder``, and write the entry points into a file. 37 | 38 | :param fp: The file-like object to write to. 39 | :param output_format: The format to write the entry points in. Can be either "json" or "ini". 40 | :return: The discovered entry points that were written into the file. 41 | """ 42 | ep = discover_entry_points(self.plugin_finder) 43 | 44 | # sort entrypoints alphabetically in each group first 45 | for group in ep: 46 | ep[group].sort() 47 | 48 | if output_format == "json": 49 | json.dump(ep, fp, sort_keys=True, indent=2) 50 | elif output_format == "ini": 51 | cfg = configparser.ConfigParser() 52 | cfg.read_dict(self.convert_to_nested_entry_point_dict(ep)) 53 | cfg.write(fp) 54 | else: 55 | raise ValueError(f"unknown plugin index output format {output_format}") 56 | 57 | return ep 58 | 59 | @staticmethod 60 | def convert_to_nested_entry_point_dict(ep: EntryPointDict) -> dict[str, dict[str, str]]: 61 | """ 62 | Converts and ``EntryPointDict`` to a nested dict, where the keys are the section names and values are 63 | dictionaries. Each dictionary maps entry point names to their values. It also sorts the output alphabetically. 64 | 65 | Example: 66 | Input EntryPointDict: 67 | { 68 | 'console_scripts': ['app=module:main', 'tool=module:cli'], 69 | 'plux.plugins': ['plugin1=pkg.module:Plugin1'] 70 | } 71 | 72 | Output nested dict: 73 | { 74 | 'console_scripts': { 75 | 'app': 'module:main', 76 | 'tool': 'module:cli' 77 | }, 78 | 'plux.plugins': { 79 | 'plugin1': 'pkg.module:Plugin1' 80 | } 81 | } 82 | """ 83 | result = {} 84 | for section_name in sorted(ep.keys()): 85 | result[section_name] = {} 86 | for entry_point in sorted(ep[section_name]): 87 | name, value = entry_point.split("=") 88 | result[section_name][name] = value 89 | return result 90 | -------------------------------------------------------------------------------- /tests/test_listener.py: -------------------------------------------------------------------------------- 1 | import typing as t 2 | from typing import Dict, List, Tuple 3 | from unittest.mock import MagicMock 4 | 5 | import pytest 6 | 7 | from plux import ( 8 | CompositePluginLifecycleListener, 9 | Plugin, 10 | PluginDisabled, 11 | PluginException, 12 | PluginFinder, 13 | PluginLifecycleListener, 14 | PluginManager, 15 | PluginSpec, 16 | ) 17 | 18 | 19 | class DummyPlugin(Plugin): 20 | load_calls: List[Tuple[Tuple, Dict]] 21 | 22 | def __init__(self) -> None: 23 | super().__init__() 24 | self.load_calls = list() 25 | 26 | def load(self, *args, **kwargs): 27 | self.load_calls.append((args, kwargs)) 28 | 29 | 30 | class ShouldNotLoadPlugin(DummyPlugin): 31 | def should_load(self) -> bool: 32 | return False 33 | 34 | 35 | class GoodPlugin(DummyPlugin): 36 | pass 37 | 38 | 39 | class _DummyPluginFinder(PluginFinder): 40 | def __init__(self, specs: List[PluginSpec]): 41 | self.specs = specs 42 | 43 | def find_plugins(self) -> List[PluginSpec]: 44 | return self.specs 45 | 46 | 47 | @pytest.fixture 48 | def dummy_plugin_finder(): 49 | return _DummyPluginFinder( 50 | [ 51 | PluginSpec("test.plugins.dummy", "plugin1", GoodPlugin), 52 | PluginSpec("test.plugins.dummy", "plugin2", GoodPlugin), 53 | ] 54 | ) 55 | 56 | 57 | class TestListener: 58 | def test_listeners_called(self, dummy_plugin_finder): 59 | listener = MagicMock() 60 | listener.on_resolve_after = MagicMock() 61 | listener.on_init_exception = MagicMock() 62 | listener.on_init_after = MagicMock() 63 | listener.on_load_before = MagicMock() 64 | listener.on_load_after = MagicMock() 65 | listener.on_load_exception = MagicMock() 66 | 67 | manager = PluginManager( 68 | "test.plugins.dummy", 69 | finder=dummy_plugin_finder, 70 | listener=CompositePluginLifecycleListener([listener]), 71 | ) 72 | 73 | plugins = manager.load_all() 74 | assert len(plugins) == 2 75 | 76 | assert listener.on_resolve_after.call_count == 2 77 | assert listener.on_init_exception.call_count == 0 78 | assert listener.on_init_after.call_count == 2 79 | assert listener.on_load_before.call_count == 2 80 | assert listener.on_load_after.call_count == 2 81 | assert listener.on_load_exception.call_count == 0 82 | 83 | def test_on_init_after_can_disable_plugin(self, dummy_plugin_finder): 84 | class _DisableListener(PluginLifecycleListener): 85 | def on_init_after(self, plugin_spec: PluginSpec, plugin: Plugin): 86 | if plugin_spec.name == "plugin2": 87 | raise PluginDisabled(plugin_spec.namespace, plugin_spec.name, "decided during init") 88 | 89 | manager = PluginManager( 90 | "test.plugins.dummy", 91 | finder=dummy_plugin_finder, 92 | listener=_DisableListener(), 93 | ) 94 | 95 | plugins = manager.load_all() 96 | assert len(plugins) == 1 97 | 98 | with pytest.raises(PluginDisabled) as e: 99 | manager.load("plugin2") 100 | 101 | assert e.match("decided during init") 102 | 103 | def test_on_load_before_can_disable_plugin(self, dummy_plugin_finder): 104 | class _DisableListener(PluginLifecycleListener): 105 | def on_load_before(self, plugin_spec: PluginSpec, _plugin: Plugin, _load_args, _load_kwargs): 106 | if plugin_spec.name == "plugin2": 107 | raise PluginDisabled(plugin_spec.namespace, plugin_spec.name, "decided during load") 108 | 109 | manager = PluginManager( 110 | "test.plugins.dummy", 111 | finder=dummy_plugin_finder, 112 | listener=_DisableListener(), 113 | ) 114 | 115 | plugins = manager.load_all() 116 | assert len(plugins) == 1 117 | 118 | with pytest.raises(PluginDisabled) as e: 119 | manager.load("plugin2") 120 | 121 | assert e.match("decided during load") 122 | 123 | def test_on_after_before_with_error(self, dummy_plugin_finder): 124 | class _DisableListener(PluginLifecycleListener): 125 | def on_load_after(self, plugin_spec: PluginSpec, plugin: Plugin, load_result: t.Any = None): 126 | if plugin_spec.name == "plugin2": 127 | raise PluginException("error loading plugin 2") 128 | 129 | manager = PluginManager( 130 | "test.plugins.dummy", 131 | finder=dummy_plugin_finder, 132 | listener=_DisableListener(), 133 | ) 134 | 135 | plugins = manager.load_all() 136 | assert len(plugins) == 1 137 | 138 | with pytest.raises(PluginException) as e: 139 | manager.load("plugin2") 140 | 141 | assert e.match("error loading plugin 2") 142 | -------------------------------------------------------------------------------- /plux/build/config.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module for our config wrapper. Currently, this only supports ``pyproject.toml`` files but could be extended in the 3 | future to support ``tox.ini``, ``setup.cfg``, etc. 4 | """ 5 | 6 | import dataclasses 7 | import enum 8 | import os 9 | import sys 10 | from importlib.util import find_spec 11 | 12 | 13 | class EntrypointBuildMode(enum.Enum): 14 | """ 15 | The build mode for entrypoints. When ``build-hook`` is used, plux hooks into the build backend's build process using 16 | various extension points. Currently, we only support setuptools, where plux generates a plugin index and adds it to 17 | the .egg-info directory, which is later used to generate entry points. 18 | 19 | The alternative is ``manual``, where build hooks are disabled and the user is responsible for generating and 20 | referencing entry points. 21 | """ 22 | 23 | MANUAL = "manual" 24 | BUILD_HOOK = "build-hook" 25 | 26 | 27 | @dataclasses.dataclass 28 | class PluxConfiguration: 29 | """ 30 | Configuration object with sane default values. 31 | """ 32 | 33 | path: str = "." 34 | """The path to scan for plugins.""" 35 | 36 | exclude: list[str] = dataclasses.field(default_factory=list) 37 | """A list of paths to exclude from scanning.""" 38 | 39 | include: list[str] = dataclasses.field(default_factory=list) 40 | """A list of paths to include in scanning. If it's specified, only the named items will be included. If it's not 41 | specified, all found items will be included. ``include`` can contain shell style wildcard patterns just like 42 | ``exclude``.""" 43 | 44 | entrypoint_build_mode: EntrypointBuildMode = EntrypointBuildMode.BUILD_HOOK 45 | """The point in the build process path plugins should be discovered and entrypoints generated.""" 46 | 47 | entrypoint_static_file: str = "plux.ini" 48 | """The name of the entrypoint ini file if entrypoint_build_mode is set to MANUAL.""" 49 | 50 | def merge( 51 | self, 52 | path: str = None, 53 | exclude: list[str] = None, 54 | include: list[str] = None, 55 | entrypoint_build_mode: EntrypointBuildMode = None, 56 | entrypoint_static_file: str = None, 57 | ) -> "PluxConfiguration": 58 | """ 59 | Merges or overwrites the given values into the current configuration and returns a new configuration object. 60 | If the passed values are None, they are not changed. 61 | """ 62 | return PluxConfiguration( 63 | path=path if path is not None else self.path, 64 | exclude=list(set((exclude if exclude is not None else []) + self.exclude)), 65 | include=list(set((include if include is not None else []) + self.include)), 66 | entrypoint_build_mode=entrypoint_build_mode 67 | if entrypoint_build_mode is not None 68 | else self.entrypoint_build_mode, 69 | entrypoint_static_file=entrypoint_static_file 70 | if entrypoint_static_file is not None 71 | else self.entrypoint_static_file, 72 | ) 73 | 74 | 75 | def read_plux_config_from_workdir(workdir: str = None) -> PluxConfiguration: 76 | """ 77 | Reads the plux configuration from the specified workdir. Currently, it only checks for a ``pyproject.toml`` to 78 | parse. If no pyproject.toml is found, a default configuration is returned. 79 | 80 | :param workdir: The workdir which defaults to the current working directory. 81 | :return: A plux configuration object 82 | """ 83 | try: 84 | pyproject_file = os.path.join(workdir or os.getcwd(), "pyproject.toml") 85 | return parse_pyproject_toml(pyproject_file) 86 | except FileNotFoundError: 87 | return PluxConfiguration() 88 | 89 | 90 | def parse_pyproject_toml(path: str | os.PathLike[str]) -> PluxConfiguration: 91 | """ 92 | Parses a pyproject.toml file and searches for the ``[tool.plux]`` section. It then creates a ``Configuration`` 93 | object from the found values. Uses tomli or tomllib to parse the file. 94 | 95 | :param path: Path to the pyproject.toml file. 96 | :return: A plux configuration object containing the parsed values. 97 | :raises FileNotFoundError: If the file does not exist. 98 | """ 99 | if find_spec("tomllib"): 100 | from tomllib import load as load_toml 101 | elif find_spec("tomli"): 102 | from tomli import load as load_toml 103 | else: 104 | raise ImportError("Could not find a TOML parser. Please install either tomllib or tomli.") 105 | 106 | # read the file 107 | if not os.path.exists(path): 108 | raise FileNotFoundError(f"No pyproject.toml found at {path}") 109 | with open(path, "rb") as file: 110 | pyproject_config = load_toml(file) 111 | 112 | # find the [tool.plux] section 113 | tool_table = pyproject_config.get("tool", {}) 114 | tool_config = tool_table.get("plux", {}) 115 | 116 | # filter out all keys that are not available in the config object 117 | kwargs = {} 118 | for key, value in tool_config.items(): 119 | if key not in PluxConfiguration.__annotations__: 120 | print(f"Warning: ignoring unknown key {key} in [tool.plux] section of {path}", file=sys.stderr) 121 | continue 122 | 123 | kwargs[key] = value 124 | 125 | # parse entrypoint_build_mode enum 126 | if mode := kwargs.get("entrypoint_build_mode"): 127 | # will raise a ValueError exception if the mode is invalid 128 | kwargs["entrypoint_build_mode"] = EntrypointBuildMode(mode) 129 | 130 | return PluxConfiguration(**kwargs) 131 | -------------------------------------------------------------------------------- /plux/build/discovery.py: -------------------------------------------------------------------------------- 1 | """ 2 | Buildtool independent utils to discover plugins from the project's source code. 3 | """ 4 | 5 | import importlib 6 | import inspect 7 | import logging 8 | import typing as t 9 | from fnmatch import fnmatchcase 10 | from types import ModuleType 11 | import os 12 | import pkgutil 13 | 14 | from plux import PluginFinder, PluginSpecResolver, PluginSpec 15 | 16 | LOG = logging.getLogger(__name__) 17 | 18 | 19 | class PackageFinder: 20 | """ 21 | Generate a list of Python packages. How these are generated depends on the implementation. 22 | 23 | Why this abstraction? The naive way to find packages is to list all directories with an ``__init__.py`` file. 24 | However, this approach does not work for distributions that have namespace packages. How do we know whether 25 | something is a namespace package or just a directory? Typically, the build configuration will tell us. For example, 26 | setuptools has the following directives for ``pyproject.toml``:: 27 | 28 | [tool.setuptools] 29 | packages = ["mypkg", "mypkg.subpkg1", "mypkg.subpkg2"] 30 | 31 | Or in hatch:: 32 | 33 | [tool.hatch.build.targets.wheel] 34 | packages = ["src/foo"] 35 | 36 | So this abstraction allows us to use the build tool internals to generate a list of packages that we should be 37 | scanning for plugins. 38 | """ 39 | 40 | def find_packages(self) -> t.Iterable[str]: 41 | """ 42 | Returns an Iterable of Python packages. Each item is a string-representation of a Python package (for example, 43 | ``plux.core``, ``myproject.mypackage.utils``, ...) 44 | 45 | :return: An Iterable of Packages 46 | """ 47 | raise NotImplementedError 48 | 49 | @property 50 | def path(self) -> str: 51 | """ 52 | The root file path under which the packages are located. 53 | 54 | :return: A file path 55 | """ 56 | raise NotImplementedError 57 | 58 | 59 | class PluginFromPackageFinder(PluginFinder): 60 | """ 61 | Finds Plugins from packages that are resolved by the given ``PackageFinder``. Under the hood this uses a 62 | ``ModuleScanningPluginFinder``, which, for each package returned by the ``PackageFinder``, imports the package using 63 | ``importlib``, and scans the module for plugins. 64 | """ 65 | 66 | finder: PackageFinder 67 | 68 | def __init__(self, finder: PackageFinder): 69 | self.finder = finder 70 | 71 | def find_plugins(self) -> list[PluginSpec]: 72 | collector = ModuleScanningPluginFinder(self._load_modules()) 73 | return collector.find_plugins() 74 | 75 | def _load_modules(self) -> t.Generator[ModuleType, None, None]: 76 | """ 77 | Generator to load all imported modules that are part of the packages returned by the ``PackageFinder``. 78 | 79 | :return: A generator of python modules 80 | """ 81 | for module_name in self._list_module_names(): 82 | try: 83 | yield importlib.import_module(module_name) 84 | except Exception as e: 85 | LOG.error("error importing module %s: %s", module_name, e) 86 | 87 | def _list_module_names(self) -> set[str]: 88 | """ 89 | This method creates a set of module names by iterating over the packages detected by the ``PackageFinder``. It 90 | includes top-level packages, as well as submodules found within those packages. 91 | 92 | :return: A set of strings where each string represents a module name. 93 | """ 94 | # adapted from https://stackoverflow.com/a/54323162/804840 95 | 96 | modules = set() 97 | 98 | for pkg in self.finder.find_packages(): 99 | modules.add(pkg) 100 | pkgpath = self.finder.path.rstrip(os.sep) + os.sep + pkg.replace(".", os.sep) 101 | for info in pkgutil.iter_modules([pkgpath]): 102 | if not info.ispkg: 103 | modules.add(pkg + "." + info.name) 104 | 105 | return modules 106 | 107 | 108 | class ModuleScanningPluginFinder(PluginFinder): 109 | """ 110 | A PluginFinder that scans the members of given modules for available PluginSpecs. Each member is evaluated with a 111 | PluginSpecResolver, and all successful calls resulting in a PluginSpec are collected and returned. This is used 112 | at build time to scan all modules that the build tool finds, and is an expensive solution that would not be 113 | practical at runtime. 114 | """ 115 | 116 | def __init__(self, modules: t.Iterable[ModuleType], resolver: PluginSpecResolver = None) -> None: 117 | super().__init__() 118 | self.modules = modules 119 | self.resolver = resolver or PluginSpecResolver() 120 | 121 | def find_plugins(self) -> list[PluginSpec]: 122 | plugins = list() 123 | 124 | for module in self.modules: 125 | LOG.debug("scanning module %s, file=%s", module.__name__, module.__file__) 126 | members = inspect.getmembers(module) 127 | 128 | for member in members: 129 | if type(member) is tuple: 130 | try: 131 | spec = self.resolver.resolve(member[1]) 132 | plugins.append(spec) 133 | LOG.debug("found plugin spec in %s:%s %s", module.__name__, member[0], spec) 134 | except Exception: 135 | pass 136 | 137 | return plugins 138 | 139 | 140 | class Filter: 141 | """ 142 | Given a list of patterns, create a callable that will be true only if 143 | the input matches at least one of the patterns. 144 | This is from `setuptools.discovery._Filter` 145 | """ 146 | 147 | def __init__(self, patterns: t.Iterable[str]): 148 | self._patterns = patterns 149 | 150 | def __call__(self, item: str): 151 | return any(fnmatchcase(item, pat) for pat in self._patterns) 152 | 153 | 154 | class MatchAllFilter(Filter): 155 | """ 156 | Filter that is equivalent to ``_Filter(["*"])``. 157 | """ 158 | 159 | def __init__(self): 160 | super().__init__([]) 161 | 162 | def __call__(self, item: str): 163 | return True 164 | -------------------------------------------------------------------------------- /plux/runtime/metadata.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module to provide easier high-level access to ``importlib.metadata`` in the context of plugins. 3 | """ 4 | 5 | import collections 6 | import inspect 7 | import io 8 | import os 9 | import typing as t 10 | from functools import lru_cache 11 | from importlib import metadata 12 | 13 | from plux.core.entrypoint import EntryPointDict, to_entry_point_dict 14 | from plux.core.plugin import PluginSpec 15 | 16 | Distribution = metadata.Distribution 17 | 18 | 19 | def metadata_packages_distributions(): 20 | """Wrapper around ``importlib.metadata.packages_distributions``, which returns a mapping of top-level packages.""" 21 | return metadata.packages_distributions() 22 | 23 | 24 | @lru_cache() 25 | def packages_distributions() -> t.Mapping[str, list[str]]: 26 | """ 27 | Cache wrapper around ``metadata.packages_distributions``, which returns a mapping of top-level packages to 28 | their distributions. This index is created by scanning all ``top_level.txt`` metadata files in the path. 29 | 30 | Unlike ``metadata.packages_distributions``, this implementation will contain a de-duplicated list of 31 | distribution names per package. With editable installs, ``packages_distributions()`` may return the 32 | distribution metadata for both the ``.egg-info`` in the linked source directory, and the ``.dist-info`` 33 | in the site-packages directory created by the editable install. Therefore, a distribution name may 34 | appear twice for the same package, which is unhelpful information, so we deduplicate it here. 35 | 36 | :return: package to distribution mapping 37 | """ 38 | distributions = dict(metadata_packages_distributions()) 39 | 40 | for k in distributions.keys(): 41 | # remove duplicates occurrences of the same distribution name 42 | distributions[k] = list(set(distributions[k])) 43 | 44 | return distributions 45 | 46 | 47 | def resolve_distribution_information(plugin_spec: PluginSpec) -> Distribution | None: 48 | """ 49 | Resolves for a PluginSpec the python distribution package it comes from. Currently, this raises an 50 | error for plugins that come from a namespace package (i.e., when a package is part of multiple 51 | distributions). 52 | 53 | :param plugin_spec: the plugin spec to resolve 54 | :return: the Distribution metadata if it exists 55 | """ 56 | package = inspect.getmodule(plugin_spec.factory).__name__ 57 | root_package = package.split(".")[0] 58 | distributions = packages_distributions().get(root_package) 59 | if not distributions: 60 | return None 61 | if len(distributions) > 1: 62 | raise ValueError("cannot deal with plugins that are part of namespace packages") 63 | 64 | return metadata.distribution(distributions[0]) 65 | 66 | 67 | def entry_points_from_metadata_path(metadata_path: str) -> EntryPointDict: 68 | """ 69 | Reads the entry_points.txt from a distribution meta dir (e.g., the .egg-info or .dist-info directory). 70 | """ 71 | dist = Distribution.at(metadata_path) 72 | return to_entry_point_dict(dist.entry_points) 73 | 74 | 75 | def resolve_entry_points( 76 | distributions: t.Iterable[Distribution] = None, 77 | ) -> list[metadata.EntryPoint]: 78 | """ 79 | Resolves all entry points using a combination of ``importlib.metadata``, and also follows entry points 80 | links in ``entry_points_editable.txt`` created by plux while building editable wheels. 81 | 82 | :return: the list of unique entry points 83 | """ 84 | entry_points = [] 85 | distributions = distributions or metadata.distributions() 86 | 87 | for dist in distributions: 88 | # this is a distribution that was installed as editable, therefore we follow the link created 89 | # by plux during the editable_wheel command 90 | entry_points_path = dist.read_text("entry_points_editable.txt") 91 | 92 | if entry_points_path and os.path.exists(entry_points_path): 93 | with open(entry_points_path, "r") as fd: 94 | editable_eps = parse_entry_points_text(fd.read()) 95 | entry_points.extend(editable_eps) 96 | else: 97 | entry_points.extend(dist.entry_points) 98 | 99 | # unique filter but preserving order 100 | seen = set() 101 | unique = [] 102 | for ep in entry_points: 103 | key = (ep.name, ep.value, ep.group) 104 | if key in seen: 105 | continue 106 | seen.add(key) 107 | unique.append(ep) 108 | 109 | return unique 110 | 111 | 112 | def build_entry_point_index( 113 | entry_points: t.Iterable[metadata.EntryPoint], 114 | ) -> dict[str, list[metadata.EntryPoint]]: 115 | """ 116 | Organizes the given list of entry points into a dictionary that maps entry point groups to their 117 | respective entry points, which resembles the data structure of an ``entry_points.txt``. 118 | 119 | :param entry_points: 120 | :return: 121 | """ 122 | result = collections.defaultdict(list) 123 | names = collections.defaultdict(set) # book-keeping to check duplicates 124 | for ep in entry_points: 125 | if ep.name in names[ep.group]: 126 | continue 127 | result[ep.group].append(ep) 128 | names[ep.group].add(ep.name) 129 | return dict(result) 130 | 131 | 132 | def parse_entry_points_text(text: str) -> list[metadata.EntryPoint]: 133 | """ 134 | Parses the content of an ``entry_points.txt`` into a list of entry point objects. 135 | 136 | :param text: the string to parse 137 | :return: a list of metadata EntryPoint objects 138 | """ 139 | return metadata.EntryPoints._from_text(text) 140 | 141 | 142 | def serialize_entry_points_text(index: dict[str, list[metadata.EntryPoint]]) -> str: 143 | """ 144 | Serializes an entry point index generated via ``build_entry_point_index`` into a string that can be 145 | written into an ``entry_point.txt``. Example:: 146 | 147 | [console_scripts] 148 | wheel = wheel.cli:main 149 | 150 | [distutils.commands] 151 | bdist_egg = setuptools.command.bdist_egg:bdist_egg 152 | 153 | 154 | :param index: the index to serialize 155 | :return: the serialized string 156 | """ 157 | 158 | buffer = io.StringIO() 159 | groups = sorted(index.keys()) 160 | for group in groups: 161 | buffer.write(f"[{group}]\n") 162 | buffer.writelines(f"{ep.name} = {ep.value}\n" for ep in index[group]) 163 | buffer.write("\n") 164 | return buffer.getvalue() 165 | 166 | 167 | class EntryPointsResolver: 168 | """ 169 | Interface for something that builds an entry point index. 170 | """ 171 | 172 | def get_entry_points(self) -> dict[str, list[metadata.EntryPoint]]: 173 | raise NotImplementedError 174 | 175 | 176 | class MetadataEntryPointsResolver(EntryPointsResolver): 177 | """ 178 | Implementation that uses regular ``importlib.metadata`` methods to resolve entry points. 179 | """ 180 | 181 | def get_entry_points(self) -> dict[str, list[metadata.EntryPoint]]: 182 | return build_entry_point_index(resolve_entry_points(metadata.distributions())) 183 | -------------------------------------------------------------------------------- /plux/runtime/cache.py: -------------------------------------------------------------------------------- 1 | import errno 2 | import glob 3 | import hashlib 4 | import os 5 | import platform 6 | import struct 7 | import sys 8 | import threading 9 | from importlib import metadata 10 | from pathlib import Path 11 | 12 | from .metadata import ( 13 | EntryPointsResolver, 14 | MetadataEntryPointsResolver, 15 | build_entry_point_index, 16 | parse_entry_points_text, 17 | serialize_entry_points_text, 18 | ) 19 | 20 | 21 | def get_user_cache_dir() -> Path: 22 | """ 23 | Returns the path of the user's cache dir (e.g., ~/.cache on Linux, or ~/Library/Caches on Mac). 24 | 25 | :return: a Path pointing to the platform-specific cache dir of the user 26 | """ 27 | 28 | if "windows" == platform.system().lower(): 29 | return Path(os.path.expandvars(r"%LOCALAPPDATA%\cache")) 30 | if "darwin" == platform.system().lower(): 31 | return Path.home() / "Library" / "Caches" 32 | if "linux" == platform.system().lower(): 33 | string_path = os.environ.get("XDG_CACHE_HOME") 34 | if string_path and os.path.isabs(string_path): 35 | return Path(string_path) 36 | 37 | # Use the common place to store caches in Linux as a default 38 | return Path.home() / ".cache" 39 | 40 | 41 | def _get_mtime(path: str) -> float: 42 | try: 43 | s = os.stat(path) 44 | return s.st_mtime 45 | except OSError as err: 46 | if err.errno not in {errno.ENOENT, errno.ENOTDIR}: 47 | raise 48 | return -1.0 49 | 50 | 51 | def _float_to_bytes(f: float) -> bytes: 52 | return struct.Struct("f").pack(f) 53 | 54 | 55 | class EntryPointsCache(EntryPointsResolver): 56 | """ 57 | Special ``EntryPointsResolver`` that uses ``importlib.metadata`` and a file system cache. The basic and 58 | hash key calculation idea is taken from stevedore, but entry point parsing/serializing is simplified 59 | slightly. Instead of writing json files, that includes metadata, we instead write one giant 60 | ``entry_points.txt`` compatible file. 61 | 62 | You can get a singleton instance via ``EntryPointsCache.instance()`` that also keeps an in-memory cache 63 | for the current ``sys.path``. 64 | """ 65 | 66 | _instance: "EntryPointsCache" = None 67 | _instance_lock: threading.RLock = threading.RLock() 68 | 69 | _cache: dict[tuple[str, ...], dict[str, list[metadata.EntryPoint]]] 70 | """For each specific path (like sys.path), we store a dict of group -> [entry point].""" 71 | 72 | def __init__(self): 73 | self._lock = threading.RLock() 74 | self._cache = {} 75 | self._resolver = MetadataEntryPointsResolver() 76 | self._cache_dir = get_user_cache_dir() / "plux" 77 | 78 | def get_entry_points(self) -> dict[str, list[metadata.EntryPoint]]: 79 | """ 80 | Returns a dictionary of entry points for the current ``sys.path``. 81 | """ 82 | path = sys.path 83 | key = tuple(path) 84 | 85 | try: 86 | return self._cache[key] 87 | except KeyError: 88 | pass 89 | 90 | # build the cache within a lock context. there's a potential race condition here when sys.path is 91 | # altered concurrently, but we consider this an unsupported fringe case. 92 | with self._lock: 93 | if key not in self._cache: 94 | # we don't need to pass the sys.path here, since it's used implicitly when calling 95 | # ``metadata.distributions()`` in ``resolve_entry_points``. 96 | self._cache[key] = self._build_and_store_index(self._cache_dir) 97 | 98 | return self._cache[key] 99 | 100 | def _build_and_store_index(self, cache_dir: Path) -> dict[str, list[metadata.EntryPoint]]: 101 | # first, try and load the index from the file system 102 | path_key = self._calculate_hash_key(sys.path) 103 | 104 | file_cache = cache_dir / f"{path_key}.entry_points.txt" 105 | 106 | if file_cache.exists(): 107 | return build_entry_point_index(parse_entry_points_text(file_cache.read_text())) 108 | 109 | # if it doesn't exist, create the index from the current path 110 | index = self._resolver.get_entry_points() 111 | 112 | # store it to disk 113 | if not file_cache.parent.exists(): 114 | file_cache.parent.mkdir(parents=True, exist_ok=True) 115 | file_cache.write_text(serialize_entry_points_text(index)) 116 | 117 | return index 118 | 119 | def _calculate_hash_key(self, path: list[str]): 120 | """ 121 | Calculates a hash of all modified times of all ``entry_point.txt`` files in the path. Basic idea 122 | taken from ``stevedore._cache._hash_settings_for_path``. The main difference to it is that it also 123 | considers entry points files linked through ``entry_points_editable.txt``. 124 | 125 | The hash considers: 126 | * the path executable (python version) 127 | * the path prefix (like a .venv path) 128 | * mtime of each path entry 129 | * mtimes of all entry_point.txt files within the path 130 | * mtimes of all entry_point.txt files linked through entry_points_editable.txt files 131 | 132 | :param path: the path (typically ``sys.path``) 133 | :return: a sha256 hash that hashes the path's entry point state 134 | """ 135 | h = hashlib.sha256() 136 | 137 | # Tie the cache to the python interpreter, in case it is part of a 138 | # virtualenv. 139 | h.update(sys.executable.encode("utf-8")) 140 | h.update(sys.prefix.encode("utf-8")) 141 | 142 | for entry in path: 143 | mtime = _get_mtime(entry) 144 | h.update(entry.encode("utf-8")) 145 | h.update(_float_to_bytes(mtime)) 146 | 147 | for ep_file in self._iter_entry_point_files(entry): 148 | mtime = _get_mtime(ep_file) 149 | h.update(ep_file.encode("utf-8")) 150 | h.update(_float_to_bytes(mtime)) 151 | 152 | return h.hexdigest() 153 | 154 | def _iter_entry_point_files(self, entry: str): 155 | """ 156 | An iterator that returns all entry_point.txt file paths in the given path entry. It transparently 157 | resolves editable entry points links. 158 | 159 | :param entry: a path entry like ``/home/user/myproject/.venv/lib/python3.11/site-packages`` 160 | """ 161 | yield from glob.iglob(os.path.join(entry, "*.dist-info", "entry_points.txt")) 162 | yield from glob.iglob(os.path.join(entry, "*.egg-info", "entry_points.txt")) 163 | 164 | # resolve editable links 165 | for item in glob.iglob(os.path.join(entry, "*.dist-info", "entry_points_editable.txt")): 166 | with open(item, "r") as fd: 167 | link = fd.read() 168 | if os.path.exists(link): 169 | yield link 170 | 171 | @staticmethod 172 | def instance() -> "EntryPointsCache": 173 | if EntryPointsCache._instance: 174 | return EntryPointsCache._instance 175 | 176 | with EntryPointsCache._instance_lock: 177 | if EntryPointsCache._instance: 178 | return EntryPointsCache._instance 179 | 180 | EntryPointsCache._instance = EntryPointsCache() 181 | return EntryPointsCache._instance 182 | -------------------------------------------------------------------------------- /plux/cli/cli.py: -------------------------------------------------------------------------------- 1 | """ 2 | A plux CLI frontend. Currently it heavily relies on setuptools, but should be extended in the future to support other 3 | build backends like hatchling or poetry. 4 | """ 5 | 6 | import argparse 7 | import logging 8 | import os 9 | import sys 10 | 11 | from plux.build import config 12 | from plux.build.project import Project 13 | 14 | LOG = logging.getLogger(__name__) 15 | 16 | 17 | def _get_build_backend() -> str | None: 18 | # TODO: should read this from the project configuration instead somehow. 19 | try: 20 | import setuptools # noqa 21 | 22 | return "setuptools" 23 | except ImportError: 24 | pass 25 | 26 | try: 27 | import hatchling # noqa 28 | 29 | return "hatchling" 30 | except ImportError: 31 | pass 32 | 33 | return None 34 | 35 | 36 | def _load_project(args: argparse.Namespace) -> Project: 37 | backend = _get_build_backend() 38 | workdir = args.workdir 39 | 40 | if args.verbose: 41 | print(f"loading project config from {workdir}, determined build backend is: {backend}") 42 | 43 | if backend == "setuptools": 44 | from plux.build.setuptools import SetuptoolsProject 45 | 46 | return SetuptoolsProject(workdir) 47 | elif backend == "hatchling": 48 | raise NotImplementedError("Hatchling is not yet supported as build backend") 49 | else: 50 | raise RuntimeError( 51 | "No supported build backend found. Plux needs either setuptools or hatchling to work." 52 | ) 53 | 54 | 55 | def entrypoints(args: argparse.Namespace): 56 | project = _load_project(args) 57 | project.config = project.config.merge( 58 | exclude=args.exclude.split(",") if args.exclude else None, 59 | include=args.include.split(",") if args.include else None, 60 | ) 61 | cfg = project.config 62 | 63 | print(f"entry point build mode: {cfg.entrypoint_build_mode.value}") 64 | 65 | if cfg.entrypoint_build_mode == config.EntrypointBuildMode.BUILD_HOOK: 66 | print("discovering plugins and building entrypoints automatically...") 67 | project.build_entrypoints() 68 | elif cfg.entrypoint_build_mode == config.EntrypointBuildMode.MANUAL: 69 | path = os.path.join(os.getcwd(), cfg.entrypoint_static_file) 70 | print(f"discovering plugins and writing to {path} ...") 71 | builder = project.create_plugin_index_builder() 72 | with open(path, "w") as fd: 73 | builder.write(fd, output_format="ini") 74 | 75 | 76 | def discover(args: argparse.Namespace): 77 | project = _load_project(args) 78 | project.config = project.config.merge( 79 | path=args.path, 80 | exclude=args.exclude.split(",") if args.exclude else None, 81 | include=args.include.split(",") if args.include else None, 82 | ) 83 | 84 | builder = project.create_plugin_index_builder() 85 | builder.write(fp=args.output, output_format=args.format) 86 | 87 | 88 | def show(args: argparse.Namespace): 89 | project = _load_project(args) 90 | 91 | try: 92 | entrypoints_file = project.find_entry_point_file() 93 | except FileNotFoundError as e: 94 | print(f"Entrypoints file could not be located: {e}") 95 | return 96 | 97 | if not entrypoints_file.exists(): 98 | print(f"No entrypoints file found at {entrypoints_file}, nothing to show") 99 | return 100 | 101 | print(entrypoints_file.read_text()) 102 | 103 | 104 | def resolve(args): 105 | for p in sys.path: 106 | print(f"path = {p}") 107 | from plux import PluginManager 108 | 109 | manager = PluginManager(namespace=args.namespace) 110 | 111 | for spec in manager.list_plugin_specs(): 112 | print(f"{spec.namespace}:{spec.name} = {spec.factory.__module__}:{spec.factory.__name__}") 113 | 114 | 115 | def main(argv=None): 116 | parser = argparse.ArgumentParser(description="Plux CLI frontend") 117 | parser.add_argument( 118 | "--workdir", 119 | type=str, 120 | default=os.getcwd(), 121 | help="overwrite the working directory", 122 | ) 123 | parser.add_argument("-v", "--verbose", action="store_true", help="enable verbose logging") 124 | subparsers = parser.add_subparsers(title="commands", dest="command", help="Available commands") 125 | 126 | # Subparser for the 'generate' subcommand 127 | generate_parser = subparsers.add_parser("entrypoints", help="Discover plugins and generate entry points") 128 | generate_parser.add_argument( 129 | "-e", 130 | "--exclude", 131 | help="a sequence of paths to exclude; '*' can be used as a wildcard in the names. 'foo.*' will exclude all subpackages of 'foo' (but not 'foo' itself).", 132 | ) 133 | generate_parser.add_argument( 134 | "-i", 135 | "--include", 136 | help="a sequence of paths to include; If it's specified, only the named items will be included. If it's not " 137 | "specified, all found items in the path will be included. 'include' can contain shell style wildcard " 138 | "patterns just like 'exclude'", 139 | ) 140 | generate_parser.set_defaults(func=entrypoints) 141 | 142 | # Subparser for the 'discover' subcommand 143 | discover_parser = subparsers.add_parser("discover", help="Discover plugins and print them") 144 | discover_parser.add_argument( 145 | "-p", 146 | "--path", 147 | help="the file path where to look for plugins'", 148 | ) 149 | discover_parser.add_argument( 150 | "-e", 151 | "--exclude", 152 | help="a sequence of paths to exclude; '*' can be used as a wildcard in the names. 'foo.*' will exclude all subpackages of 'foo' (but not 'foo' itself).", 153 | ) 154 | discover_parser.add_argument( 155 | "-i", 156 | "--include", 157 | help="a sequence of paths to include; If it's specified, only the named items will be included. If it's not " 158 | "specified, all found items in the path will be included. 'include' can contain shell style wildcard " 159 | "patterns just like 'exclude'", 160 | ) 161 | discover_parser.add_argument( 162 | "-f", 163 | "--format", 164 | help="the format in which to output the entrypoints. can be 'json' or 'ini', defaults to 'dict'.", 165 | default="json", 166 | choices=["json", "ini"], 167 | ) 168 | discover_parser.add_argument( 169 | "-o", 170 | "--output", 171 | type=argparse.FileType("w"), 172 | default=sys.stdout, 173 | help="Output file path, defaults to stdout", 174 | ) 175 | discover_parser.set_defaults(func=discover) 176 | 177 | # Subparser for the 'resolve' subcommand 178 | resolve_parser = subparsers.add_parser( 179 | "resolve", help="Resolve a plugin namespace and list all its plugins" 180 | ) 181 | resolve_parser.add_argument("--namespace", help="the plugin namespace", required=True) 182 | resolve_parser.set_defaults(func=resolve) 183 | 184 | # Subparser for the 'discover' subcommand 185 | show_parser = subparsers.add_parser("show", help="Show entrypoints that were generated") 186 | show_parser.set_defaults(func=show) 187 | 188 | args = parser.parse_args(argv) 189 | 190 | if args.verbose: 191 | logging.basicConfig(level=logging.DEBUG) 192 | 193 | os.chdir(args.workdir) 194 | 195 | if not hasattr(args, "func"): 196 | parser.print_help() 197 | exit(1) 198 | args.func(args) 199 | 200 | 201 | if __name__ == "__main__": 202 | main() 203 | -------------------------------------------------------------------------------- /tests/cli/test_entrypoints.py: -------------------------------------------------------------------------------- 1 | import glob 2 | import os.path 3 | import shutil 4 | import sys 5 | import tarfile 6 | import zipfile 7 | from pathlib import Path 8 | 9 | import pytest 10 | 11 | 12 | @pytest.mark.parametrize("project_name", ["pyproject", "setupcfg"]) 13 | def test_entrypoints(project_name): 14 | from plux.__main__ import main 15 | 16 | project = os.path.join(os.path.dirname(__file__), "projects", project_name) 17 | os.chdir(project) 18 | 19 | sys.path.append(project) 20 | try: 21 | try: 22 | main(["--workdir", project, "entrypoints"]) 23 | except SystemExit: 24 | pass 25 | finally: 26 | sys.path.remove(project) 27 | 28 | with open(os.path.join(project, "test_project.egg-info", "entry_points.txt"), "r") as f: 29 | lines = [line.strip() for line in f.readlines() if line.strip()] 30 | assert lines == [ 31 | "[plux.test.plugins]", 32 | "mynestedplugin = mysrc.subpkg.plugins:MyNestedPlugin", 33 | "myplugin = mysrc.plugins:MyPlugin", 34 | ] 35 | 36 | # make sure that SOURCES.txt contain no absolute paths 37 | with open(os.path.join(project, "test_project.egg-info", "SOURCES.txt"), "r") as f: 38 | lines = [line.strip() for line in f.readlines() if line.strip()] 39 | for line in lines: 40 | assert not line.startswith("/") 41 | 42 | 43 | @pytest.mark.parametrize("project_name", ["pyproject", "setupcfg"]) 44 | def test_entrypoints_exclude(project_name): 45 | from plux.__main__ import main 46 | 47 | project = os.path.join(os.path.dirname(__file__), "projects", project_name) 48 | os.chdir(project) 49 | 50 | sys.path.append(project) 51 | try: 52 | try: 53 | main(["--workdir", project, "entrypoints", "--exclude", "*/subpkg*"]) 54 | except SystemExit: 55 | pass 56 | finally: 57 | sys.path.remove(project) 58 | 59 | with open(os.path.join(project, "test_project.egg-info", "entry_points.txt"), "r") as f: 60 | lines = [line.strip() for line in f.readlines() if line.strip()] 61 | assert lines == [ 62 | "[plux.test.plugins]", 63 | "myplugin = mysrc.plugins:MyPlugin", 64 | ] 65 | 66 | # make sure that SOURCES.txt contain no absolute paths 67 | with open(os.path.join(project, "test_project.egg-info", "SOURCES.txt"), "r") as f: 68 | lines = [line.strip() for line in f.readlines() if line.strip()] 69 | for line in lines: 70 | assert not line.startswith("/") 71 | 72 | 73 | @pytest.mark.parametrize("project_name", ["pyproject", "setupcfg"]) 74 | def test_entrypoints_include(project_name): 75 | from plux.__main__ import main 76 | 77 | project = os.path.join(os.path.dirname(__file__), "projects", project_name) 78 | os.chdir(project) 79 | 80 | sys.path.append(project) 81 | try: 82 | try: 83 | main(["--workdir", project, "entrypoints", "--include", "*/subpkg*"]) 84 | except SystemExit: 85 | pass 86 | finally: 87 | sys.path.remove(project) 88 | 89 | with open(os.path.join(project, "test_project.egg-info", "entry_points.txt"), "r") as f: 90 | lines = [line.strip() for line in f.readlines() if line.strip()] 91 | assert lines == [ 92 | "[plux.test.plugins]", 93 | "mynestedplugin = mysrc.subpkg.plugins:MyNestedPlugin", 94 | ] 95 | 96 | 97 | def test_entrypoints_exclude_from_pyproject_config(tmp_path): 98 | from plux.__main__ import main 99 | 100 | src_project = os.path.join(os.path.dirname(__file__), "projects", "pyproject") 101 | dest_project = os.path.join(str(tmp_path), "pyproject") 102 | 103 | shutil.copytree(src_project, dest_project) 104 | 105 | pyproject_toml_path = os.path.join(dest_project, "pyproject.toml") 106 | 107 | with open(pyproject_toml_path, "a") as fp: 108 | fp.write('\n[tool.plux]\nexclude = ["*.subpkg.*"]\n') 109 | 110 | os.chdir(dest_project) 111 | 112 | sys.path.append(dest_project) 113 | try: 114 | try: 115 | main(["--workdir", dest_project, "entrypoints"]) 116 | except SystemExit: 117 | pass 118 | finally: 119 | sys.path.remove(dest_project) 120 | 121 | with open(os.path.join(dest_project, "test_project.egg-info", "entry_points.txt"), "r") as f: 122 | lines = [line.strip() for line in f.readlines() if line.strip()] 123 | assert lines == [ 124 | "[plux.test.plugins]", 125 | "myplugin = mysrc.plugins:MyPlugin", 126 | ] 127 | 128 | # make sure that SOURCES.txt contain no absolute paths 129 | with open(os.path.join(dest_project, "test_project.egg-info", "SOURCES.txt"), "r") as f: 130 | lines = [line.strip() for line in f.readlines() if line.strip()] 131 | for line in lines: 132 | assert not line.startswith("/") 133 | 134 | 135 | def test_entrypoints_with_manual_build_mode(): 136 | from build.__main__ import main as build_main 137 | 138 | from plux.__main__ import main 139 | 140 | project = os.path.join(os.path.dirname(__file__), "projects", "manual_build_mode") 141 | os.chdir(project) 142 | 143 | # remove dist and egg info dir from previous runs 144 | shutil.rmtree(os.path.join(project, "dist"), ignore_errors=True) 145 | shutil.rmtree(os.path.join(project, "test_project.egg-info"), ignore_errors=True) 146 | 147 | expected_entry_points = [ 148 | "[plux.test.plugins]", 149 | "mynestedplugin = mysrc.subpkg.plugins:MyNestedPlugin", 150 | "myplugin = mysrc.plugins:MyPlugin", 151 | ] 152 | 153 | sys.path.append(project) 154 | try: 155 | try: 156 | main(["--workdir", project, "entrypoints"]) 157 | except SystemExit: 158 | pass 159 | finally: 160 | sys.path.remove(project) 161 | 162 | # make sure no egg info is created by entrypoints (given manual build mode) 163 | assert list(glob.glob("*egg-info")) == [] 164 | 165 | # instead, it creates a plux.ini 166 | ini_file = Path(project, "plux.ini") 167 | assert ini_file.exists() 168 | 169 | lines = ini_file.read_text().strip().splitlines() 170 | assert lines == expected_entry_points 171 | 172 | # now, build the project, make sure that entrypoints are there 173 | build_main([]) # python -m build 174 | 175 | # now the egg info should exist (created by setuptools) 176 | assert list(glob.glob("*egg-info")) == ["test_project.egg-info"] 177 | 178 | # make sure the entry points are in the egg info 179 | entry_points_txt = Path(project, "test_project.egg-info", "entry_points.txt") 180 | assert entry_points_txt.exists() 181 | 182 | lines = entry_points_txt.read_text().strip().splitlines() 183 | assert lines == expected_entry_points 184 | 185 | # also inspect the generated source distribution, make sure the plux ini is created correctly 186 | with tarfile.open(os.path.join(project, "dist", "test_project-0.1.0.tar.gz")) as tar: 187 | members = tar.getnames() 188 | assert "test_project-0.1.0/plux.ini" in members, "plux.ini should be in the archive" 189 | plux_ini = tar.extractfile("test_project-0.1.0/plux.ini") 190 | assert plux_ini is not None 191 | lines = [line.decode().strip() for line in plux_ini.readlines() if line.strip()] 192 | assert lines == expected_entry_points 193 | 194 | # now inspect the wheel 195 | wheel_file = glob.glob(os.path.join(project, "dist", "test_project-*.whl"))[0] 196 | with zipfile.ZipFile(wheel_file, "r") as zip: 197 | members = zip.namelist() 198 | # find the entry_points.txt in the .dist-info directory 199 | entry_points_file = next(m for m in members if m.endswith(".dist-info/entry_points.txt")) 200 | with zip.open(entry_points_file) as f: 201 | lines = [line.decode().strip() for line in f.readlines() if line.strip()] 202 | assert lines == expected_entry_points 203 | -------------------------------------------------------------------------------- /tests/test_manager.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, List, Tuple 2 | from unittest.mock import MagicMock 3 | 4 | import pytest 5 | 6 | from plux import Plugin, PluginDisabled, PluginFinder, PluginManager, PluginSpec 7 | from plux.runtime.filter import global_plugin_filter 8 | 9 | 10 | class DummyPlugin(Plugin): 11 | load_calls: List[Tuple[Tuple, Dict]] 12 | 13 | def __init__(self) -> None: 14 | super().__init__() 15 | self.load_calls = list() 16 | 17 | def load(self, *args, **kwargs): 18 | self.load_calls.append((args, kwargs)) 19 | 20 | 21 | class ShouldNotLoadPlugin(DummyPlugin): 22 | def should_load(self) -> bool: 23 | return False 24 | 25 | 26 | class GoodPlugin(DummyPlugin): 27 | pass 28 | 29 | 30 | class ThrowsExceptionOnLoadPlugin(DummyPlugin): 31 | def load(self, *args, **kwargs): 32 | super().load(*args, **kwargs) 33 | raise ValueError("controlled load fail") 34 | 35 | 36 | class ThrowsExceptionOnInitPlugin(DummyPlugin): 37 | def __init__(self) -> None: 38 | super().__init__() 39 | raise ValueError("controlled __init__ fail") 40 | 41 | 42 | class DummyPluginFinder(PluginFinder): 43 | def __init__(self, specs: List[PluginSpec]): 44 | self.specs = specs 45 | 46 | def find_plugins(self) -> List[PluginSpec]: 47 | return self.specs 48 | 49 | 50 | @pytest.fixture 51 | def dummy_plugin_finder(): 52 | return DummyPluginFinder( 53 | [ 54 | PluginSpec("test.plugins.dummy", "shouldload", GoodPlugin), 55 | PluginSpec("test.plugins.dummy", "shouldnotload", ShouldNotLoadPlugin), 56 | PluginSpec("test.plugins.dummy", "load_errors", ThrowsExceptionOnLoadPlugin), 57 | PluginSpec("test.plugins.dummy", "init_errors", ThrowsExceptionOnInitPlugin), 58 | PluginSpec("test.plugins.dummy", "shouldalsoload", GoodPlugin), 59 | PluginSpec("test.plugins.others", "shouldload", DummyPlugin), # different namespace 60 | ] 61 | ) 62 | 63 | 64 | class TestPluginManager: 65 | def test_load_all(self, dummy_plugin_finder): 66 | manager = PluginManager("test.plugins.dummy", finder=dummy_plugin_finder) 67 | 68 | assert manager.is_loaded("shouldload") is False 69 | assert manager.is_loaded("shouldalsoload") is False 70 | assert manager.is_loaded("shouldnotload") is False 71 | assert manager.is_loaded("load_errors") is False 72 | assert manager.is_loaded("init_errors") is False 73 | 74 | plugins = manager.load_all() 75 | 76 | assert manager.is_loaded("shouldload") is True 77 | assert manager.is_loaded("shouldalsoload") is True 78 | assert manager.is_loaded("shouldnotload") is False 79 | assert manager.is_loaded("load_errors") is False 80 | assert manager.is_loaded("init_errors") is False 81 | 82 | assert len(plugins) == 2 # shouldload and shouldalsoload 83 | 84 | assert type(plugins[0]) is GoodPlugin 85 | assert type(plugins[1]) is GoodPlugin 86 | 87 | def test_list_names(self, dummy_plugin_finder): 88 | manager = PluginManager("test.plugins.dummy", finder=dummy_plugin_finder) 89 | names = manager.list_names() 90 | 91 | assert len(names) == 5 92 | assert "shouldload" in names 93 | assert "shouldnotload" in names 94 | assert "load_errors" in names 95 | assert "init_errors" in names 96 | assert "shouldalsoload" in names 97 | 98 | def test_exists(self, dummy_plugin_finder): 99 | manager = PluginManager("test.plugins.dummy", finder=dummy_plugin_finder) 100 | 101 | assert manager.exists("shouldload") 102 | assert manager.exists("shouldnotload") 103 | assert manager.exists("load_errors") 104 | assert manager.exists("init_errors") 105 | assert manager.exists("shouldalsoload") 106 | assert not manager.exists("foobar") 107 | 108 | def test_load_all_load_is_only_called_once(self): 109 | finder = DummyPluginFinder( 110 | [ 111 | PluginSpec("test.plugins.dummy", "shouldload", GoodPlugin), 112 | PluginSpec("test.plugins.dummy", "shouldalsoload", GoodPlugin), 113 | ] 114 | ) 115 | 116 | manager: PluginManager[DummyPlugin] = PluginManager( 117 | "test.plugins.dummy", finder=finder, load_kwargs={"foo": "bar"} 118 | ) 119 | 120 | plugins = manager.load_all() 121 | assert len(plugins[0].load_calls) == 1 122 | assert len(plugins[1].load_calls) == 1 123 | assert plugins[0].load_calls[0][1] == {"foo": "bar"} 124 | 125 | plugins = manager.load_all() 126 | assert len(plugins[0].load_calls) == 1 127 | assert len(plugins[1].load_calls) == 1 128 | 129 | def test_load_on_non_existing_plugin(self): 130 | manager = PluginManager("test.plugins.dummy", finder=DummyPluginFinder([])) 131 | 132 | with pytest.raises(ValueError) as ex: 133 | manager.load("foo") 134 | 135 | ex.match("no plugin named foo in namespace test.plugins.dummy") 136 | 137 | def test_load_all_container_has_errors(self, dummy_plugin_finder): 138 | manager = PluginManager("test.plugins.dummy", finder=dummy_plugin_finder) 139 | 140 | c_shouldload = manager.get_container("shouldload") 141 | c_shouldnotload = manager.get_container("shouldnotload") 142 | c_load_errors = manager.get_container("load_errors") 143 | c_init_errors = manager.get_container("init_errors") 144 | c_shouldalsoload = manager.get_container("shouldalsoload") 145 | 146 | manager.load_all() 147 | 148 | assert c_shouldload.init_error is None 149 | assert c_shouldnotload.init_error is None 150 | assert type(c_init_errors.init_error) is ValueError 151 | assert c_load_errors.init_error is None 152 | assert c_shouldalsoload.init_error is None 153 | 154 | assert c_shouldload.load_error is None 155 | assert c_shouldnotload.load_error is None 156 | assert c_init_errors.load_error is None 157 | assert type(c_load_errors.load_error) is ValueError 158 | assert c_shouldalsoload.load_error is None 159 | 160 | def test_load_all_propagate_exception(self): 161 | manager = PluginManager( 162 | "test.plugins.dummy", 163 | finder=DummyPluginFinder( 164 | [ 165 | PluginSpec("test.plugins.dummy", "load_errors", ThrowsExceptionOnLoadPlugin), 166 | ] 167 | ), 168 | ) 169 | 170 | with pytest.raises(ValueError) as ex: 171 | manager.load_all(propagate_exceptions=True) 172 | 173 | ex.match("controlled load fail") 174 | 175 | def test_load_disabled_plugin(self, dummy_plugin_finder): 176 | manager = PluginManager("test.plugins.dummy", finder=dummy_plugin_finder) 177 | 178 | with pytest.raises(PluginDisabled) as ex: 179 | manager.load("shouldnotload") 180 | 181 | assert ex.value.namespace == "test.plugins.dummy" 182 | assert ex.value.name == "shouldnotload" 183 | 184 | def test_lifecycle_listener(self, dummy_plugin_finder): 185 | listener = MagicMock() 186 | 187 | manager = PluginManager("test.plugins.dummy", finder=dummy_plugin_finder, listener=listener) 188 | manager.load_all() 189 | 190 | assert listener.on_init_after.call_count == 4 191 | assert listener.on_init_exception.call_count == 1 192 | assert listener.on_load_after.call_count == 2 193 | 194 | container = manager.get_container("shouldalsoload") 195 | listener.on_load_after.assert_called_with(container.plugin_spec, container.plugin, None) 196 | 197 | 198 | class TestGlobalPluginFilter: 199 | def test_disable_namespace(self, dummy_plugin_finder): 200 | manager = PluginManager("test.plugins.dummy", finder=dummy_plugin_finder) 201 | 202 | global_plugin_filter.add_exclusion(namespace="test.plugins.*") 203 | 204 | manager.load_all() 205 | assert manager.is_loaded("shouldload") is False 206 | assert manager.is_loaded("shouldalsoload") is False 207 | 208 | global_plugin_filter.exclusions.clear() 209 | 210 | def test_non_matching_namespace(self, dummy_plugin_finder): 211 | manager = PluginManager("test.plugins.dummy", finder=dummy_plugin_finder) 212 | 213 | global_plugin_filter.add_exclusion(namespace="test.plugins.dummy.*") 214 | 215 | manager.load_all() 216 | assert manager.is_loaded("shouldload") is True 217 | assert manager.is_loaded("shouldalsoload") is True 218 | global_plugin_filter.exclusions.clear() 219 | 220 | def test_disable_name(self, dummy_plugin_finder): 221 | manager = PluginManager("test.plugins.dummy", finder=dummy_plugin_finder) 222 | 223 | global_plugin_filter.add_exclusion(name="*also*") 224 | 225 | manager.load_all() 226 | assert manager.is_loaded("shouldload") is True 227 | assert manager.is_loaded("shouldalsoload") is False 228 | global_plugin_filter.exclusions.clear() 229 | 230 | def test_disable_value(self, dummy_plugin_finder): 231 | manager = PluginManager("test.plugins.dummy", finder=dummy_plugin_finder) 232 | 233 | global_plugin_filter.add_exclusion(value="tests.test_manager:*") 234 | 235 | manager.load_all() 236 | assert manager.is_loaded("shouldload") is False 237 | assert manager.is_loaded("shouldalsoload") is False 238 | global_plugin_filter.exclusions.clear() 239 | -------------------------------------------------------------------------------- /plux/core/plugin.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import functools 3 | import inspect 4 | import typing as t 5 | 6 | 7 | class PluginException(Exception): 8 | def __init__(self, message, namespace: str = None, name: str = None) -> None: 9 | super().__init__(message) 10 | self.namespace = namespace 11 | self.name = name 12 | 13 | 14 | class PluginDisabled(PluginException): 15 | reason: str 16 | 17 | def __init__(self, namespace: str, name: str, reason: str = None): 18 | message = f"plugin {namespace}:{name} is disabled" 19 | if reason: 20 | message = f"{message}, reason: {reason}" 21 | super(PluginDisabled, self).__init__(message) 22 | self.namespace = namespace 23 | self.name = name 24 | self.reason = reason 25 | 26 | 27 | class Plugin(abc.ABC): 28 | """A generic LocalStack plugin. 29 | 30 | A Plugin's purpose is to be loaded dynamically at runtime and defer code imports into the Plugin::load method. 31 | Abstract subtypes of plugins (e.g., a LocalstackCliPlugin) may overwrite the load method with concrete call 32 | arguments that they agree upon with the PluginManager. In other words, a Plugin and a PluginManager for that 33 | particular Plugin have an informal contracts to use the same argument types when load is invoked. 34 | """ 35 | 36 | namespace: str 37 | name: str 38 | requirements: list[str] 39 | 40 | def should_load(self) -> bool: 41 | return True 42 | 43 | def load(self, *args, **kwargs): 44 | """ 45 | Called by a PluginLoader when it loads the Plugin. 46 | """ 47 | return None 48 | 49 | 50 | PluginType = t.Type[Plugin] 51 | PluginFactory = t.Callable[[], Plugin] 52 | 53 | 54 | class PluginSpec: 55 | """ 56 | A PluginSpec describes a plugin through a namespace and its unique name within in that namespace, and holds the 57 | imported code that can instantiate the plugin (a PluginFactory). In the simplest case, the PluginFactory that can 58 | just be the Plugin's class. 59 | 60 | Internally a PluginSpec is essentially a wrapper around an importlib EntryPoint. An entrypoint is a tuple: ( 61 | "name", "module:object") inside a namespace that can be loaded. The entrypoint object of a Plugin can point to a 62 | PluginSpec, or a Plugin that defines its own namespace and name, in which case the PluginSpec will be instantiated 63 | dynamically by, e.g., a PluginSpecResolver. 64 | """ 65 | 66 | namespace: str 67 | name: str 68 | factory: PluginFactory 69 | 70 | def __init__( 71 | self, 72 | namespace: str, 73 | name: str, 74 | factory: PluginFactory, 75 | ) -> None: 76 | super().__init__() 77 | self.namespace = namespace 78 | self.name = name 79 | self.factory = factory 80 | 81 | def __str__(self): 82 | return "PluginSpec(%s.%s = %s)" % (self.namespace, self.name, self.factory) 83 | 84 | def __repr__(self): 85 | return self.__str__() 86 | 87 | def __eq__(self, other): 88 | return self.namespace == other.namespace and self.name == other.name and self.factory == other.factory 89 | 90 | 91 | class PluginFinder(abc.ABC): 92 | """ 93 | Basic abstractions to find plugins, either at build time (e.g., using the PackagePathPluginFinder) or at run time 94 | (e.g., using ``MetadataPluginFinder`` that finds plugins from entrypoints) 95 | """ 96 | 97 | def find_plugins(self) -> list[PluginSpec]: 98 | raise NotImplementedError # pragma: no cover 99 | 100 | 101 | class PluginSpecResolver: 102 | """ 103 | A PluginSpecResolver finds or creates PluginSpec instances from sources, e.g., from analyzing a Plugin class. 104 | """ 105 | 106 | def resolve(self, source: t.Any) -> PluginSpec: 107 | """ 108 | Tries to create a PluginSpec from the given source. 109 | 110 | :param source: anything that can produce a PluginSpec (Plugin class, ...) 111 | :return: a PluginSpec instance 112 | """ 113 | if isinstance(source, PluginSpec): 114 | return source 115 | 116 | if inspect.isclass(source): 117 | if issubclass(source, Plugin): 118 | return PluginSpec(source.namespace, source.name, source) 119 | 120 | if inspect.isfunction(source): 121 | spec = getattr(source, "__pluginspec__", None) 122 | if spec and isinstance(spec, PluginSpec): 123 | return spec 124 | 125 | # TODO: add more options to specify plugin specs 126 | 127 | raise ValueError("cannot resolve plugin specification from %s" % source) 128 | 129 | 130 | class PluginLifecycleListener: # pragma: no cover 131 | """ 132 | Listener that can be attached to a PluginManager to react to plugin lifecycle events. 133 | """ 134 | 135 | def on_resolve_exception(self, namespace: str, entrypoint, exception: Exception): 136 | pass 137 | 138 | def on_resolve_after(self, plugin_spec: PluginSpec): 139 | pass 140 | 141 | def on_init_exception(self, plugin_spec: PluginSpec, exception: Exception): 142 | pass 143 | 144 | def on_init_after(self, plugin_spec: PluginSpec, plugin: Plugin): 145 | pass 146 | 147 | def on_load_before( 148 | self, 149 | plugin_spec: PluginSpec, 150 | plugin: Plugin, 151 | load_args: list | tuple, 152 | load_kwargs: dict, 153 | ): 154 | pass 155 | 156 | def on_load_after(self, plugin_spec: PluginSpec, plugin: Plugin, load_result: t.Any = None): 157 | pass 158 | 159 | def on_load_exception(self, plugin_spec: PluginSpec, plugin: Plugin, exception: Exception): 160 | pass 161 | 162 | 163 | class CompositePluginLifecycleListener(PluginLifecycleListener): 164 | """A PluginLifecycleListener decorator that dispatches to multiple delegates.""" 165 | 166 | listeners: list[PluginLifecycleListener] 167 | 168 | def __init__(self, initial: t.Iterable[PluginLifecycleListener] = None): 169 | self.listeners = list(initial) if initial else [] 170 | 171 | def add_listener(self, listener: PluginLifecycleListener): 172 | self.listeners.append(listener) 173 | 174 | def on_resolve_exception(self, *args, **kwargs): 175 | for listener in self.listeners: 176 | listener.on_resolve_exception(*args, **kwargs) 177 | 178 | def on_resolve_after(self, *args, **kwargs): 179 | for listener in self.listeners: 180 | listener.on_resolve_after(*args, **kwargs) 181 | 182 | def on_init_exception(self, *args, **kwargs): 183 | for listener in self.listeners: 184 | listener.on_init_exception(*args, **kwargs) 185 | 186 | def on_init_after(self, *args, **kwargs): 187 | for listener in self.listeners: 188 | listener.on_init_after(*args, **kwargs) 189 | 190 | def on_load_before(self, *args, **kwargs): 191 | for listener in self.listeners: 192 | listener.on_load_before(*args, **kwargs) 193 | 194 | def on_load_after(self, *args, **kwargs): 195 | for listener in self.listeners: 196 | listener.on_load_after(*args, **kwargs) 197 | 198 | def on_load_exception(self, *args, **kwargs): 199 | for listener in self.listeners: 200 | listener.on_load_exception(*args, **kwargs) 201 | 202 | 203 | class FunctionPlugin(Plugin): 204 | """ 205 | Exposes a function as a Plugin. 206 | """ 207 | 208 | fn: t.Callable 209 | 210 | def __init__( 211 | self, 212 | fn: t.Callable, 213 | should_load: bool | t.Callable[[], bool] = None, 214 | load: t.Callable = None, 215 | ) -> None: 216 | super().__init__() 217 | self.fn = fn 218 | self._should_load = should_load 219 | self._load = load 220 | 221 | def __call__(self, *args, **kwargs): 222 | return self.fn(*args, **kwargs) 223 | 224 | def load(self, *args, **kwargs): 225 | if self._load: 226 | return self._load(*args, **kwargs) 227 | 228 | def should_load(self) -> bool: 229 | if self._should_load is not None: 230 | if type(self._should_load) is bool: 231 | return self._should_load 232 | else: 233 | return self._should_load() 234 | 235 | return True 236 | 237 | 238 | def plugin( 239 | namespace, 240 | name=None, 241 | should_load: bool | t.Callable[[], bool] = None, 242 | load: t.Callable = None, 243 | ): 244 | """ 245 | Expose a function as discoverable and loadable FunctionPlugin. 246 | 247 | :param namespace: the plugin namespace 248 | :param name: the name of the plugin (by default the function name will be used) 249 | :param should_load: optional either a boolean value or a callable returning a boolean 250 | :param load: optional load function 251 | :return: plugin decorator 252 | """ 253 | 254 | def wrapper(fn): 255 | plugin_name = name or fn.__name__ 256 | 257 | # this causes the plugin framework to point the entrypoint to the original function rather than the 258 | # nested factory function (which would not be resolvable) 259 | @functools.wraps(fn) 260 | def factory(): 261 | fn_plugin = FunctionPlugin(fn, should_load=should_load, load=load) 262 | fn_plugin.namespace = namespace 263 | fn_plugin.name = plugin_name 264 | return fn_plugin 265 | 266 | # at discovery-time the factory will point to the method being decorated, and at load-time the factory from 267 | # this spec instance be used instead of the one being created 268 | fn.__pluginspec__ = PluginSpec(namespace, plugin_name, factory) 269 | 270 | return fn 271 | 272 | return wrapper 273 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Plux 2 | ==== 3 | 4 |

5 | CI badge 6 | PyPI Version 7 | PyPI License 8 | Code style: ruff 9 |

10 | 11 | plux is the dynamic code loading framework used in [LocalStack](https://github.com/localstack/localstack). 12 | 13 | 14 | Overview 15 | -------- 16 | 17 | Plux builds a higher-level plugin mechanism around [Python's entry point mechanism](https://packaging.python.org/specifications/entry-points/). 18 | It provides tools to load plugins from entry points at run time, and to discover entry points from plugins at build time (so you don't have to declare entry points statically in your `setup.py`). 19 | 20 | ### Core concepts 21 | 22 | * `PluginSpec`: describes a `Plugin`. Each plugin has a namespace, a unique name in that namespace, and a `PluginFactory` (something that creates `Plugin` the spec is describing. 23 | In the simplest case, that can just be the Plugin's class). 24 | * `Plugin`: an object that exposes a `should_load` and `load` method. 25 | Note that it does not function as a domain object (it does not hold the plugins lifecycle state, like initialized, loaded, etc..., or other metadata of the Plugin) 26 | * `PluginFinder`: finds plugins, either at build time (by scanning the modules using `pkgutil` and `setuptools`) or at run time (reading entrypoints of the distribution using [importlib](https://docs.python.org/3/library/importlib.metadata.html#entry-points)) 27 | * `PluginManager`: manages the run time lifecycle of a Plugin, which has three states: 28 | * resolved: the entrypoint pointing to the PluginSpec was imported and the `PluginSpec` instance was created 29 | * init: the `PluginFactory` of the `PluginSpec` was successfully invoked 30 | * loaded: the `load` method of the `Plugin` was successfully invoked 31 | 32 | ![architecture](https://raw.githubusercontent.com/localstack/plux/main/docs/plux-architecture.png) 33 | 34 | ### Loading Plugins 35 | 36 | At run time, a `PluginManager` uses a `PluginFinder` that in turn uses importlib to scan the available entrypoints for things that look like a `PluginSpec`. 37 | With `PluginManager.load(name: str)` or `PluginManager.load_all()`, plugins within the namespace that are discoverable in entrypoints can be loaded. 38 | If an error occurs at any state of the lifecycle, the `PluginManager` informs the `PluginLifecycleListener` about it, but continues operating. 39 | 40 | ### Discovering entrypoints 41 | 42 | Plux supports two modes for building entry points: **build-hooks mode** (default) and **manual mode**. 43 | 44 | #### Build-hooks mode (default) 45 | 46 | To build a source distribution and a wheel of your code with your plugins as entrypoints, simply run `python setup.py plugins sdist bdist_wheel`. 47 | If you don't have a `setup.py`, you can use the plux build frontend and run `python -m plux entrypoints`. 48 | 49 | How it works: 50 | For discovering plugins at build time, plux provides a custom setuptools command `plugins`, invoked via `python setup.py plugins`. 51 | The command uses a special `PluginFinder` that collects from the codebase anything that can be interpreted as a `PluginSpec`, and creates from it a plugin index file `plux.json`, that is placed into the `.egg-info` distribution metadata directory. 52 | When a setuptools command is used to create the distribution (e.g., `python setup.py sdist/bdist_wheel/...`), plux finds the `plux.json` plugin index and extends automatically the list of entry points (collected into `.egg-info/entry_points.txt`). 53 | The `plux.json` file becomes a part of the distribution, s.t., the plugins do not have to be discovered every time your distribution is installed elsewhere. 54 | Discovering at build time also works when using `python -m build`, since it calls registered setuptools scripts. 55 | 56 | #### Manual mode 57 | 58 | Manual mode is useful for isolated build environments where dependencies cannot be installed, or when build hooks are not suitable for your build process. 59 | 60 | To enable manual mode, add the following to your `pyproject.toml`: 61 | 62 | ```toml 63 | [tool.plux] 64 | entrypoint_build_mode = "manual" 65 | ``` 66 | 67 | In manual mode, plux does not use build hooks. Instead, you manually generate entry points by running: 68 | 69 | ```bash 70 | python -m plux entrypoints 71 | ``` 72 | 73 | This creates a `plux.ini` file in your working directory with the discovered plugins. You can then include this file in your distribution by configuring your `pyproject.toml`: 74 | 75 | ```toml 76 | [project] 77 | dynamic = ["entry-points"] 78 | 79 | [tool.setuptools.package-data] 80 | "*" = ["plux.ini"] 81 | 82 | [tool.setuptools.dynamic] 83 | entry-points = {file = ["plux.ini"]} 84 | ``` 85 | 86 | You can also manually control the output format and location: 87 | 88 | ```bash 89 | python -m plux discover --format ini --output plux.ini 90 | ``` 91 | 92 | 93 | Examples 94 | -------- 95 | 96 | To build something using the plugin framework, you will first want to introduce a Plugin that does something when it is loaded. 97 | And then, at runtime, you need a component that uses the `PluginManager` to get those plugins. 98 | 99 | ### One class per plugin 100 | 101 | This is the way we went with `LocalstackCliPlugin`. Every plugin class (e.g., `ProCliPlugin`) is essentially a singleton. 102 | This is easy, as the classes are discoverable as plugins. 103 | Simply create a Plugin class with a name and namespace and it will be discovered by the build time `PluginFinder`. 104 | 105 | ```python 106 | from plux import Plugin 107 | 108 | # abstract case (not discovered at build time, missing name) 109 | class CliPlugin(Plugin): 110 | namespace = "my.plugins.cli" 111 | 112 | def load(self, cli): 113 | self.attach(cli) 114 | 115 | def attach(self, cli): 116 | raise NotImplementedError 117 | 118 | # discovered at build time (has a namespace, name, and is a Plugin) 119 | class MyCliPlugin(CliPlugin): 120 | name = "my" 121 | 122 | def attach(self, cli): 123 | # ... attach commands to cli object 124 | 125 | ``` 126 | 127 | now we need a `PluginManager` (which has a generic type) to load the plugins for us: 128 | 129 | ```python 130 | cli = # ... needs to come from somewhere 131 | 132 | manager: PluginManager[CliPlugin] = PluginManager("my.plugins.cli", load_args=(cli,)) 133 | 134 | plugins: List[CliPlugin] = manager.load_all() 135 | 136 | # todo: do stuff with the plugins, if you want/need 137 | # in this example, we simply use the plugin mechanism to run a one-shot function (attach) on a load argument 138 | 139 | ``` 140 | 141 | ### Re-usable plugins 142 | 143 | When you have lots of plugins that are structured in a similar way, we may not want to create a separate Plugin class 144 | for each plugin. Instead we want to use the same `Plugin` class to do the same thing, but use several instances of it. 145 | The `PluginFactory`, and the fact that `PluginSpec` instances defined at module level are discoverable (inpired 146 | by [pluggy](https://github.com/pytest-dev/pluggy)), can be used to achieve that. 147 | 148 | ```python 149 | from plux import Plugin, PluginFactory, PluginSpec 150 | import importlib 151 | 152 | class ServicePlugin(Plugin): 153 | 154 | def __init__(self, service_name): 155 | self.service_name = service_name 156 | self.service = None 157 | 158 | def should_load(self): 159 | return self.service_name in config.SERVICES 160 | 161 | def load(self): 162 | module = importlib.import_module("localstack.services.%s" % self.service_name) 163 | # suppose we define a convention that each service module has a Service class, like moto's `Backend` 164 | self.service = module.Service() 165 | 166 | def service_plugin_factory(name) -> PluginFactory: 167 | def create(): 168 | return ServicePlugin(name) 169 | 170 | return create 171 | 172 | # discoverable 173 | s3 = PluginSpec("localstack.plugins.services", "s3", service_plugin_factory("s3")) 174 | 175 | # discoverable 176 | dynamodb = PluginSpec("localstack.plugins.services", "dynamodb", service_plugin_factory("dynamodb")) 177 | 178 | # ... could be simplified with convenience framework code, but the principle will stay the same 179 | 180 | ``` 181 | 182 | Then we could use the `PluginManager` to build a Supervisor 183 | 184 | ```python 185 | from plux import PluginManager 186 | 187 | class Supervisor: 188 | manager: PluginManager[ServicePlugin] 189 | 190 | def start(self, service_name): 191 | plugin = self.manager.load(service_name) 192 | service = plugin.service 193 | service.start() 194 | 195 | ``` 196 | 197 | ### Functions as plugins 198 | 199 | with the `@plugin` decorator, you can expose functions as plugins. They will be wrapped by the framework 200 | into `FunctionPlugin` instances, which satisfy both the contract of a Plugin, and that of the function. 201 | 202 | ```python 203 | from plux import plugin 204 | 205 | @plugin(namespace="localstack.configurators") 206 | def configure_logging(runtime): 207 | logging.basicConfig(level=runtime.config.loglevel) 208 | 209 | 210 | @plugin(namespace="localstack.configurators") 211 | def configure_somethingelse(runtime): 212 | # do other stuff with the runtime object 213 | pass 214 | ``` 215 | 216 | With a PluginManager via `load_all`, you receive the `FunctionPlugin` instances, that you can call like the functions 217 | 218 | ```python 219 | 220 | runtime = LocalstackRuntime() 221 | 222 | for configurator in PluginManager("localstack.configurators").load_all(): 223 | configurator(runtime) 224 | ``` 225 | 226 | Configuring your distribution 227 | ----------------------------- 228 | 229 | If you are building a python distribution that exposes plugins discovered by plux, you need to configure your projects build system so other dependencies creates the `entry_points.txt` file when installing your distribution. 230 | 231 | For a [`pyproject.toml`](https://pip.pypa.io/en/stable/reference/build-system/pyproject-toml/) template this involves adding the `build-system` section: 232 | 233 | ```toml 234 | [build-system] 235 | requires = ['setuptools', 'wheel', 'plux>=1.3.1'] 236 | build-backend = "setuptools.build_meta" 237 | 238 | # ... 239 | ``` 240 | 241 | Additional configuration 242 | ------------------------ 243 | 244 | You can pass additional configuration to Plux, either via the command line or your project `pyproject.toml`. 245 | 246 | ### Configuration options 247 | 248 | The following options can be configured in the `[tool.plux]` section of your `pyproject.toml`: 249 | 250 | ```toml 251 | [tool.plux] 252 | # The build mode for entry points: "build-hooks" (default) or "manual" 253 | entrypoint_build_mode = "manual" 254 | 255 | # The file path to scan for plugins (optional) 256 | path = "mysrc" 257 | 258 | # Python packages to exclude during discovery (optional) 259 | exclude = ["**/database/alembic*"] 260 | 261 | # Python packages to include during discovery (optional), setting this will ignore all other paths 262 | include = ["**/database*"] 263 | ``` 264 | 265 | #### `entrypoint_build_mode` 266 | 267 | Controls how plux generates entry points: 268 | - `build-hooks` (default): Plux automatically hooks into the build process to generate entry points 269 | - `manual`: You manually control when and how entry points are generated (see [Manual mode](#manual-mode)) 270 | 271 | #### `path` 272 | 273 | Specifies the file path to scan for plugins. By default, plux scans the entire project. 274 | 275 | #### `include` 276 | 277 | A list of paths to include during plugin discovery. If specified, only the named items will be included. If not specified, all found items in the path will be included. The `include` parameter supports shell-style wildcard patterns. 278 | 279 | Examples: 280 | ```bash 281 | # Include multiple patterns 282 | python -m plux discover --include "myapp/plugins*,myapp/extensions*" --format ini 283 | ``` 284 | 285 | You can also specify these values in the `[tool.plux]` section of your `pyproject.toml` as shown above. 286 | 287 | **Note:** When `include` is specified, plux ignores all other paths that would otherwise be found. 288 | 289 | 290 | #### `exclude` 291 | 292 | When [discovering entrypoints](#discovering-entrypoints), Plux will try importing your code to discover Plugins. 293 | Some parts of your codebase might have side effects, or raise errors when imported outside a specific context like some database 294 | migration scripts. 295 | 296 | You can ignore those Python packages by specifying the `--exclude` flag to the entrypoints discovery commands: 297 | 298 | ```bash 299 | # Exclude database migration scripts 300 | python -m plux entrypoints --exclude "**/database/alembic*" 301 | 302 | # Exclude multiple patterns (comma-separated) 303 | python -m plux discover --exclude "tests*,docs*" --format ini 304 | ``` 305 | 306 | The option takes a list of comma-separated values that can be paths or package names with shell-style wildcards. `'foo.*'` will exclude all subpackages of `foo` (but not `foo` itself). 307 | 308 | You can also specify these values in the `[tool.plux]` section of your `pyproject.toml` as shown above. 309 | 310 | Install 311 | ------- 312 | 313 | pip install plux 314 | 315 | Develop 316 | ------- 317 | 318 | Create the virtual environment, install dependencies, and run tests 319 | 320 | make venv 321 | make test 322 | 323 | Run the code formatter 324 | 325 | make format 326 | 327 | Upload the pypi package using twine 328 | 329 | make upload 330 | -------------------------------------------------------------------------------- /plux/runtime/manager.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import threading 3 | import typing as t 4 | 5 | from plux.core.plugin import ( 6 | Plugin, 7 | PluginDisabled, 8 | PluginException, 9 | PluginFinder, 10 | PluginLifecycleListener, 11 | PluginSpec, 12 | ) 13 | 14 | from .filter import PluginFilter, global_plugin_filter 15 | from .metadata import Distribution, resolve_distribution_information 16 | from .resolve import MetadataPluginFinder 17 | 18 | LOG = logging.getLogger(__name__) 19 | 20 | P = t.TypeVar("P", bound=Plugin) 21 | 22 | 23 | def _call_safe(func: t.Callable, args: tuple, exception_message: str): 24 | """ 25 | Call the given function with the given arguments, and if it fails, log the given exception_message. If 26 | logging.DEBUG is set for the logger, then we also log the traceback. An exception is made for any 27 | ``PluginException``, which should be handled by the caller. 28 | 29 | :param func: function to call 30 | :param args: arguments to pass 31 | :param exception_message: message to log on exception 32 | :return: whatever the func returns 33 | """ 34 | try: 35 | return func(*args) 36 | except PluginException: 37 | raise 38 | except Exception as e: 39 | if LOG.isEnabledFor(logging.DEBUG): 40 | LOG.exception(exception_message) 41 | else: 42 | LOG.error("%s: %s", exception_message, e) 43 | 44 | 45 | class PluginLifecycleNotifierMixin: 46 | """ 47 | Mixin that provides functions to dispatch calls to a PluginLifecycleListener in a safe way. 48 | """ 49 | 50 | listeners: list[PluginLifecycleListener] 51 | 52 | def _fire_on_resolve_after(self, plugin_spec): 53 | for listener in self.listeners: 54 | _call_safe( 55 | listener.on_resolve_after, 56 | (plugin_spec,), # 57 | "error while calling on_resolve_after", 58 | ) 59 | 60 | def _fire_on_resolve_exception(self, namespace, entrypoint, exception): 61 | for listener in self.listeners: 62 | _call_safe( 63 | listener.on_resolve_exception, 64 | (namespace, entrypoint, exception), 65 | "error while calling on_resolve_exception", 66 | ) 67 | 68 | def _fire_on_init_after(self, plugin_spec, plugin): 69 | for listener in self.listeners: 70 | _call_safe( 71 | listener.on_init_after, 72 | ( 73 | plugin_spec, 74 | plugin, 75 | ), # 76 | "error while calling on_init_after", 77 | ) 78 | 79 | def _fire_on_init_exception(self, plugin_spec, exception): 80 | for listener in self.listeners: 81 | _call_safe( 82 | listener.on_init_exception, 83 | (plugin_spec, exception), 84 | "error while calling on_init_exception", 85 | ) 86 | 87 | def _fire_on_load_before(self, plugin_spec, plugin, load_args, load_kwargs): 88 | for listener in self.listeners: 89 | _call_safe( 90 | listener.on_load_before, 91 | (plugin_spec, plugin, load_args, load_kwargs), 92 | "error while calling on_load_before", 93 | ) 94 | 95 | def _fire_on_load_after(self, plugin_spec, plugin, result): 96 | for listener in self.listeners: 97 | _call_safe( 98 | listener.on_load_after, 99 | (plugin_spec, plugin, result), 100 | "error while calling on_load_after", 101 | ) 102 | 103 | def _fire_on_load_exception(self, plugin_spec, plugin, exception): 104 | for listener in self.listeners: 105 | _call_safe( 106 | listener.on_load_exception, 107 | (plugin_spec, plugin, exception), 108 | "error while calling on_load_exception", 109 | ) 110 | 111 | 112 | class PluginContainer(t.Generic[P]): 113 | """ 114 | Object to pass around the plugin state inside a PluginManager. 115 | """ 116 | 117 | name: str 118 | lock: threading.RLock 119 | 120 | plugin_spec: PluginSpec 121 | plugin: P = None 122 | load_value: t.Any = None 123 | 124 | is_init: bool = False 125 | is_loaded: bool = False 126 | 127 | init_error: Exception = None 128 | load_error: Exception = None 129 | 130 | is_disabled: bool = False 131 | disabled_reason = str = None 132 | 133 | @property 134 | def distribution(self) -> Distribution: 135 | """ 136 | Uses metadata from importlib to resolve the distribution information for this plugin. 137 | 138 | :return: the importlib.metadata.Distribution object 139 | """ 140 | return resolve_distribution_information(self.plugin_spec) 141 | 142 | 143 | class PluginManager(PluginLifecycleNotifierMixin, t.Generic[P]): 144 | """ 145 | Manages Plugins within a namespace discovered by a PluginFinder. The default mechanism is to resolve plugins from 146 | entry points using a ImportlibPluginFinder. 147 | 148 | A Plugin that is managed by a PluginManager can be in three states: 149 | * resolved: the entrypoint pointing to the PluginSpec was imported and the PluginSpec instance was created 150 | * init: the PluginFactory of the PluginSpec was successfully invoked 151 | * loaded: the load method of the Plugin was successfully invoked 152 | 153 | Internally, the PluginManager uses PluginContainer instances to keep the state of Plugin instances. 154 | """ 155 | 156 | namespace: str 157 | 158 | load_args: list | tuple 159 | load_kwargs: dict[str, t.Any] 160 | listeners: list[PluginLifecycleListener] 161 | filters: list[PluginFilter] 162 | 163 | def __init__( 164 | self, 165 | namespace: str, 166 | load_args: list | tuple = None, 167 | load_kwargs: dict = None, 168 | listener: PluginLifecycleListener | t.Iterable[PluginLifecycleListener] = None, 169 | finder: PluginFinder = None, 170 | filters: list[PluginFilter] = None, 171 | ): 172 | """ 173 | Create a new PluginManager. 174 | 175 | :param namespace: the namespace (entry point group) that will be managed 176 | :param load_args: positional arguments passed to ``Plugin.load()`` 177 | :param load_kwargs: keyword arguments passed to ``Plugin.load()`` 178 | :param listener: plugin lifecycle listeners, can either be a single listener or a list 179 | :param finder: the plugin finder to be used, by default it uses a ``MetadataPluginFinder` 180 | :param filters: filters exclude specific plugins. when no filters are provided, a list is created 181 | and ``global_plugin_filter`` is added to it. filters can later be modified via 182 | ``plugin_manager.filters``. 183 | """ 184 | self.namespace = namespace 185 | 186 | self.load_args = load_args or list() 187 | self.load_kwargs = load_kwargs or dict() 188 | 189 | if listener: 190 | if isinstance(listener, (list, set, tuple)): 191 | self.listeners = list(listener) 192 | else: 193 | self.listeners = [listener] 194 | else: 195 | self.listeners = [] 196 | 197 | if not filters: 198 | self.filters = [global_plugin_filter] 199 | 200 | self.finder = finder or MetadataPluginFinder(self.namespace, self._fire_on_resolve_exception) 201 | 202 | self._plugin_index = None 203 | self._init_mutex = threading.RLock() 204 | 205 | def add_listener(self, listener: PluginLifecycleListener): 206 | self.listeners.append(listener) 207 | 208 | def load(self, name: str) -> P: 209 | """ 210 | Loads the Plugin with the given name using the load args and kwargs set in the plugin manager constructor. 211 | If at any point in the lifecycle the plugin loading fails, the load method will raise the respective exception. 212 | 213 | Load is idempotent, so once the plugin is loaded, load will return the same instance again. 214 | """ 215 | container = self._require_plugin(name) 216 | 217 | if container.is_disabled: 218 | raise PluginDisabled(container.plugin_spec.namespace, name, container.disabled_reason) 219 | 220 | if not container.is_loaded: 221 | try: 222 | self._load_plugin(container) 223 | except PluginDisabled as e: 224 | container.is_disabled = True 225 | container.disabled_reason = e.reason 226 | raise 227 | 228 | if container.init_error: 229 | raise container.init_error 230 | 231 | if container.load_error: 232 | raise container.load_error 233 | 234 | if not container.is_loaded: 235 | raise PluginException("plugin did not load correctly", namespace=self.namespace, name=name) 236 | 237 | return container.plugin 238 | 239 | def load_all(self, propagate_exceptions=False) -> list[P]: 240 | """ 241 | Attempts to load all plugins found in the namespace, and returns those that were loaded successfully. If 242 | propagate_exception is set to True, then the method will re-raise any errors as soon as it encouters them. 243 | """ 244 | plugins = list() 245 | 246 | for name, container in self._plugins.items(): 247 | if container.is_loaded: 248 | plugins.append(container.plugin) 249 | continue 250 | 251 | try: 252 | plugin = self.load(name) 253 | plugins.append(plugin) 254 | except PluginDisabled as e: 255 | LOG.debug("%s", e) 256 | except Exception as e: 257 | if propagate_exceptions: 258 | raise 259 | else: 260 | LOG.error("exception while loading plugin %s:%s: %s", self.namespace, name, e) 261 | 262 | return plugins 263 | 264 | def list_plugin_specs(self) -> list[PluginSpec]: 265 | return [container.plugin_spec for container in self._plugins.values()] 266 | 267 | def list_names(self) -> list[str]: 268 | return [spec.name for spec in self.list_plugin_specs()] 269 | 270 | def list_containers(self) -> list[PluginContainer[P]]: 271 | return list(self._plugins.values()) 272 | 273 | def get_container(self, name: str) -> PluginContainer[P]: 274 | return self._require_plugin(name) 275 | 276 | def exists(self, name: str) -> bool: 277 | return name in self._plugins 278 | 279 | def is_loaded(self, name: str) -> bool: 280 | return self._require_plugin(name).is_loaded 281 | 282 | @property 283 | def _plugins(self) -> dict[str, PluginContainer[P]]: 284 | if self._plugin_index is None: 285 | with self._init_mutex: 286 | if self._plugin_index is None: 287 | self._plugin_index = self._init_plugin_index() 288 | 289 | return self._plugin_index 290 | 291 | def _require_plugin(self, name: str) -> PluginContainer[P]: 292 | if name not in self._plugins: 293 | raise ValueError("no plugin named %s in namespace %s" % (name, self.namespace)) 294 | 295 | return self._plugins[name] 296 | 297 | def _load_plugin(self, container: PluginContainer): 298 | with container.lock: 299 | plugin_spec = container.plugin_spec 300 | 301 | if self.filters: 302 | for filter_ in self.filters: 303 | if filter_(plugin_spec): 304 | raise PluginDisabled( 305 | namespace=self.namespace, 306 | name=container.plugin_spec.name, 307 | reason="A plugin filter disabled this plugin before it was initialized", 308 | ) 309 | 310 | # instantiate Plugin from spec if necessary 311 | if not container.is_init: 312 | try: 313 | LOG.debug("instantiating plugin %s", plugin_spec) 314 | container.plugin = self._plugin_from_spec(plugin_spec) 315 | container.is_init = True 316 | self._fire_on_init_after(plugin_spec, container.plugin) 317 | except PluginDisabled: 318 | raise 319 | except Exception as e: 320 | # TODO: maybe we should move these logging blocks to `load_all`, since this is the only instance 321 | # where exceptions messages may get lost. 322 | if LOG.isEnabledFor(logging.DEBUG): 323 | LOG.exception("error instantiating plugin %s", plugin_spec) 324 | 325 | self._fire_on_init_exception(plugin_spec, e) 326 | container.init_error = e 327 | return 328 | 329 | plugin = container.plugin 330 | 331 | if not plugin.should_load(): 332 | raise PluginDisabled( 333 | namespace=self.namespace, 334 | name=container.plugin_spec.name, 335 | reason="Load condition for plugin was false", 336 | ) 337 | 338 | args = self.load_args 339 | kwargs = self.load_kwargs 340 | 341 | try: 342 | self._fire_on_load_before(plugin_spec, plugin, args, kwargs) 343 | LOG.debug("loading plugin %s:%s", self.namespace, plugin_spec.name) 344 | result = plugin.load(*args, **kwargs) 345 | self._fire_on_load_after(plugin_spec, plugin, result) 346 | container.load_value = result 347 | container.is_loaded = True 348 | except PluginDisabled: 349 | raise 350 | except Exception as e: 351 | if LOG.isEnabledFor(logging.DEBUG): 352 | LOG.exception("error loading plugin %s", plugin_spec) 353 | self._fire_on_load_exception(plugin_spec, plugin, e) 354 | container.load_error = e 355 | 356 | def _plugin_from_spec(self, plugin_spec: PluginSpec) -> P: 357 | factory = plugin_spec.factory 358 | 359 | # functional decorators can overwrite the spec factory (pointing to the decorator) with a custom factory 360 | spec = getattr(factory, "__pluginspec__", None) 361 | if spec: 362 | factory = spec.factory 363 | 364 | return factory() 365 | 366 | def _init_plugin_index(self) -> dict[str, PluginContainer]: 367 | return {plugin.name: plugin for plugin in self._import_plugins() if plugin} 368 | 369 | def _import_plugins(self) -> t.Iterable[PluginContainer]: 370 | for spec in self.finder.find_plugins(): 371 | self._fire_on_resolve_after(spec) 372 | 373 | if spec.namespace != self.namespace: 374 | continue 375 | 376 | yield self._create_container(spec) 377 | 378 | def _create_container(self, plugin_spec: PluginSpec) -> PluginContainer: 379 | container = PluginContainer() 380 | container.lock = threading.RLock() 381 | container.name = plugin_spec.name 382 | container.plugin_spec = plugin_spec 383 | return container 384 | -------------------------------------------------------------------------------- /plux/build/setuptools.py: -------------------------------------------------------------------------------- 1 | """ 2 | Bindings to integrate plux into setuptools build processes. 3 | """ 4 | 5 | import json 6 | import logging 7 | import os 8 | import re 9 | import shutil 10 | import sys 11 | import typing as t 12 | from pathlib import Path 13 | 14 | import setuptools 15 | from setuptools.command.egg_info import egg_info 16 | 17 | from plux.build import config 18 | from plux.build.config import EntrypointBuildMode 19 | from plux.build.project import Project 20 | 21 | try: 22 | from setuptools.command.editable_wheel import editable_wheel 23 | except ImportError: 24 | # this means we're probably working on an old setuptools version, perhaps because we are on an older 25 | # Python version. 26 | class editable_wheel: 27 | def run(self): 28 | raise NotImplementedError("Compatibility with editable wheels requires Python 3.10") 29 | 30 | def _ensure_dist_info(self, *args, **kwargs): 31 | pass 32 | 33 | 34 | from setuptools.command.egg_info import InfoCommon, write_entries 35 | 36 | from plux.core.entrypoint import EntryPointDict, discover_entry_points 37 | from plux.runtime.metadata import entry_points_from_metadata_path 38 | from plux.build.discovery import PluginFromPackageFinder, PackageFinder, Filter, MatchAllFilter 39 | from plux.build.index import PluginIndexBuilder 40 | 41 | LOG = logging.getLogger(__name__) 42 | 43 | 44 | # SETUPTOOLS HOOKS 45 | # ================ 46 | # The following classes and methods are a way for plux to hook into 47 | # the setuptools build process either indirectly or programmatically. 48 | 49 | 50 | class plugins(InfoCommon, setuptools.Command): 51 | """ 52 | Setuptools command that discovers plugins and writes them into the egg_info directory to a ``plux.json`` file. 53 | 54 | TODO: This only exists for compatibility with older ``setup.py`` workflows. It was meant as a frontend for 55 | setuptools (to be called via ``setup.py plugins``. The modern way to build is either ``pip install -e .`` or 56 | ``python -m build``, so the command is basically obsolete. We should remove it with a future release, and 57 | instead rely on either the plux CLI frontend, or a transparent instrumentation of the build backend. It does help 58 | *slightly* with the integration with setuptools, since we can call ``dist.run_command('plugins')`` 59 | programmatically, and then the ``egg_info`` command in a chain. However, we already have 60 | ``patch_egg_info_command`` where we could implement this logic. More background can be found here: 61 | https://github.com/pypa/setuptools/discussions/4223. 62 | """ 63 | 64 | description = "Discover plux plugins and store them in .egg_info" 65 | 66 | user_options: t.ClassVar[list[tuple[str, str, str]]] = [ 67 | ( 68 | "exclude=", 69 | "e", 70 | "a sequence of paths to exclude; '*' can be used as a wildcard in the names. 'foo.*' will exclude all subpackages of 'foo' (but not 'foo' itself).", 71 | ), 72 | ( 73 | "include=", 74 | "i", 75 | "a sequence of paths to include; If it's specified, only the named items will be included. If it's not " 76 | "specified, all found items in the path will be included. 'include' can contain shell style wildcard " 77 | "patterns just like 'exclude'", 78 | ), 79 | # TODO: add more, this should mirror the entrypoints or discover cli 80 | ] 81 | 82 | egg_info: str 83 | 84 | def initialize_options(self) -> None: 85 | self.plux_json_path = None 86 | self.exclude = None 87 | self.include = None 88 | self.plux_config = None 89 | 90 | def finalize_options(self) -> None: 91 | self.plux_json_path = get_plux_json_path(self.distribution) 92 | self.ensure_string_list("exclude") 93 | self.ensure_string_list("include") 94 | 95 | # we merge the configuration from the CLI arguments with the configuration read from the `pyproject.toml` 96 | # [tool.plux] section 97 | self.plux_config = read_plux_configuration(self.distribution) 98 | self.plux_config = self.plux_config.merge( 99 | exclude=self.exclude, 100 | include=self.include, 101 | ) 102 | 103 | def run(self) -> None: 104 | # TODO: should be reconciled with Project.build_entrypoints() 105 | index_builder = create_plugin_index_builder(self.plux_config, self.distribution) 106 | self.debug_print(f"writing discovered plugins into {self.plux_json_path}") 107 | self.mkpath(os.path.dirname(self.plux_json_path)) 108 | with open(self.plux_json_path, "w") as fp: 109 | ep = index_builder.write(fp) 110 | 111 | update_entrypoints(self.distribution, ep) 112 | 113 | 114 | def patch_editable_wheel_command(): 115 | """ 116 | This patch creates a way for plux to maintain a link between the entry_points.txt contained in source 117 | distribution's egg-info, and the installed dist-info directory created by the editable_wheel command. 118 | This is so we can resolve entry points re-generated by ``python -m plux plugins`` from the source code, 119 | while using editable installs via ``pip install -e .`` into the venv. 120 | 121 | For instance, when you use an editable install, the path will look something like:: 122 | 123 | ~/my-project/my_project/ 124 | ~/my-project/my_project.egg-info/ 125 | ~/my-project/.venv/ 126 | ~/my-project/.venv/lib/python3.11/site-packages/my_project-0.0.1.dev0.dist-info 127 | 128 | The .egg-info directory will correctly contain an entry-points.txt, but not the generated dist-info 129 | directory in site-packages. This is a problem under certain circumstances because importlib may not 130 | find the entry points. The reasons why dist-info doesn't contain the entry_points.txt are because of 131 | how pip works. We could make sure that we correctly create an entry_points.txt during the building 132 | process, but that would ultimately not help, as we would have to ``pip install -e .`` again every time 133 | the plugins change. So a dynamic approach is better, which the linking facilitates. During runtime, 134 | simply follow the link left behind in ``entry_points_editable.txt`` to the real ``entry_points.txt`` in 135 | the .egg-info directory. 136 | 137 | TODO: this is a hacky patch that relies on setuptools internals. It would be better to find a clean way 138 | to completely overwrite the ``editable_wheel`` command, perhaps via the command class resolution 139 | mechanism. i looked into that for an hour or so and concluded that this is an acceptable workaround for 140 | now. 141 | """ 142 | _ensure_dist_info_orig = editable_wheel._ensure_dist_info 143 | 144 | def _ensure_dist_info(self): 145 | _ensure_dist_info_orig(self) 146 | # then we can create a link to the original file 147 | # this is the egg info dir from the distribution source (the pip install -e target) 148 | 149 | # create what is basically an application-layer symlink to the original entry points 150 | target = Path(self.dist_info_dir, "entry_points_editable.txt") 151 | target.write_text(os.path.join(find_egg_info_dir(), "entry_points.txt")) 152 | 153 | editable_wheel._ensure_dist_info = _ensure_dist_info 154 | 155 | 156 | # patch will be applied implicitly through the entry point that loads the ``plugins`` command. 157 | patch_editable_wheel_command() 158 | 159 | 160 | def patch_egg_info_command(): 161 | """ 162 | This patch fixes the build process when pip builds a wheel from a source distribution. A source distribution built 163 | with plux will already contain an `.egg_info` directory, but during the build with pip, a new .egg-info 164 | directory is created from scratch from the python distribution configuration files (like setup.cfg or 165 | pyproject.toml). This is a problem, as building a wheel involves creating a .dist-info dir, which is populated from 166 | this new .egg-info directory. The .egg-info shipped with the source distribution is completely ignored during this 167 | process, including the `plux.json`, leading to a wheel that does not correctly contain the `entry_points.txt`. 168 | 169 | This patch hooks into this procedure and makes sure when this new .egg-info directory is created, we first locate 170 | the one shipped with the source distribution, locate the ``plux.json`` file, and copy it into the new build context. 171 | """ 172 | _run_orig = egg_info.run 173 | 174 | def _run(self): 175 | LOG.debug("Running egg_info command patch from plux") 176 | 177 | cfg = read_plux_configuration(self.distribution) 178 | if cfg.entrypoint_build_mode == EntrypointBuildMode.MANUAL: 179 | LOG.debug("Entrypoint build mode is manual, skipping egg_info patch") 180 | return _run_orig(self) 181 | 182 | # the working directory may be something like `/tmp/pip-req-build-bwekzpi_` where the source distribution 183 | # was copied into. this happens when you `pip install ` where is 184 | # some source distribution. 185 | should_read, meta_dir = _should_read_existing_egg_info() 186 | 187 | # if the .egg_info from the source distribution contains the plux file, we prepare the new .egg-info by copying 188 | # it there. everything else should be handled implicitly by the command 189 | if should_read: 190 | LOG.debug("Locating plux.json from local build context %s", meta_dir) 191 | plux_json = os.path.join(meta_dir, "plux.json") 192 | if os.path.exists(plux_json): 193 | self.mkpath(self.egg_info) # this is what egg_info.run() does but it's idempotent 194 | if not os.path.exists(os.path.join(self.egg_info, "plux.json")): 195 | LOG.debug("copying %s into temporary %s", plux_json, self.egg_info) 196 | shutil.copy(plux_json, self.egg_info) 197 | 198 | return _run_orig(self) 199 | 200 | egg_info.run = _run 201 | 202 | 203 | patch_egg_info_command() 204 | 205 | 206 | def load_plux_entrypoints(cmd, file_name, file_path): 207 | """ 208 | This method is called indirectly by setuptools through the ``egg_info.writers`` plugin. It is used as the main hook 209 | to generate entry points from the index file that plux generates. The plux distribution defines the following 210 | setuptools entrypoint:: 211 | 212 | [project.entry-points."egg_info.writers"] 213 | # this is actually not a writer, it's a reader :-) 214 | "plux.json" = "plux.build.setuptools:load_plux_entrypoints" 215 | 216 | It works in the following way: when setuptools builds the egg_info directory to the additional files that are 217 | defined as egg_info.writers plugins. This is our hook into the build process of setuptools. The trick here is that 218 | we have already previously built the ``plux.json``, and we use this hook to read the file, and use it to write the 219 | ``entry_points.txt`` file. 220 | """ 221 | if not os.path.exists(file_path): 222 | return 223 | 224 | cfg = read_plux_configuration(cmd.distribution) 225 | if cfg.entrypoint_build_mode == EntrypointBuildMode.MANUAL: 226 | return 227 | 228 | cmd.debug_print(f"extend entrypoints with plux plugins from {file_path}") 229 | with open(file_path, "r") as fd: 230 | ep = json.load(fd) 231 | 232 | update_entrypoints(cmd.distribution, ep) 233 | 234 | # this is kind of a hack, but we cannot rely on load_plux_entrypoints being called before the regular 235 | # entry_points.txt egg_info.writers plugin to write the updated entry points 236 | ep_file = "entry_points.txt" 237 | ep_path = os.path.join(os.path.dirname(file_path), ep_file) 238 | write_entries(cmd, ep_file, ep_path) 239 | 240 | 241 | def find_plugins(where=".", exclude=(), include=("*",)) -> EntryPointDict: 242 | """ 243 | Utility for setup.py that collects all plugins from the specified path, and creates a dictionary for 244 | entry_points. 245 | 246 | For example: 247 | 248 | setup( 249 | entry_points=find_plugins() 250 | ) 251 | """ 252 | 253 | return discover_entry_points(PackagePathPluginFinder(where=where, exclude=exclude, include=include)) 254 | 255 | 256 | def load_entry_points(where=".", exclude=(), include=("*",), merge: EntryPointDict = None) -> EntryPointDict: 257 | """ 258 | Finds plugins and builds and entry point map. This is to be used in a setup.py in the setup call: 259 | 260 | setup( 261 | entry_points=load_entry_points(exclude=("tests", "tests.*"), merge={ 262 | "my.static.entrypoint": [ 263 | "foo=bar" 264 | ] 265 | }) 266 | ... 267 | ) 268 | 269 | This is a hack for installing from source distributions. When running pip install on a source 270 | distribution, the egg_info directory is always re-built, even though it comes with the source 271 | distribution package data. This also means the entry points are resolved, and in extent, `find_plugins` 272 | is called, which is problematic at this point, because find_plugins will scan the code, and that will 273 | fail if requirements aren't yet installed, which they aren't when running pip install. However, 274 | since source distributions package the .egg-info directory, we can read the entry points from there 275 | instead, acting as sort of a cache. 276 | 277 | TODO: I'm not sure this is used or needed anymore, as we're moving away from ``setup.py`` files for which this 278 | api was designed. Though the idea of merging manually defined entry points with those discovered by plux is 279 | still valid and maybe useful in some cases. 280 | 281 | :param where: the file path to look for plugins (default, the current working dir) 282 | :param exclude: the shell style wildcard patterns to exclude 283 | :param include: the shell style wildcard patterns to include 284 | :param merge: a map of entry points that are always added 285 | """ 286 | should_read, meta_dir = _should_read_existing_egg_info() 287 | if should_read: 288 | print("reading entry points from existing meta dir", meta_dir) 289 | eps = entry_points_from_egg_info(meta_dir) 290 | else: 291 | print("resolving plugin entry points") 292 | eps = find_plugins(where, exclude, include) 293 | 294 | if merge: 295 | # TODO: merge the dicts instead of overwriting (will involve de-duplicating entry points) 296 | eps.update(merge) 297 | 298 | return eps 299 | 300 | 301 | # UTILITIES 302 | # ========= 303 | # The remaining methods are utilities 304 | 305 | 306 | class SetuptoolsProject(Project): 307 | distribution: setuptools.Distribution 308 | 309 | def __init__(self, workdir: str = None): 310 | super().__init__(workdir) 311 | 312 | self.distribution = get_distribution_from_workdir(str(self.workdir)) 313 | 314 | def find_entry_point_file(self) -> Path: 315 | if egg_info_dir := find_egg_info_dir(): 316 | return Path(egg_info_dir, "entry_points.txt") 317 | raise FileNotFoundError("No .egg-info directory found. Have you run `python -m plux entrypoints`?") 318 | 319 | def find_plux_index_file(self) -> Path: 320 | if self.config.entrypoint_build_mode == EntrypointBuildMode.MANUAL: 321 | return self.workdir / self.config.entrypoint_static_file 322 | 323 | return Path(get_plux_json_path(self.distribution)) 324 | 325 | def create_plugin_index_builder(self) -> PluginIndexBuilder: 326 | return create_plugin_index_builder(self.config, self.distribution) 327 | 328 | def create_package_finder(self) -> PackageFinder: 329 | exclude = [_path_to_module(item) for item in self.config.exclude] 330 | include = [_path_to_module(item) for item in self.config.include] 331 | return DistributionPackageFinder(self.distribution, exclude=exclude, include=include) 332 | 333 | def build_entrypoints(self): 334 | dist = self.distribution 335 | 336 | dist.command_options["plugins"] = { 337 | "exclude": ("command line", ",".join(self.config.exclude) or None), 338 | "include": ("command line", ",".join(self.config.include) or None), 339 | } 340 | dist.run_command("plugins") 341 | 342 | print(f"building {dist.get_name().replace('-', '_')}.egg-info...") 343 | dist.run_command("egg_info") 344 | 345 | print("discovered plugins:") 346 | # print discovered plux plugins 347 | with open(get_plux_json_path(dist)) as fd: 348 | plux_json = json.load(fd) 349 | json.dump(plux_json, sys.stdout, indent=2) 350 | 351 | 352 | def get_plux_json_path(distribution: setuptools.Distribution) -> str: 353 | """ 354 | Returns the full path of ``plux.json`` file for the given distribution. The file is located within the .egg-info 355 | directory that contains the built metadata of the distribution. 356 | """ 357 | dirs = distribution.package_dir 358 | egg_base = (dirs or {}).get("", os.curdir) 359 | egg_info_dir = _to_filename(_safe_name(distribution.get_name())) + ".egg-info" 360 | egg_info_dir = os.path.join(egg_base, egg_info_dir) 361 | return os.path.join(egg_info_dir, "plux.json") 362 | 363 | 364 | def read_plux_configuration(distribution: setuptools.Distribution) -> config.PluxConfiguration: 365 | """ 366 | Try reading the ``[tool.plux]`` section of the distribution's ``pyproject.toml`` file and parse it using our 367 | config parser. Note this method will use the distribution's ``package_dir`` to try and resolve the path first, 368 | so if you have a project with a namespace package, it may not work as expected if you call it from a CLI process. 369 | 370 | :param distribution: The distribution containing the pyproject.toml file. 371 | :return: The parsed configuration object 372 | """ 373 | dirs = distribution.package_dir 374 | pyproject_base = (dirs or {}).get("", os.curdir) 375 | return config.read_plux_config_from_workdir(pyproject_base) 376 | 377 | 378 | def update_entrypoints(distribution: setuptools.Distribution, ep: EntryPointDict): 379 | """ 380 | Updates ``distribution.entry_points`` with the given entry point dict. Currently, it simply overwrites sections 381 | within the entrypoints. 382 | """ 383 | if distribution.entry_points is None: 384 | distribution.entry_points = {} 385 | 386 | # TODO: merge entry point groups 387 | distribution.entry_points.update(ep) 388 | 389 | 390 | def create_plugin_index_builder( 391 | cfg: config.PluxConfiguration, 392 | distribution: setuptools.Distribution, 393 | ) -> PluginIndexBuilder: 394 | """ 395 | Creates a PluginIndexBuilder instance for discovering plugins from a setuptools distribution. It uses a 396 | ``DistributionPackageFinder`` to resolve the packages to scan for plugins from the given distribution (instead of 397 | the path given in the PluxConfiguration). However, it respects excludes given in the PluxConfiguration. 398 | """ 399 | exclude = [_path_to_module(item) for item in cfg.exclude] 400 | include = [_path_to_module(item) for item in cfg.include] 401 | plugin_finder = PluginFromPackageFinder( 402 | DistributionPackageFinder(distribution, exclude=exclude, include=include) 403 | ) 404 | return PluginIndexBuilder(plugin_finder) 405 | 406 | 407 | def entry_points_from_egg_info(egg_info_dir: str) -> EntryPointDict: 408 | """ 409 | Reads the entry_points.txt from a distribution meta dir (e.g., the .egg-info directory). 410 | """ 411 | return entry_points_from_metadata_path(egg_info_dir) 412 | 413 | 414 | def _should_read_existing_egg_info() -> tuple[bool, str | None]: 415 | # we want to read the .egg-info dir only if it exists, and if we are creating the egg_info or 416 | # installing it with pip install -e (which calls 'setup.py develop') 417 | 418 | if not (_is_pip_build_context() or _is_local_build_context()): 419 | return False, None 420 | 421 | egg_info_dir = find_egg_info_dir() 422 | if not egg_info_dir: 423 | return False, None 424 | 425 | if not os.path.isfile(os.path.join(egg_info_dir, "entry_points.txt")): 426 | return False, None 427 | 428 | return True, egg_info_dir 429 | 430 | 431 | def _is_pip_build_context(): 432 | # when pip builds packages or wheels from source distributions, it creates a temporary directory with a 433 | # marker file that we can use to determine whether we are in such a build context. 434 | for f in os.listdir(os.getcwd()): 435 | if f == "pip-delete-this-directory.txt": 436 | return True 437 | 438 | return False 439 | 440 | 441 | def _is_local_build_context(): 442 | # when installing with `pip install -e` pip also tries to create a "modern-metadata" (dist-info) 443 | # directory. 444 | if "dist_info" in sys.argv: 445 | try: 446 | i = sys.argv.index("--egg-base") 447 | except ValueError: 448 | try: 449 | # this code path is for building wheels from source distributions via pip 450 | i = sys.argv.index("--output-dir") 451 | except ValueError: 452 | return False 453 | 454 | if "pip-modern-metadata" in sys.argv[i + 1]: 455 | return True 456 | 457 | # it's unfortunately not really distinguishable whether or not a user calls `python setup.py develop` 458 | # in the project, or calls `pip install -e ..` to install the project from somewhere else. 459 | if len(sys.argv) > 1 and sys.argv[1] in ["egg_info", "develop"]: 460 | return True 461 | 462 | return False 463 | 464 | 465 | def find_egg_info_dir() -> str | None: 466 | """ 467 | Heuristic to find the .egg-info dir of the current build context. 468 | """ 469 | workdir = os.getcwd() 470 | distribution = get_distribution_from_workdir(workdir) 471 | dirs = distribution.package_dir 472 | egg_base = (dirs or {}).get("", workdir) 473 | if not egg_base: 474 | return None 475 | egg_info_dir = _to_filename(_safe_name(distribution.get_name())) + ".egg-info" 476 | candidate = os.path.join(workdir, egg_base, egg_info_dir) 477 | if os.path.exists(candidate): 478 | return candidate 479 | return None 480 | 481 | 482 | def get_distribution_from_workdir(workdir: str) -> setuptools.Distribution: 483 | """ 484 | Reads from the current workdir the available project configs and parses them to create a 485 | ``setuptools.Distribution`` object, which can later be used to invoke distutils commands. 486 | 487 | :param workdir: the workdir containing the project files 488 | :return: a distribution object 489 | """ 490 | config_files = ["pyproject.toml", "setup.cfg"] 491 | config_files = [os.path.join(workdir, file) for file in config_files] 492 | config_files = [file for file in config_files if os.path.exists(file)] 493 | 494 | if not config_files: 495 | raise ValueError(f"no distribution config files found in {workdir}") 496 | 497 | dist = setuptools.Distribution() 498 | dist.parse_config_files(config_files) 499 | if os.path.exists(os.path.join(workdir, "setup.py")): 500 | # use setup.py script if available 501 | dist.script_name = os.path.join(workdir, "setup.py") 502 | else: 503 | # else use a config file (seems to work regardless) 504 | dist.script_name = config_files[0] 505 | 506 | # note: the property Distribution.script_name is added to `SOURCES.txt` during the `sdist` command. the path 507 | # must be a relative path. see https://github.com/localstack/plux/issues/23 508 | dist.script_name = os.path.relpath(dist.script_name, workdir) 509 | 510 | return dist 511 | 512 | 513 | def _safe_name(name): 514 | """Convert an arbitrary string to a standard distribution name. Copied from pkg_resources. 515 | 516 | Any runs of non-alphanumeric/. characters are replaced with a single '-'. 517 | """ 518 | return re.sub("[^A-Za-z0-9.]+", "-", name) 519 | 520 | 521 | def _to_filename(name): 522 | """Convert a project or version name to its filename-escaped form. Copied from pkg_resources. 523 | 524 | Any '-' characters are currently replaced with '_'. 525 | """ 526 | return name.replace("-", "_") 527 | 528 | 529 | def _path_to_module(path): 530 | """ 531 | Convert a path to a Python module to its module representation 532 | Example: plux/core/test -> plux.core.test 533 | """ 534 | return ".".join(Path(path).with_suffix("").parts) 535 | 536 | 537 | class DistributionPackageFinder(PackageFinder): 538 | """ 539 | PackageFinder that returns the packages found in the distribution. The Distribution will already have a 540 | list of resolved packages depending on the setup config. For example, if a ``pyproject.toml`` is used, 541 | then the ``[tool.setuptools.package.find]`` config will be interpreted, resolved, and then 542 | ``distribution.packages`` will contain the resolved packages. This already contains namespace packages 543 | correctly if configured. 544 | You can additionally pass a sequence of values to the ``exclude`` parameters to provide a list of Unix shell style 545 | patterns that will be matched against the Python packages to exclude them from the resolved packages. 546 | Wildcards are allowed in the patterns with '*'. 'foo.*' will exclude all subpackages of 'foo' (but not 'foo' 547 | itself). You can also pass ``include`` parameters, as specified in the ``PluxConfiguration``. 548 | """ 549 | 550 | def __init__( 551 | self, 552 | distribution: setuptools.Distribution, 553 | exclude: t.Iterable[str] | None = None, 554 | include: t.Iterable[str] | None = None, 555 | ): 556 | self.distribution = distribution 557 | self.exclude = Filter(exclude or []) 558 | self.include = Filter(include) if include else MatchAllFilter() 559 | 560 | def find_packages(self) -> t.Iterable[str]: 561 | if self.distribution.packages is None: 562 | raise ValueError( 563 | "No packages found in setuptools distribution. Is your project configured correctly?" 564 | ) 565 | return self.filter_packages(self.distribution.packages) 566 | 567 | @property 568 | def path(self) -> str: 569 | if not self.distribution.package_dir: 570 | where = "." 571 | else: 572 | if self.distribution.package_dir[""]: 573 | where = self.distribution.package_dir[""] 574 | else: 575 | LOG.warning("plux doesn't know how to resolve multiple package_dir directories") 576 | where = "." 577 | return where 578 | 579 | def filter_packages(self, packages: t.Iterable[str]) -> t.Iterable[str]: 580 | return [item for item in packages if not self.exclude(item) and self.include(item)] 581 | 582 | 583 | class SetuptoolsPackageFinder(PackageFinder): 584 | """ 585 | Uses setuptools internals to resolve packages. 586 | """ 587 | 588 | def __init__(self, where=".", exclude=(), include=("*",), namespace=True) -> None: 589 | self.where = where 590 | self.exclude = exclude 591 | self.include = include 592 | self.namespace = namespace 593 | 594 | def find_packages(self) -> t.Iterable[str]: 595 | if self.namespace: 596 | return setuptools.find_namespace_packages(self.where, self.exclude, self.include) 597 | else: 598 | return setuptools.find_packages(self.where, self.exclude, self.include) 599 | 600 | @property 601 | def path(self) -> str: 602 | return self.where 603 | 604 | 605 | class PackagePathPluginFinder(PluginFromPackageFinder): 606 | """ 607 | Uses setuptools and pkgutil to find and import modules within a given path and then uses a 608 | ModuleScanningPluginFinder to resolve the available plugins. The constructor has the same signature as 609 | setuptools.find_packages(where, exclude, include). 610 | """ 611 | 612 | def __init__(self, where=".", exclude=(), include=("*",), namespace=True) -> None: 613 | super().__init__(SetuptoolsPackageFinder(where, exclude, include, namespace=namespace)) 614 | --------------------------------------------------------------------------------