├── .github └── workflows │ ├── moban-update.yml │ ├── pythonpublish.yml │ └── tests.yml ├── .gitignore ├── .gitmodules ├── .isort.cfg ├── .moban.d ├── CUSTOM_README.rst.jj2 ├── custom_setup.py.jj2 ├── description.rst.jj2 ├── docs │ └── source │ │ ├── custom_conf.py.jj2 │ │ ├── custom_index.rst.jj2 │ │ └── myconf.py.jj2 ├── lml-enabled-projects.rst.jj2 ├── requirements.txt.jj2 └── test.sh.jj2 ├── .moban.yml ├── .readthedocs.yml ├── .travis.yml ├── CHANGELOG.rst ├── CONTRIBUTORS.rst ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.rst ├── changelog.yml ├── docs ├── requirements.txt └── source │ ├── _static │ └── images │ │ ├── loading_sequence.svg │ │ ├── robot_chef.svg │ │ ├── robotchef_allinone_lml.svg │ │ ├── robotchef_api_crd.svg │ │ └── robotchef_crd.svg │ ├── allinone_lml_tutorial.rst │ ├── allinone_tutorial.rst │ ├── api.rst │ ├── api_tutorial.rst │ ├── appendix.rst │ ├── conf.py │ ├── design.rst │ ├── index.rst │ ├── lml_log.rst │ ├── lml_tutorial.rst │ ├── spelling_wordlist.txt │ ├── tutorial.rst │ └── uml │ ├── loading_sequence.uml │ ├── robot_chef.uml │ ├── robotchef_allinone_lml.uml │ ├── robotchef_api_crd.uml │ └── robotchef_crd.uml ├── examples └── README.rst ├── lml.yml ├── lml ├── __init__.py ├── _version.py ├── loader.py ├── plugin.py └── utils.py ├── requirements.txt ├── rnd_requirements.txt ├── setup.cfg ├── setup.py ├── test.sh └── tests ├── __init__.py ├── requirements.txt ├── sample_plugin ├── sample_plugin │ ├── __init__.py │ ├── manager.py │ └── reader.py └── setup.py ├── test_plugin ├── pyexcel_test │ └── __init__.py └── setup.py ├── test_plugin_info.py ├── test_plugin_loader.py ├── test_plugin_manager.py └── test_utils.py /.github/workflows/moban-update.yml: -------------------------------------------------------------------------------- 1 | on: [push] 2 | 3 | jobs: 4 | run_moban: 5 | runs-on: ubuntu-latest 6 | name: synchronize templates via moban 7 | steps: 8 | - uses: actions/checkout@v2 9 | with: 10 | ref: ${{ github.head_ref }} 11 | token: ${{ secrets.PAT }} 12 | - name: Set up Python 13 | uses: actions/setup-python@v1 14 | with: 15 | python-version: '3.11' 16 | - name: check changes 17 | run: | 18 | pip install markupsafe==2.0.1 19 | pip install ruamel.yaml moban gitfs2 pypifs moban-jinja2-github moban-ansible 20 | moban 21 | git status 22 | git diff --exit-code 23 | - name: Auto-commit 24 | if: failure() 25 | uses: stefanzweifel/git-auto-commit-action@v4 26 | with: 27 | commit_message: >- 28 | This is an auto-commit, updating project meta data, 29 | such as changelog.rst, contributors.rst 30 | -------------------------------------------------------------------------------- /.github/workflows/pythonpublish.yml: -------------------------------------------------------------------------------- 1 | name: Upload Python Package 2 | 3 | on: 4 | release: 5 | types: [created] 6 | 7 | jobs: 8 | pypi-publish: 9 | name: upload release to PyPI 10 | runs-on: ubuntu-latest 11 | # Specifying a GitHub environment is optional, but strongly encouraged 12 | environment: pypi 13 | permissions: 14 | # IMPORTANT: this permission is mandatory for trusted publishing 15 | id-token: write 16 | steps: 17 | # retrieve your distributions here 18 | - uses: actions/checkout@v1 19 | - name: Set up Python 20 | uses: actions/setup-python@v1 21 | with: 22 | python-version: '3.x' 23 | - name: Install dependencies 24 | run: | 25 | python -m pip install --upgrade pip 26 | pip install setuptools wheel 27 | - name: Build 28 | run: | 29 | python setup.py sdist bdist_wheel 30 | - name: Publish package distributions to PyPI 31 | uses: pypa/gh-action-pypi-publish@release/v1 32 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Run unit tests on Windows, Ubuntu and Mac 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | 7 | test: 8 | name: ${{ matrix.os }} / ${{ matrix.python_version }} 9 | runs-on: ${{ matrix.os }}-latest 10 | strategy: 11 | fail-fast: false 12 | matrix: 13 | os: [Ubuntu] 14 | python_version: ["3.9.16"] 15 | 16 | steps: 17 | - uses: actions/checkout@v2 18 | - name: Set up Python 19 | uses: actions/setup-python@v5 20 | with: 21 | python-version: ${{ matrix.python_version }} 22 | architecture: x64 23 | 24 | - name: install 25 | run: | 26 | pip --use-deprecated=legacy-resolver install -r requirements.txt 27 | pip --use-deprecated=legacy-resolver install -r tests/requirements.txt 28 | - name: test 29 | run: | 30 | pip freeze 31 | make test 32 | - name: Upload coverage 33 | uses: codecov/codecov-action@v1 34 | with: 35 | name: ${{ matrix.os }} Python ${{ matrix.python-version }} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # April 2016 2 | # reference: https://github.com/github/gitignore/blob/master/Python.gitignore 3 | # Byte-compiled / optimized / DLL files 4 | __pycache__/ 5 | *.py[cod] 6 | *$py.class 7 | 8 | # C extensions 9 | *.so 10 | 11 | # Distribution / packaging 12 | .Python 13 | env/ 14 | build/ 15 | develop-eggs/ 16 | dist/ 17 | downloads/ 18 | eggs/ 19 | .eggs/ 20 | lib/ 21 | lib64/ 22 | parts/ 23 | sdist/ 24 | var/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .coverage 43 | .coverage.* 44 | .cache 45 | nosetests.xml 46 | coverage.xml 47 | *,cover 48 | .hypothesis/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | 58 | # Flask stuff: 59 | instance/ 60 | .webassets-cache 61 | 62 | # Scrapy stuff: 63 | .scrapy 64 | 65 | # Sphinx documentation 66 | docs/_build/ 67 | 68 | # PyBuilder 69 | target/ 70 | 71 | # IPython Notebook 72 | .ipynb_checkpoints 73 | 74 | # pyenv 75 | .python-version 76 | 77 | # celery beat schedule file 78 | celerybeat-schedule 79 | 80 | # dotenv 81 | .env 82 | 83 | # virtualenv 84 | venv/ 85 | ENV/ 86 | 87 | # Spyder project settings 88 | .spyderproject 89 | 90 | # Rope project settings 91 | .ropeproject 92 | *~ 93 | commons/ 94 | commons 95 | .moban.hashes 96 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "examples/robotchef_allinone"] 2 | path = examples/robotchef_allinone 3 | url = https://github.com/python-lml/robotchef_allinone.git 4 | [submodule "examples/robotchef_allinone_lml"] 5 | path = examples/robotchef_allinone_lml 6 | url = https://github.com/python-lml/robotchef_allinone_lml.git 7 | [submodule "examples/robotchef"] 8 | path = examples/robotchef 9 | url = https://github.com/python-lml/robotchef.git 10 | [submodule "examples/robotchef_cook"] 11 | path = examples/robotchef_cook 12 | url = https://github.com/python-lml/robotchef_cook.git 13 | [submodule "examples/robotchef_britishcuisine"] 14 | path = examples/robotchef_britishcuisine 15 | url = https://github.com/python-lml/robotchef_britishcuisine.git 16 | [submodule "examples/robotchef_chinesecuisine"] 17 | path = examples/robotchef_chinesecuisine 18 | url = https://github.com/python-lml/robotchef_chinesecuisine.git 19 | [submodule "examples/v2/robotchef_v2"] 20 | path = examples/v2/robotchef_v2 21 | url = https://github.com/python-lml/robotchef_v2.git 22 | [submodule "examples/v2/robotchef_britishcuisine"] 23 | path = examples/v2/robotchef_britishcuisine 24 | url = https://github.com/python-lml/robotchef_britishcuisine_v2 25 | [submodule "examples/v2/robotchef_api"] 26 | path = examples/v2/robotchef_api 27 | url = https://github.com/python-lml/robotchef_api 28 | -------------------------------------------------------------------------------- /.isort.cfg: -------------------------------------------------------------------------------- 1 | [settings] 2 | line_length=79 3 | known_first_party= 4 | known_third_party=nose 5 | indent=' ' 6 | multi_line_output=3 7 | length_sort=1 8 | default_section=FIRSTPARTY 9 | no_lines_before=LOCALFOLDER 10 | sections=FUTURE,STDLIB,FIRSTPARTY,THIRDPARTY,LOCALFOLDER 11 | -------------------------------------------------------------------------------- /.moban.d/CUSTOM_README.rst.jj2: -------------------------------------------------------------------------------- 1 | {% extends "README.rst.jj2" %} 2 | 3 | {%block features%} 4 | {%include "description.rst.jj2" %} 5 | {%endblock%} 6 | 7 | {%block bottom_block%} 8 | lml enabled project 9 | ================================================================================ 10 | 11 | {%include "lml-enabled-projects.rst.jj2" %} 12 | 13 | License 14 | ================================================================================ 15 | 16 | New BSD 17 | 18 | {%endblock%} -------------------------------------------------------------------------------- /.moban.d/custom_setup.py.jj2: -------------------------------------------------------------------------------- 1 | {%extends "setup.py.jj2"%} 2 | 3 | 4 | {%block platform_block%} 5 | {%endblock %} -------------------------------------------------------------------------------- /.moban.d/description.rst.jj2: -------------------------------------------------------------------------------- 1 | **{{name}}** seamlessly finds the lml based plugins from your current python 2 | environment but loads your plugins on demand. It is designed to support 3 | plugins that have external dependencies, especially bulky and/or 4 | memory hungry ones. {{name}} provides the plugin management system only and the 5 | plugin interface is on your shoulder. 6 | 7 | **{{name}}** enabled applications helps your customers [#f1]_ in two ways: 8 | 9 | #. Your customers could cherry-pick the plugins from pypi per python environment. 10 | They could remove a plugin using `pip uninstall` command. 11 | #. Only the plugins used at runtime gets loaded into computer memory. 12 | 13 | When you would use **lml** to refactor your existing code, it aims to flatten the 14 | complexity and to shrink the size of your bulky python library by 15 | distributing the similar functionalities across its plugins. However, you as 16 | the developer need to do the code refactoring by yourself and lml would lend you a hand. 17 | 18 | .. [#f1] the end developers who uses your library and packages achieve their 19 | objectives. 20 | 21 | 22 | Quick start 23 | ================================================================================ 24 | 25 | The following code tries to get you started quickly with **non-lazy** loading. 26 | 27 | .. code-block:: python 28 | 29 | from lml.plugin import PluginInfo, PluginManager 30 | 31 | 32 | @PluginInfo("cuisine", tags=["Portable Battery"]) 33 | class Boost(object): 34 | def make(self, food=None, **keywords): 35 | print("I can cook %s for robots" % food) 36 | 37 | 38 | class CuisineManager(PluginManager): 39 | def __init__(self): 40 | PluginManager.__init__(self, "cuisine") 41 | 42 | def get_a_plugin(self, food_name=None, **keywords): 43 | return PluginManager.get_a_plugin(self, key=food_name, **keywords) 44 | 45 | 46 | if __name__ == '__main__': 47 | manager = CuisineManager() 48 | chef = manager.get_a_plugin("Portable Battery") 49 | chef.make() 50 | 51 | 52 | At a glance, above code simply replaces the Factory pattern should you write 53 | them without lml. What's not obvious is, that once you got hands-on with it, 54 | you can start work on how to do **lazy** loading. 55 | 56 | -------------------------------------------------------------------------------- /.moban.d/docs/source/custom_conf.py.jj2: -------------------------------------------------------------------------------- 1 | {% include "docs/source/myconf.py.jj2" %} 2 | 3 | master_doc = "index" -------------------------------------------------------------------------------- /.moban.d/docs/source/custom_index.rst.jj2: -------------------------------------------------------------------------------- 1 | `{{name}}` - {{description}} 2 | ================================================================================ 3 | 4 | 5 | :Author: C.W. 6 | :Source code: http://github.com/{{organisation}}/{{name}}.git 7 | :Issues: http://github.com/{{organisation}}/{{name}}/issues 8 | :License: New BSD License 9 | :Released: |version| 10 | :Generated: |today| 11 | 12 | 13 | Introduction 14 | ------------- 15 | 16 | {% include "description.rst.jj2" %} 17 | 18 | 19 | Documentation 20 | ---------------- 21 | 22 | .. toctree:: 23 | :maxdepth: 2 24 | 25 | design 26 | tutorial 27 | lml_log 28 | api 29 | 30 | {%include "lml-enabled-projects.rst.jj2" %} 31 | -------------------------------------------------------------------------------- /.moban.d/docs/source/myconf.py.jj2: -------------------------------------------------------------------------------- 1 | {%extends "docs/source/conf.py.jj2"%} 2 | 3 | {%block SPHINX_EXTENSIONS%} 4 | 'sphinx.ext.napoleon', 5 | 'sphinxcontrib.spelling' 6 | {%endblock%} 7 | 8 | {%block additional_config%} 9 | spelling_lang = 'en_GB' 10 | spelling_word_list_filename = 'spelling_wordlist.txt' 11 | {%endblock%} -------------------------------------------------------------------------------- /.moban.d/lml-enabled-projects.rst.jj2: -------------------------------------------------------------------------------- 1 | Beyond the documentation above, here is a list of projects using lml: 2 | 3 | #. `pyexcel `_ 4 | #. `pyecharts `_ 5 | #. `moban `_ 6 | 7 | lml is available on these distributions: 8 | 9 | #. `ARCH linux `_ 10 | #. `Conda forge `_ 11 | #. `OpenSuse `_ 12 | 13 | -------------------------------------------------------------------------------- /.moban.d/requirements.txt.jj2: -------------------------------------------------------------------------------- 1 | {% for dependency in dependencies: %} 2 | {{dependency}} 3 | {% endfor %} 4 | -------------------------------------------------------------------------------- /.moban.d/test.sh.jj2: -------------------------------------------------------------------------------- 1 | {%block pretest%} 2 | {%endblock%} 3 | {%if external_module_library%} 4 | {%set package=external_module_library%} 5 | {%else%} 6 | {%if command_line_interface%} 7 | {%set package=command_line_interface + '_cli' %} 8 | {%else%} 9 | {%set package=name%} 10 | {%endif%} 11 | {%endif%} 12 | pip freeze 13 | cd tests/test_plugin 14 | python setup.py install 15 | cd - 16 | pytest tests --verbosity=3 --cov=lml --doctest-glob=*.rst && flake8 . --exclude=.moban.d,docs,setup.py {%block flake8_options%}--builtins=unicode,xrange,long{%endblock%} 17 | 18 | {%block posttest%} 19 | {%endblock%} 20 | -------------------------------------------------------------------------------- /.moban.yml: -------------------------------------------------------------------------------- 1 | configuration: 2 | template_dir: 3 | - "git://github.com/moremoban/pypi-mobans.git?submodule=true&brach=dev!/statics" 4 | - "git://github.com/moremoban/pypi-mobans.git?submodule=true&branch=dev!/templates" 5 | - ".moban.d" 6 | configuration: lml.yml 7 | targets: 8 | - README.rst: CUSTOM_README.rst.jj2 9 | - setup.py: custom_setup.py.jj2 10 | - requirements.txt: requirements.txt.jj2 11 | - "docs/source/conf.py": "docs/source/custom_conf.py.jj2" 12 | - "docs/source/index.rst": "docs/source/custom_index.rst.jj2" 13 | - test.sh: test.sh.jj2 14 | - "lml/_version.py": _version.py.jj2 15 | - output: CHANGELOG.rst 16 | configuration: changelog.yml 17 | template: CHANGELOG.rst.jj2 18 | - ".github/workflows/pythonpublish.yml": "pythonpublish.yml" 19 | - ".github/workflows/moban-update.yml": "moban-update.yml" 20 | - CONTRIBUTORS.rst: CONTRIBUTORS.rst.jj2 21 | - MANIFEST.in: MANIFEST.in.jj2 22 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | build: 9 | os: ubuntu-22.04 10 | tools: 11 | python: "3.12" 12 | 13 | # Build documentation in the docs/ directory with Sphinx 14 | sphinx: 15 | configuration: docs/source/conf.py 16 | 17 | # Optionally build your docs in additional formats such as PDF 18 | formats: 19 | - pdf 20 | 21 | python: 22 | install: 23 | - requirements: docs/requirements.txt 24 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: python 3 | notifications: 4 | email: false 5 | python: 6 | - 3.9-dev 7 | - 3.8 8 | - 3.7 9 | - 3.6 10 | before_install: 11 | - pip install -r tests/requirements.txt 12 | script: 13 | - make test 14 | after_success: 15 | codecov 16 | -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | Change log 2 | ================================================================================ 3 | 4 | 0.2.0 - 16/03/2025 5 | -------------------------------------------------------------------------------- 6 | 7 | **Updated** 8 | 9 | #. replace __import__ with importlib 10 | 11 | **Removed** 12 | 13 | #. removed python 2 support 14 | 15 | 0.1.0 - 21/10/2020 16 | -------------------------------------------------------------------------------- 17 | 18 | **Updated** 19 | 20 | #. non class object can be a plugin too 21 | #. `#20 `_: When a plugin was not 22 | installed, it now calls raise_exception method 23 | 24 | 0.0.9 - 7/1/2019 25 | -------------------------------------------------------------------------------- 26 | 27 | **Updated** 28 | 29 | #. `#11 `_: more test contents for 30 | OpenSuse package validation 31 | 32 | 0.0.8 - 4/1/2019 33 | -------------------------------------------------------------------------------- 34 | 35 | **Updated** 36 | 37 | #. `#9 `_: include tests, docs for 38 | OpenSuse package validation 39 | 40 | 0.0.7 - 17/11/2018 41 | -------------------------------------------------------------------------------- 42 | 43 | **Fixed** 44 | 45 | #. `#8 `_: get_primary_key will fail 46 | when a module is loaded later 47 | #. deprecated old style plugin scanner: scan_plugins 48 | 49 | 0.0.6 - 07/11/2018 50 | -------------------------------------------------------------------------------- 51 | 52 | **Fixed** 53 | 54 | #. Revert the version 0.0.5 changes. Raise Import error and log the exception 55 | 56 | 0.0.5 - 06/11/2018 57 | -------------------------------------------------------------------------------- 58 | 59 | **Fixed** 60 | 61 | #. `#6 `_: Catch and Ignore 62 | ModuleNotFoundError 63 | 64 | 0.0.4 - 07.08.2018 65 | -------------------------------------------------------------------------------- 66 | 67 | **Added** 68 | 69 | #. `#4 `_: to find plugin names with 70 | different naming patterns 71 | 72 | 0.0.3 - 12/06/2018 73 | -------------------------------------------------------------------------------- 74 | 75 | **Added** 76 | 77 | #. `dict` can be a pluggable type in addition to `function`, `class` 78 | #. get primary tag of your tag, helping you find out which category of plugins 79 | your tag points to 80 | 81 | 0.0.2 - 23/10/2017 82 | -------------------------------------------------------------------------------- 83 | 84 | **Updated** 85 | 86 | #. `pyexcel#103 `_: include 87 | LICENSE in tar ball 88 | 89 | 0.0.1 - 30/05/2017 90 | -------------------------------------------------------------------------------- 91 | 92 | **Added** 93 | 94 | #. First release 95 | -------------------------------------------------------------------------------- /CONTRIBUTORS.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 contributors 4 | ================================================================================ 5 | 6 | In alphabetical order: 7 | 8 | * `Ayan Banerjee `_ 9 | * `Fabian Affolter `_ 10 | * `Matěj Cepl `_ 11 | * `pgajdos `_ 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2017 by Onni Software Ltd. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms of the software as well 5 | as documentation, with or without modification, are permitted provided 6 | that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright notice, this 9 | list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright notice, 12 | this list of conditions and the following disclaimer in the documentation 13 | and/or other materials provided with the distribution. 14 | 15 | * Neither the name of 'lml' nor the names of the contributors 16 | may not be used to endorse or promote products derived from this software 17 | without specific prior written permission. 18 | 19 | THIS SOFTWARE AND DOCUMENTATION IS PROVIDED BY THE COPYRIGHT HOLDERS AND 20 | CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT 21 | NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER 23 | OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 24 | EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 25 | PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 26 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 27 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 28 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 29 | SOFTWARE AND DOCUMENTATION, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 30 | DAMAGE. -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst 2 | include LICENSE 3 | include CHANGELOG.rst 4 | include CONTRIBUTORS.rst 5 | recursive-include tests * 6 | recursive-include docs * 7 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: test 2 | 3 | test: 4 | bash test.sh 5 | 6 | document: 7 | python setup.py install 8 | sphinx-build -b html docs/source/ docs/build 9 | 10 | spelling: 11 | sphinx-build -b spelling docs/source/ docs/build/spelling 12 | 13 | uml: 14 | plantuml -tsvg -o ../_static/images/ docs/source/uml/*.uml 15 | 16 | format: 17 | isort -y $(find lml -name "*.py"|xargs echo) $(find tests -name "*.py"|xargs echo) 18 | black -l 79 lml 19 | black -l 79 tests 20 | black -l 79 examples 21 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ================================================================================ 2 | lml - Load me later. A lazy plugin management system. 3 | ================================================================================ 4 | 5 | 6 | .. image:: https://codecov.io/github/python-lml/lml/coverage.png 7 | :target: https://codecov.io/github/python-lml/lml 8 | .. image:: https://badge.fury.io/py/lml.svg 9 | :target: https://pypi.org/project/lml 10 | 11 | .. image:: https://pepy.tech/badge/lml/month 12 | :target: https://pepy.tech/project/lml 13 | 14 | .. image:: https://img.shields.io/github/stars/python-lml/lml.svg?style=social&maxAge=3600&label=Star 15 | :target: https://github.com/python-lml/lml/stargazers 16 | 17 | .. image:: https://img.shields.io/static/v1?label=continuous%20templating&message=%E6%A8%A1%E7%89%88%E6%9B%B4%E6%96%B0&color=blue&style=flat-square 18 | :target: https://moban.readthedocs.io/en/latest/#at-scale-continous-templating-for-open-source-projects 19 | 20 | .. image:: https://img.shields.io/static/v1?label=coding%20style&message=black&color=black&style=flat-square 21 | :target: https://github.com/psf/black 22 | 23 | .. image:: https://readthedocs.org/projects/lml/badge/?version=latest 24 | :target: http://lml.readthedocs.org/en/latest/ 25 | 26 | **lml** seamlessly finds the lml based plugins from your current python 27 | environment but loads your plugins on demand. It is designed to support 28 | plugins that have external dependencies, especially bulky and/or 29 | memory hungry ones. lml provides the plugin management system only and the 30 | plugin interface is on your shoulder. 31 | 32 | **lml** enabled applications helps your customers [#f1]_ in two ways: 33 | 34 | #. Your customers could cherry-pick the plugins from pypi per python environment. 35 | They could remove a plugin using `pip uninstall` command. 36 | #. Only the plugins used at runtime gets loaded into computer memory. 37 | 38 | When you would use **lml** to refactor your existing code, it aims to flatten the 39 | complexity and to shrink the size of your bulky python library by 40 | distributing the similar functionalities across its plugins. However, you as 41 | the developer need to do the code refactoring by yourself and lml would lend you a hand. 42 | 43 | .. [#f1] the end developers who uses your library and packages achieve their 44 | objectives. 45 | 46 | 47 | Quick start 48 | ================================================================================ 49 | 50 | The following code tries to get you started quickly with **non-lazy** loading. 51 | 52 | .. code-block:: python 53 | 54 | from lml.plugin import PluginInfo, PluginManager 55 | 56 | 57 | @PluginInfo("cuisine", tags=["Portable Battery"]) 58 | class Boost(object): 59 | def make(self, food=None, **keywords): 60 | print("I can cook %s for robots" % food) 61 | 62 | 63 | class CuisineManager(PluginManager): 64 | def __init__(self): 65 | PluginManager.__init__(self, "cuisine") 66 | 67 | def get_a_plugin(self, food_name=None, **keywords): 68 | return PluginManager.get_a_plugin(self, key=food_name, **keywords) 69 | 70 | 71 | if __name__ == '__main__': 72 | manager = CuisineManager() 73 | chef = manager.get_a_plugin("Portable Battery") 74 | chef.make() 75 | 76 | 77 | At a glance, above code simply replaces the Factory pattern should you write 78 | them without lml. What's not obvious is, that once you got hands-on with it, 79 | you can start work on how to do **lazy** loading. 80 | 81 | 82 | Installation 83 | ================================================================================ 84 | 85 | 86 | You can install lml via pip: 87 | 88 | .. code-block:: bash 89 | 90 | $ pip install lml 91 | 92 | 93 | or clone it and install it: 94 | 95 | .. code-block:: bash 96 | 97 | $ git clone https://github.com/python-lml/lml.git 98 | $ cd lml 99 | $ python setup.py install 100 | 101 | lml enabled project 102 | ================================================================================ 103 | 104 | Beyond the documentation above, here is a list of projects using lml: 105 | 106 | #. `pyexcel `_ 107 | #. `pyecharts `_ 108 | #. `moban `_ 109 | 110 | lml is available on these distributions: 111 | 112 | #. `ARCH linux `_ 113 | #. `Conda forge `_ 114 | #. `OpenSuse `_ 115 | 116 | 117 | License 118 | ================================================================================ 119 | 120 | New BSD 121 | -------------------------------------------------------------------------------- /changelog.yml: -------------------------------------------------------------------------------- 1 | name: lml 2 | organisation: python-lml 3 | releases: 4 | - changes: 5 | - action: Updated 6 | details: 7 | - "replace __import__ with importlib" 8 | - action: Removed 9 | details: 10 | - "removed python 2 support" 11 | date: 16/03/2025 12 | version: 0.2.0 13 | - changes: 14 | - action: Updated 15 | details: 16 | - "non class object can be a plugin too" 17 | - "`#20`: When a plugin was not installed, it now calls raise_exception method" 18 | date: 21/10/2020 19 | version: 0.1.0 20 | - changes: 21 | - action: Updated 22 | details: 23 | - "`#11`: more test contents for OpenSuse package validation" 24 | date: 7/1/2019 25 | version: 0.0.9 26 | - changes: 27 | - action: Updated 28 | details: 29 | - "`#9`: include tests, docs for OpenSuse package validation" 30 | date: 4/1/2019 31 | version: 0.0.8 32 | - changes: 33 | - action: Fixed 34 | details: 35 | - "`#8`: get_primary_key will fail when a module is loaded later" 36 | - "deprecated old style plugin scanner: scan_plugins" 37 | date: 17/11/2018 38 | version: 0.0.7 39 | - changes: 40 | - action: Fixed 41 | details: 42 | - "Revert the version 0.0.5 changes. Raise Import error and log the exception" 43 | date: 07/11/2018 44 | version: 0.0.6 45 | - changes: 46 | - action: Fixed 47 | details: 48 | - "`#6`: Catch and Ignore ModuleNotFoundError" 49 | date: 06/11/2018 50 | version: 0.0.5 51 | - changes: 52 | - action: Added 53 | details: 54 | - "`#4`: to find plugin names with different naming patterns" 55 | date: 07.08.2018 56 | version: 0.0.4 57 | - changes: 58 | - action: Added 59 | details: 60 | - "`dict` can be a pluggable type in addition to `function`, `class`" 61 | - get primary tag of your tag, helping you find out which category of plugins your tag points to 62 | date: 12/06/2018 63 | version: 0.0.3 64 | - changes: 65 | - action: Updated 66 | details: 67 | - "`pyexcel#103 `_: include LICENSE in tar ball" 68 | date: 23/10/2017 69 | version: 0.0.2 70 | - changes: 71 | - action: Added 72 | details: 73 | - First release 74 | date: 30/05/2017 75 | version: 0.0.1 76 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinxcontrib-plantuml 2 | sphinx_rtd_theme 3 | 4 | -------------------------------------------------------------------------------- /docs/source/_static/images/loading_sequence.svg: -------------------------------------------------------------------------------- 1 | bobbobrobotchefrobotcheflmllmlrobotchef_britishcuisinerobotchef_britishcuisine> robotchef "Jacket Potato"scan for plugins.read plugin chain in the moduleI can help with "Jacket Potato" and others.read the built-in robot_cuisinebuilt-in chef knows "Portable Battery"scanning doneget me a plugin that knows "Jacket Potato"robotchef_britishcuisine.bake.Bake can domake the food"I can bake Jacket Potato" -------------------------------------------------------------------------------- /docs/source/_static/images/robot_chef.svg: -------------------------------------------------------------------------------- 1 | ChefBoostFryBake -------------------------------------------------------------------------------- /docs/source/_static/images/robotchef_allinone_lml.svg: -------------------------------------------------------------------------------- 1 | lmlrobotchef_allinone_lmlPluginManagerPluginInfoCuisineManagerget_a_plugin()raise_exception()Chefmake()BoostFryBakecuisine -------------------------------------------------------------------------------- /docs/source/_static/images/robotchef_api_crd.svg: -------------------------------------------------------------------------------- 1 | lmlrobotchef_apirobotchef.robot_cuisinerobotchef_britishcuisinePluginManagerPluginInfoChainPluginInfoCuisineManagerget_a_plugin()raise_exception()Chefmake()BoostFryBakerobotchef_v2registers plugin infocuisine -------------------------------------------------------------------------------- /docs/source/_static/images/robotchef_crd.svg: -------------------------------------------------------------------------------- 1 | lmlrobotchefrobotchef.robot_cuisinerobotchef_britishcuisinePluginManagerPluginInfoChainPluginInfoCuisineManagerget_a_plugin()raise_exception()Chefmake()BoostFryBakeregisters plugin infocuisine -------------------------------------------------------------------------------- /docs/source/allinone_lml_tutorial.rst: -------------------------------------------------------------------------------- 1 | Robot Chef all in one package with lml 2 | ================================================================================ 3 | 4 | Now let us bring in lml and see see how the lml package can be used 5 | to rewrite **Robot Chef** but in a single package. This chapter introduces 6 | two classes: :class:`lml.plugin.PluginManager` and :class:`lml.plugin.PluginInfo`. 7 | And show how those classes can be used to make factory pattern. 8 | Meanwhile, it demonstrates that the lml based plugins can be made to load 9 | immediately and in a single package. And this sections helps you to understand 10 | the next section where we will make the plugins to be loaded later. 11 | 12 | Demo 13 | -------------------------------------------------------------------------------- 14 | 15 | Please navigate to robotchef_allinone_lml and its packages. Do the following:: 16 | 17 | $ git clone https://github.com/python-lml/robotchef_allinone_lml 18 | $ cd robotchef_allinone_lml 19 | $ python setup.py install 20 | 21 | And then you could try:: 22 | 23 | $ robotchef_allinone_lml "Fish and Chips" 24 | I can fry Fish and Chips 25 | 26 | Lml plugins and plugin manager 27 | ------------------------------- 28 | 29 | .. image:: _static/images/robotchef_allinone_lml.svg 30 | 31 | .. _cuisine_manager: 32 | 33 | plugin.py 34 | ++++++++++ 35 | 36 | `CuisineManager` inherits from :class:`~lml.plugin.PluginManager` class and 37 | replaces the static registry `PLUGINS` and the modular function `get_a_plugin`. 38 | Please note that `CuisineManager` declares that it is a manager for plugin_type named 39 | **cuisine**. 40 | 41 | 42 | .. literalinclude:: ../../examples/robotchef_allinone_lml/robotchef_allinone_lml/plugin.py 43 | :language: python 44 | :lines: 8-17 45 | 46 | Next, the :class:`~lml.plugin.PluginInfo` decorates all Chef's subclasses as 47 | **cuisine** plugins and register the decorated classes with the manager class 48 | for **cuisine**, `CuisineManager`. The food names become the tags which will 49 | be used to look up the classes. 50 | 51 | .. literalinclude:: ../../examples/robotchef_allinone_lml/robotchef_allinone_lml/plugin.py 52 | :language: python 53 | :lines: 1, 18- 54 | 55 | Here is the :ref:`rb3-diff-rb0-plugin`. 56 | 57 | main.py 58 | +++++++++ 59 | 60 | The main code has been updated to reflect the changes in plugin.py. `CuisineManager` 61 | has to be instantiated to be the a factory manager. 62 | 63 | .. literalinclude:: ../../examples/robotchef_allinone_lml/robotchef_allinone_lml/main.py 64 | :diff: ../../examples/robotchef_allinone/robotchef_allinone/main.py 65 | 66 | Remember this interaction:: 67 | 68 | $ robotchef "Portable Battery" 69 | I can cook Portable Battery for robots 70 | 71 | The response comes from class `Boost`. It is obtained via CuisineManager when user types 72 | 'Portable Battery'. And the food parameter was passed to the instance of Boost. 73 | `make` method was called and it prints 'I can cook Portable Battery for robots'. 74 | 75 | 76 | See also 77 | ------------- 78 | 79 | #. pyexcel-chart: `use lml to refactor existing plugins `_ 80 | 81 | -------------------------------------------------------------------------------- /docs/source/allinone_tutorial.rst: -------------------------------------------------------------------------------- 1 | Robot Chef all in one package without lml 2 | ================================================================================ 3 | 4 | In this chapter, we are going to see how **Robot Chef** could be implemented 5 | without lml. In later on chapters, we will bring in **lml** step by step. 6 | 7 | Demo 8 | -------------------------------------------------------------------------------- 9 | 10 | Please checkout the robot chef example:: 11 | 12 | $ git clone https://github.com/python-lml/robotchef_allinone 13 | $ cd robotchef_allinone 14 | $ python setup.py install 15 | 16 | And then you could try:: 17 | 18 | $ robotchef_allinone "Fish and Chips" 19 | I can fry Fish and Chips 20 | 21 | 22 | Conventional plugin and plugin factory 23 | --------------------------------------- 24 | 25 | plugin.py 26 | +++++++++++ 27 | 28 | .. image:: _static/images/robot_chef.svg 29 | 30 | `Chef` is the plugin interface that makes food. Boost, Bake and Fry are the 31 | actual implementations. Boost are for "robots". Bake and Fry are for human. 32 | 33 | .. note:: 34 | 35 | The plugin interface is your responsibility. **lml** gives the freedom to you. 36 | 37 | .. literalinclude:: ../../examples/robotchef_allinone/robotchef_allinone/plugin.py 38 | :language: python 39 | :lines: 5-30 40 | 41 | Line 13, class `Chef` defines the plugin class interface. For robotchef, `make` is 42 | defined to illustrate the functionality. Naturally you will be deciding the 43 | interface for your plugins. 44 | 45 | Some of you might suggest that class `Chef` is unnecessary because 46 | Python uses duck-typing, meaning as long as the plugin has `make` method, 47 | it should work. Yes, it would work but it is a short term solution. 48 | Look at the long term, you could pass on additional functionalities 49 | through class `Chef` without touching the plugins. What's more, for 50 | plugin developers, a clear defined interface is better than no class 51 | at all. And I believe the functions of a real plugin are more than 52 | just one here. 53 | 54 | Next in the plugin.py file, PLUGINS is the dictionary that has food name as 55 | key and Chef descendants as values. `get_a_plugin` method returns a Chef or 56 | raises NoChefException. 57 | 58 | .. literalinclude:: ../../examples/robotchef_allinone/robotchef_allinone/plugin.py 59 | :language: python 60 | :lines: 33- 61 | 62 | main.py 63 | +++++++++++ 64 | 65 | Let us glimpse through the main code: 66 | 67 | .. literalinclude:: ../../examples/robotchef_allinone/robotchef_allinone/main.py 68 | :language: python 69 | 70 | The code takes the first command option as food name and feeds it to the 71 | factory method `get_a_plugin`, which returns a Chef to "make" the food. 72 | If no chef was found, it prints the default string: I do not know. 73 | 74 | That is all about the all in one **Robot Chef**. 75 | -------------------------------------------------------------------------------- /docs/source/api.rst: -------------------------------------------------------------------------------- 1 | API documentation 2 | ==================== 3 | 4 | .. automodule:: lml.loader 5 | 6 | .. autofunction:: scan_plugins_regex 7 | 8 | .. automodule:: lml.plugin 9 | 10 | .. autoclass:: PluginInfo 11 | 12 | .. autoclass:: PluginInfoChain 13 | 14 | .. autoclass:: PluginManager 15 | :members: 16 | -------------------------------------------------------------------------------- /docs/source/api_tutorial.rst: -------------------------------------------------------------------------------- 1 | Robot Chef version 2: Use lml to write a shared library 2 | ================================================================================ 3 | 4 | In previous chapter, lml was used to split all in one Robot Chef into 5 | one core package and several plugins module and packages. In this 6 | chapter, we are going to go one step further to split the core package into 7 | two so as to showcase how to use lml to write a shared api library. 8 | 9 | .. image:: _static/images/robotchef_api_crd.svg 10 | 11 | Demo 12 | -------------------------------------------------------------------------------- 13 | 14 | Please checkout the following examples:: 15 | 16 | $ virtualenv --no-site-packages robotchefv2 17 | $ source robotchefv2/bin/activate 18 | $ git clone https://github.com/python-lml/robotchef_v2 19 | $ cd robotchef_v2 20 | $ python setup.py install 21 | $ cd .. 22 | $ git clone https://github.com/python-lml/robotchef_api 23 | $ cd robotchef_api 24 | $ python setup.py install 25 | 26 | And then you can type in and test the second version of Robot Chef:: 27 | 28 | $ robotchef_v2 "Portable Battery" 29 | I can cook Portable Battery for robots 30 | $ robotchef_v2 "Jacket Potato" 31 | I do not know how to cook Jacket Potato 32 | 33 | In order to add "Jacket Potato" in the know-how, you would need to install 34 | robotchef_britishcuisine in this folder:: 35 | 36 | $ git clone https://github.com/python-lml/robotchef_britishcuisine_v2 37 | $ cd robotchef_britishcuisine_v2 38 | $ python setup.py install 39 | $ robotchef_v2 "Jacket Potato" 40 | I can bake Jacket Potato 41 | 42 | Robot Chef v2 code 43 | ----------------------- 44 | 45 | Let us look at main code robotchef_v2: 46 | 47 | .. literalinclude:: ../../examples/v2/robotchef_v2/robotchef_v2/main.py 48 | :diff: ../../examples/robotchef/robotchef/main.py 49 | 50 | 51 | The code highlighted in red are removed from main.py and are placed into 52 | **robotchef_api** package. And robotchef_v2 becomes the consumer of 53 | the robotchef api. 54 | 55 | And plugin.py and robot_cuisine has been moved to **robotchef_api** package. 56 | 57 | Robot Chef API 58 | -------------------- 59 | 60 | Now let us look at robotchef_api. In the following directory listing, the 61 | plugin.py And robot_cuisine is exactly the same as the :ref:`plugin.py ` 62 | and :ref:`robot_cuisine ` in robotchef:: 63 | 64 | __init__.py plugin.py robot_cuisine 65 | 66 | Notably, the plugin loader is put in the __init__.py: 67 | 68 | .. literalinclude:: ../../examples/v2/robotchef_api/robotchef_api/__init__.py 69 | :language: python 70 | 71 | scan_plugins_regex here loads all modules that start with "robotchef_" and as well as 72 | the module `robotchef_api.robot_cuisine` in the white_list. 73 | 74 | This is how you will write the main component as a library. 75 | 76 | Built-in plugin and Standalone plugin 77 | -------------------------------------- 78 | 79 | You may have noticed that a copy of robotchef_britishcuisine is placed in v2 directory. 80 | Why not using the same one above v2 directory? although they are almost identical, 81 | there is a minor difference. robotchef_britishcuisine in v2 directory depends on 82 | robotchef_api but the other British cuisine package depends on robotchef. Hence, if you 83 | look at the fry.py in v2 directory, you will notice a slight difference: 84 | 85 | .. literalinclude:: ../../examples/v2/robotchef_britishcuisine/robotchef_britishcuisine/fry.py 86 | :diff: ../../examples/robotchef_britishcuisine/robotchef_britishcuisine/fry.py 87 | -------------------------------------------------------------------------------- /docs/source/appendix.rst: -------------------------------------------------------------------------------- 1 | Appendix 2 | =============== 3 | 4 | .. _rb-plugin: 5 | 6 | Robot Chef plugin.py 7 | ---------------- 8 | 9 | .. literalinclude:: ../../examples/robotchef/robotchef/plugin.py 10 | :language: python 11 | 12 | 13 | Robot Chef Version 3 14 | ------------------------ 15 | 16 | .. _rb3-diff-rb0-plugin: 17 | 18 | code difference with Robot Chef All In One solution: plugin.py 19 | ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 20 | 21 | .. literalinclude:: ../../examples/robotchef_allinone_lml/robotchef_allinone_lml/plugin.py 22 | :diff: ../../examples/robotchef_allinone/robotchef_allinone/plugin.py 23 | 24 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | DESCRIPTION = ( 3 | 'Load me later. A lazy plugin management system.' + 4 | '' 5 | ) 6 | # Configuration file for the Sphinx documentation builder. 7 | # 8 | # This file only contains a selection of the most common options. For a full 9 | # list see the documentation: 10 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 11 | 12 | # -- Path setup -------------------------------------------------------------- 13 | 14 | # If extensions (or modules to document with autodoc) are in another directory, 15 | # add these directories to sys.path here. If the directory is relative to the 16 | # documentation root, use os.path.abspath to make it absolute, like shown here. 17 | # 18 | # import os 19 | # import sys 20 | # sys.path.insert(0, os.path.abspath('.')) 21 | 22 | # -- Project information ----------------------------------------------------- 23 | 24 | project = 'lml' 25 | copyright = '2017-2025 C.W.' 26 | author = 'C.W.' 27 | # The short X.Y version 28 | version = '0.2.0' 29 | # The full version, including alpha/beta/rc tags 30 | release = '0.2.0' 31 | 32 | # -- General configuration --------------------------------------------------- 33 | 34 | # Add any Sphinx extension module names here, as strings. They can be 35 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 36 | # ones. 37 | extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.viewcode',] 38 | 39 | # Add any paths that contain templates here, relative to this directory. 40 | templates_path = ['_templates'] 41 | 42 | # The language for content autogenerated by Sphinx. Refer to documentation 43 | # for a list of supported languages. 44 | # 45 | # This is also used if you do content translation via gettext catalogs. 46 | # Usually you set "language" from the command line for these cases. 47 | language = 'en' 48 | 49 | # List of patterns, relative to source directory, that match files and 50 | # directories to ignore when looking for source files. 51 | # This pattern also affects html_static_path and html_extra_path. 52 | exclude_patterns = [] 53 | 54 | 55 | # -- Options for HTML output ------------------------------------------------- 56 | 57 | # The theme to use for HTML and HTML Help pages. See the documentation for 58 | # a list of builtin themes. 59 | # 60 | html_theme = 'sphinx_rtd_theme' 61 | 62 | # Add any paths that contain custom static files (such as style sheets) here, 63 | # relative to this directory. They are copied after the builtin static files, 64 | # so a file named "default.css" will overwrite the builtin "default.css". 65 | html_static_path = ['_static'] 66 | 67 | # -- Extension configuration ------------------------------------------------- 68 | # -- Options for intersphinx extension --------------------------------------- 69 | 70 | # Example configuration for intersphinx: refer to the Python standard library. 71 | intersphinx_mapping = {'python': ('https://docs.python.org/3', 72 | 'python-inv.txt')} 73 | # TODO: html_theme not configurable upstream 74 | html_theme = 'sphinx_rtd_theme' 75 | 76 | # TODO: DESCRIPTION not configurable upstream 77 | texinfo_documents = [ 78 | ('index', 'lml', 79 | 'lml Documentation', 80 | 'C.W.', 'lml', 81 | DESCRIPTION, 82 | 'Miscellaneous'), 83 | ] 84 | intersphinx_mapping.update({ 85 | }) 86 | master_doc = "index" 87 | 88 | master_doc = "index" -------------------------------------------------------------------------------- /docs/source/design.rst: -------------------------------------------------------------------------------- 1 | Design idea 2 | ================================================================================ 3 | 4 | The idea, to load the plugins later, originated from pyexcel project [#f1]_ which uses 5 | loosely coupled plugins to extend the main package to read more file formats. During 6 | its code growth, the code in pyexcel packages to manage the external and internal 7 | plugins becomes a independent library, lml. 8 | 9 | Lml is similar to **Factories** in 10 | Zope Component Architecture [#f2]_. It provides functionalities to 11 | discover, register and load lml based plugins. It cares how the meta data were 12 | written but it does NOT care how the plugin interface is written. 13 | 14 | Simply, lml promises to load your external dependency when they are used, but 15 | only when you follow lazy-loading design principle below. Otherwise, lml does 16 | immediate import and takes away the developer's responsibility to manage plugin 17 | registry and discovery. 18 | 19 | In terms of extensibility of your proud package, lml keeps the door open even 20 | if you use lml for immediate import. As a developer, you give the choice to other 21 | contributor to write up a plugin for your package. As long as the user would have 22 | installed community created extensions, lml will discover them and use them. 23 | 24 | 25 | Plugin discovery 26 | -------------------- 27 | 28 | Prior to lml, three different ways of loading external plugins have been tried in pyexcel. 29 | namespace package [#f3]_ comes from Python 3 or pkgutil style in Python 2 and 3. 30 | It allows the developer to split a bigger packages into a smaller ones and 31 | publish them separately. sphinxcontrib [#f4]_ uses a typical namespace package based 32 | method. However, namespace package places a strict requirement 33 | on the module's `__init__.py`: nothing other than name space declaration should 34 | be present. It means no module level functions can be place there. This restriction 35 | forces the plugin to be driven by the main package but the plugin cannot use 36 | the main package as its own library to do specific things. So namespace package 37 | was ruled out. 38 | 39 | The Flask extension management system was used early versions of pyexcel(=<0.21). 40 | This system manipulates sys.path so that your plugin package appears in the namespace 41 | of your main package. For example, there is a xls plugin called pyexcel-xls. To 42 | import it, you can use "import pyexcel.ext.xls". The shortcomings are: 43 | 44 | #. explicit statement "import pyexcel.ext.xls" becomes a useless statement in your code. 45 | static code analyser(flake8/pep8/pycharm) would flag it up. 46 | #. you have to explicitly import it. Otherwise, your plugin is not imported. 47 | `PR 7 `_ of pyexcel-io has extended 48 | discussion on this topic. 49 | #. flask extension management system become deprecated by itself in Flask's recent 50 | development since 2016. 51 | 52 | In order to overcome those shortcomings, implicit imports were coded into module's 53 | `__init__.py`. By iterating through currently installed modules in your python 54 | environment, the relevant plugins are imported automatically. 55 | 56 | lml uses implicit import. In order to manage the plugins, pip can be used to 57 | install cherry-picked plugins or to remove unwanted plugins. In the situation 58 | where two plugins perform the same thing but have to co-exist in your current 59 | python path, you can nominate one plugin to be picked. 60 | 61 | Plugin registration 62 | -------------------------------------------------------------------------------- 63 | 64 | In terms of plugin registrations, three different approaches have been tried. 65 | Monkey-patching was easy to implement. When a plugin is imported, it loads 66 | the plugin dictionary from the main package and add itself. But it is generally 67 | perceived as a "bad" idea. Another way of doing it is to place the plugin code 68 | in the main component and the plugin just need to declare a dictionary as the 69 | plugin's meta data. The main package register the meta data when it is imported. 70 | tablib [#f5]_ uses such a approach. The third way is to use meta-classes. 71 | M. Alchin (2008) [#f6]_ explained how meta class can be used to register plugin 72 | classes in a simpler way. 73 | 74 | lml uses meta data for plugin registration. Since lml load your plugin later, 75 | the meta data is stored in the module's __init__.py. For example, to load plugins 76 | later in tablib, the 'exports' variable should be taken out from the actual 77 | class file and replace the hard reference to the classes with class path string. 78 | 79 | Plugin distribution 80 | --------------------- 81 | 82 | yapsy [#f7]_ and GEdit plugin management system [#f8]_ load plugins from file system. 83 | To install a plugin in those systems, is to copy and paste the plugin code to a 84 | designated directory. zope components, namespace packages and flask extensions 85 | can be installed via pypi. lml support the latter approach. lml plugins can be 86 | released to pypi and be installed by your end developers. 87 | 88 | Design principle 89 | ------------------ 90 | 91 | To use lml, it asks you to avoid importing your "heavy" dependencies 92 | in `__init__.py`. lml respects the independence of individual packages. You can 93 | put modular level functions in your __init__.py as long as it does not trigger 94 | immediate import of your dependency. This is to allow the individual plugin to 95 | become useful as it is, rather to be integrated with your main package. For example, 96 | pyexcel-xls can be an independent package to read and write xls data, without pyexcel. 97 | 98 | With lml, as long as your third party developer respect the plugin name prefix, 99 | they could publish their plugins as they do to any normal pypi packages. And the end 100 | developer of yours would only need to do pip install. 101 | 102 | References 103 | ------------- 104 | 105 | .. [#f1] https://github.com/pyexcel/pyexcel 106 | .. [#f2] http://zopecomponent.readthedocs.io/en/latest/ 107 | .. [#f3] https://packaging.python.org/namespace_packages/ 108 | .. [#f4] https://bitbucket.org/birkenfeld/sphinx-contrib/ 109 | .. [#f5] https://github.com/kennethreitz/tablib 110 | .. [#f6] M. Alchin, 2008, A Simple Plugin Framework, http://martyalchin.com/2008/jan/10/simple-plugin-framework/ 111 | .. [#f7] http://yapsy.sourceforge.net/ 112 | .. [#f8] https://wiki.gnome.org/Apps/Gedit/PythonPluginHowToOld 113 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | `lml` - Load me later. A lazy plugin management system. 2 | ================================================================================ 3 | 4 | 5 | :Author: C.W. 6 | :Source code: http://github.com/python-lml/lml.git 7 | :Issues: http://github.com/python-lml/lml/issues 8 | :License: New BSD License 9 | :Released: |version| 10 | :Generated: |today| 11 | 12 | 13 | Introduction 14 | ------------- 15 | 16 | **lml** seamlessly finds the lml based plugins from your current python 17 | environment but loads your plugins on demand. It is designed to support 18 | plugins that have external dependencies, especially bulky and/or 19 | memory hungry ones. lml provides the plugin management system only and the 20 | plugin interface is on your shoulder. 21 | 22 | **lml** enabled applications helps your customers [#f1]_ in two ways: 23 | 24 | #. Your customers could cherry-pick the plugins from pypi per python environment. 25 | They could remove a plugin using `pip uninstall` command. 26 | #. Only the plugins used at runtime gets loaded into computer memory. 27 | 28 | When you would use **lml** to refactor your existing code, it aims to flatten the 29 | complexity and to shrink the size of your bulky python library by 30 | distributing the similar functionalities across its plugins. However, you as 31 | the developer need to do the code refactoring by yourself and lml would lend you a hand. 32 | 33 | .. [#f1] the end developers who uses your library and packages achieve their 34 | objectives. 35 | 36 | 37 | Quick start 38 | ================================================================================ 39 | 40 | The following code tries to get you started quickly with **non-lazy** loading. 41 | 42 | .. code-block:: python 43 | 44 | from lml.plugin import PluginInfo, PluginManager 45 | 46 | 47 | @PluginInfo("cuisine", tags=["Portable Battery"]) 48 | class Boost(object): 49 | def make(self, food=None, **keywords): 50 | print("I can cook %s for robots" % food) 51 | 52 | 53 | class CuisineManager(PluginManager): 54 | def __init__(self): 55 | PluginManager.__init__(self, "cuisine") 56 | 57 | def get_a_plugin(self, food_name=None, **keywords): 58 | return PluginManager.get_a_plugin(self, key=food_name, **keywords) 59 | 60 | 61 | if __name__ == '__main__': 62 | manager = CuisineManager() 63 | chef = manager.get_a_plugin("Portable Battery") 64 | chef.make() 65 | 66 | 67 | At a glance, above code simply replaces the Factory pattern should you write 68 | them without lml. What's not obvious is, that once you got hands-on with it, 69 | you can start work on how to do **lazy** loading. 70 | 71 | 72 | 73 | Documentation 74 | ---------------- 75 | 76 | .. toctree:: 77 | :maxdepth: 2 78 | 79 | design 80 | tutorial 81 | lml_log 82 | api 83 | 84 | Beyond the documentation above, here is a list of projects using lml: 85 | 86 | #. `pyexcel `_ 87 | #. `pyecharts `_ 88 | #. `moban `_ 89 | 90 | lml is available on these distributions: 91 | 92 | #. `ARCH linux `_ 93 | #. `Conda forge `_ 94 | #. `OpenSuse `_ 95 | -------------------------------------------------------------------------------- /docs/source/lml_log.rst: -------------------------------------------------------------------------------- 1 | Logging facility 2 | ================================================================================ 3 | 4 | During the development of lml package, the logging facility helps debugging a 5 | lot. Let me show you how to enable the logs of lml. 6 | 7 | 8 | Enable the logging 9 | ------------------- 10 | 11 | Let us open robotchef's `main.py `_. Insert the highlighted codes. 12 | 13 | .. code-block:: python 14 | :emphasize-lines: 5-10 15 | 16 | import sys 17 | 18 | from robotchef.plugin import CuisineManager, NoChefException 19 | 20 | import logging 21 | import logging.config 22 | 23 | logging.basicConfig( 24 | format='%(name)s:%(lineno)d - %(levelname)s - %(message)s', 25 | level=logging.DEBUG) 26 | 27 | 28 | def main(): 29 | if len(sys.argv) < 2: 30 | sys.exit(-1) 31 | 32 | manager = CuisineManager() 33 | ... 34 | 35 | Then you will need to run the installation again:: 36 | 37 | $ cd robotchef 38 | $ python setup.py install 39 | 40 | Let us run the command again:: 41 | 42 | $ robotchef "Jacket Potato" 43 | lml.plugin:226 - DEBUG - declare 'cuisine' plugin manager 44 | lml.loader:52 - DEBUG - scanning for plugins... 45 | lml.utils:48 - DEBUG - found robotchef_allinone_lml 46 | lml.plugin.PluginInfoChain:139 - DEBUG - add robotchef_britishcuisine.fry.Fry as 'cuisine' plugin 47 | robotchef.plugin.CuisineManager:178 - DEBUG - load robotchef_britishcuisine.fry.Fry later 48 | lml.plugin.PluginInfoChain:139 - DEBUG - add robotchef_britishcuisine.bake.Bake as 'cuisine' plugin 49 | robotchef.plugin.CuisineManager:178 - DEBUG - load robotchef_britishcuisine.bake.Bake later 50 | lml.utils:48 - DEBUG - found robotchef_britishcuisine 51 | lml.plugin.PluginInfoChain:139 - DEBUG - add robotchef.robot_cuisine.electrify.Boost as 'cuisine' plugin 52 | robotchef.plugin.CuisineManager:178 - DEBUG - load robotchef.robot_cuisine.electrify.Boost later 53 | lml.utils:48 - DEBUG - found robotchef.robot_cuisine 54 | lml.loader:82 - DEBUG - scanning done 55 | robotchef.plugin.CuisineManager:160 - DEBUG - get a plugin called 56 | robotchef.plugin.CuisineManager:210 - DEBUG - import robotchef_britishcuisine.bake.Bake 57 | robotchef.plugin.CuisineManager:202 - DEBUG - load now for 'Jacket Potato' 58 | I can bake Jacket Potato 59 | 60 | Reading the log with the loading sequence, 61 | 62 | .. image:: _static/images/loading_sequence.svg 63 | 64 | Three Chef plugins were discovered: robotchef_britishcuisine.fry.Fry, 65 | robotchef_britishcuisine.bake.Bake and robotchef.robot_cuisine.electricity.Boost. 66 | However, they are not imported yet. When the robotchef try to look up a plugin, 67 | it logs "get a plugin called". And it is actual time when a plugin is imported. 68 | -------------------------------------------------------------------------------- /docs/source/lml_tutorial.rst: -------------------------------------------------------------------------------- 1 | Robot Chef distributed in multiple packages 2 | ================================================================================ 3 | 4 | In previous chapter, **Robot Chef** was written using lml but in a single 5 | package and its plugins are loaded immediately. In this chapter, we will 6 | decouple the plugin and the main package using lml. And we will 7 | demonstrates the changes needed to plugin them back with the main package. 8 | 9 | Demo 10 | -------------------------------------------------------------------------------- 11 | 12 | Do the following:: 13 | 14 | $ git clone https://github.com/python-lml/robotchef 15 | $ cd robotchef 16 | $ python setup.py install 17 | 18 | The main command line interface module does simply this:: 19 | 20 | $ robotchef "Portable Battery" 21 | I can cook Portable Battery for robots 22 | 23 | Although it does not understand all the cuisines in the world as you see 24 | as below:: 25 | 26 | $ robotchef "Jacket Potato" 27 | I do not know how to cook Jacket Potato 28 | 29 | it starts to understand it once you install Chinese cuisine package to complement 30 | its knowledge:: 31 | 32 | $ git clone https://github.com/python-lml/robotchef_britishcuisine 33 | $ cd robotchef_britishcuisine 34 | $ python setup.py install 35 | 36 | And then type in the following:: 37 | 38 | $ robotchef "Fish and Chips" 39 | I can fry Fish and Chips 40 | 41 | The more cuisine packages you install, the more dishes it understands. Here 42 | is the loading sequence: 43 | 44 | .. image:: _static/images/loading_sequence.svg 45 | 46 | 47 | Decoupling the plugins with the main package 48 | -------------------------------------------------------------------------------- 49 | 50 | .. image:: _static/images/robotchef_crd.svg 51 | 52 | 53 | In order to demonstrate the capabilities of lml, `Boost` class is singled out and 54 | placed into an internal module **robotchef.robot_cuisine**. `Fry` and `Bake` are 55 | relocated to **robotchef_britishcuisine** package, which is separately installable. 56 | :ref:`built-in` and :ref:`standalone-plugin` will explain how to *glue* them up. 57 | 58 | After the separation, in order to piece all together, a special function 59 | :meth:`lml.loader.scan_plugins` needs to be called before the plugins are used. 60 | 61 | .. literalinclude:: ../../examples/robotchef/robotchef/main.py 62 | :diff: ../../examples/robotchef_allinone_lml/robotchef_allinone_lml/main.py 63 | 64 | What's more, :meth:`lml.loader.scan_plugins` search through all 65 | installed python modules and register plugin modules that has prefix "robotchef_". 66 | 67 | The second parameter of scan_plugins is to inform pyinstaller about the 68 | package path if your package is to be packaged up using pyinstaller. 69 | `white_list` lists the built-ins packages. 70 | 71 | Once scan_plugins is executed, all 'cuisine' plugins in your python path, including 72 | the built-in ones will be discovered and will be collected by 73 | :class:`~lml.plugin.PluginInfoChain` in a dictionary for 74 | :meth:`~lml.PluginManager.get_a_plugin` to look up. 75 | 76 | 77 | Plugin management 78 | ----------------------- 79 | 80 | As you see in the class relationship diagram, There has not been any changes for 81 | `CuisineManager` which inherits from `:class:lml.PluginManager` and manages 82 | **cuisine** plugins. Please read the discussion in 83 | :ref:`previous chapter `. Let us look at the plugins. 84 | 85 | 86 | .. _builtin-plugin: 87 | 88 | Built-in plugin 89 | ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 90 | 91 | `Boost` plugin has been placed in a submodule, **robotchef.robot_cuisine**. Let 92 | us see how it was done. The *magic* lies in robot_cuisine module's __init__.py 93 | 94 | .. literalinclude:: ../../examples/robotchef/robotchef/robot_cuisine/__init__.py 95 | :language: python 96 | 97 | A unnamed instance of :class:`lml.plugin.PluginInfoChain` registers the meta 98 | data internally with `CuisineManager`. `__name__` variable 99 | refers to the module name, and in this case it equals 'robotchef.robot_cuisine'. 100 | It is used to form the absolute import path for `Boost` class. 101 | 102 | First parameter **cuisine** indicates that `electrify.Boost` is a **cuisine** plugin. 103 | **lml** will associate it with `CuisineManager`. It is why CuisineMananger 104 | has initialized as 'cuisine'. The second parameter is used 105 | the absolute import path 'robotchef.robot_cuisine.electricity.Boost'. The third 106 | parameter `tags` are the dictionary keys to look up class `Boost`. 107 | 108 | Here is a warning: to achieve lazy loading as promised by **lml**, you shall avoid 109 | heavy duty loading in __init__.py. 110 | this design principle: **not to import any un-necessary modules in your plugin 111 | module's __init__.py**. 112 | 113 | That's all you need to write a built-in plugin. 114 | 115 | .. _standaline-plugin: 116 | 117 | Standalone plugin 118 | ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 119 | 120 | Before we go to examine the source code of **robotchef_britishcuisine**, 121 | please let me dictate that the standalone plugins shall respect the package 122 | prefix, which is set by the main package. In this case, the plugin packages 123 | shall start with 'robotchef_'. Hence for British Cuisine, it is named as 124 | 'robotchef_britishcuisine'. 125 | 126 | Now let us have look at the module's __init__.py, you would find similar the 127 | plugin declaration code as in the following. But nothing else. 128 | 129 | .. literalinclude:: ../../examples/robotchef_britishcuisine/robotchef_britishcuisine/__init__.py 130 | :language: python 131 | :linenos: 132 | 133 | Because we have relocated `Fry` and `Bake` in this package, 134 | the instance of :class:`~lml.plugin.PluginInfoChain` issues two chained call 135 | :meth:`~lml.plugin.PluginInfoChain.add_a_plugin` but with corresponding 136 | parameters. 137 | 138 | 139 | .. note:: 140 | In your plugin package, you can add as many plugin class as you need. And 141 | the tags can be as long as you deem necessary. 142 | 143 | Let me wrap up this section. All you will need to do, in order to make a 144 | standalone plugin, is to provide a package installer(setup.py and other related 145 | package files) for a built-in plugin. 146 | 147 | 148 | The end 149 | -------------------------------------------------------------------------------- 150 | 151 | That is all you need to make your main component to start using component based 152 | approach to expand its functionalities. Here is the takeaway for you: 153 | 154 | #. :class:`lml.plugin.PluginManager` is just another factory pattern that hides 155 | the complexity away. 156 | #. You will need to call :meth:`lml.loader.scan_plugins` in your __init__.py or 157 | where appropriate before your factory class is called. 158 | 159 | 160 | More standalone plugins 161 | ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 162 | 163 | You are left to install robotchef_chinesecuisine and robotchef_cook yourself and 164 | explore their functionalities. 165 | 166 | How to ask robotchef to forget British cuisine? 167 | ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 168 | 169 | The management of standalone plugins are left in the hands of the user. To prevent 170 | robotchef from finding British cuisine, you can use pip to uninstall it, like this:: 171 | 172 | $ pip uninstall robotchef_britishcuisine 173 | -------------------------------------------------------------------------------- /docs/source/spelling_wordlist.txt: -------------------------------------------------------------------------------- 1 | plugins 2 | chinesecuisine 3 | setup 4 | py 5 | plugin 6 | lml 7 | robotchef 8 | uninstall 9 | api 10 | britishcuisine 11 | init 12 | allinone 13 | pypi 14 | pyinstaller 15 | pyexcel 16 | xls 17 | namespace 18 | GEdit 19 | yapsy 20 | sys 21 | pycharm 22 | sphinxcontrib 23 | zope 24 | tablib 25 | Alchin 26 | 27 | -------------------------------------------------------------------------------- /docs/source/tutorial.rst: -------------------------------------------------------------------------------- 1 | Tutorial 2 | ================================================================================ 3 | 4 | In this tutorial, we are going to go through various ways to build 5 | the command line application: **Robot Chef**. One is to build it as a single 6 | package. Another is to build it using lml: one main component 7 | with many plugins which are separately installable. By comparing the 8 | different approaches to build Robot Chef, we could see how lml can be used 9 | in practice. 10 | 11 | **Robot Chef** would report what it knows about the food in the world. For 12 | example:: 13 | 14 | $ robotchef "Portable Battery" 15 | I can cook Portable Battery for robots 16 | 17 | When you type "Fish and Chips", it could reports it does not know:: 18 | 19 | $ robotchef "Fish and Chips" 20 | I do not know how to cook Fish and Chips 21 | 22 | For it to understand all the cuisines in the world, there are two ways to 23 | enlarge its knowledge base: one is obviously to grow by itself. the other 24 | is to open the api interface so that others could join your effort. 25 | 26 | .. toctree:: 27 | 28 | allinone_tutorial 29 | allinone_lml_tutorial 30 | lml_tutorial 31 | api_tutorial 32 | 33 | Additional references 34 | ---------------------- 35 | 36 | #. pyexcel-chart: `use lml to refactor existing plugins `_ 37 | -------------------------------------------------------------------------------- /docs/source/uml/loading_sequence.uml: -------------------------------------------------------------------------------- 1 | @startuml 2 | 3 | actor bob 4 | participant robotchef 5 | participant lml 6 | participant robotchef_britishcuisine 7 | 8 | bob -> robotchef : > robotchef "Jacket Potato" 9 | robotchef -> lml : scan for plugins. 10 | lml -> robotchef_britishcuisine : read plugin chain in the module 11 | robotchef_britishcuisine -> lml: I can help with "Jacket Potato" and others. 12 | lml -> robotchef : read the built-in robot_cuisine 13 | robotchef -> lml : built-in chef knows "Portable Battery" 14 | lml --> robotchef : scanning done 15 | robotchef -> lml : get me a plugin that knows "Jacket Potato" 16 | lml -> robotchef : robotchef_britishcuisine.bake.Bake can do 17 | robotchef -> robotchef: make the food 18 | robotchef -> bob : "I can bake Jacket Potato" 19 | @enduml -------------------------------------------------------------------------------- /docs/source/uml/robot_chef.uml: -------------------------------------------------------------------------------- 1 | @startuml 2 | 3 | Interface Chef 4 | 5 | Chef <|-- Boost 6 | Chef <|-- Fry 7 | Chef <|-- Bake 8 | 9 | @enduml -------------------------------------------------------------------------------- /docs/source/uml/robotchef_allinone_lml.uml: -------------------------------------------------------------------------------- 1 | @startuml 2 | 3 | package lml { 4 | PluginManager o-- PluginInfo 5 | } 6 | 7 | package robotchef_allinone_lml { 8 | class CuisineManager { 9 | + get_a_plugin() 10 | + raise_exception() 11 | } 12 | interface Chef { 13 | + make() 14 | } 15 | PluginManager <|-- CuisineManager : cuisine 16 | Chef <|-- Boost 17 | Chef <|-- Fry 18 | Chef <|-- Bake 19 | PluginInfo .. Fry 20 | PluginInfo .. Bake 21 | PluginInfo .. Boost 22 | } 23 | 24 | 25 | @enduml -------------------------------------------------------------------------------- /docs/source/uml/robotchef_api_crd.uml: -------------------------------------------------------------------------------- 1 | @startuml 2 | 3 | 4 | package lml { 5 | PluginManager .. PluginInfoChain : registers plugin info 6 | PluginManager o-- PluginInfo 7 | PluginInfoChain -right- PluginInfo 8 | } 9 | 10 | package robotchef_api { 11 | class CuisineManager { 12 | + get_a_plugin() 13 | + raise_exception() 14 | } 15 | interface Chef { 16 | + make() 17 | } 18 | PluginManager <|-- CuisineManager : cuisine 19 | package robotchef.robot_cuisine { 20 | Chef <|-- Boost 21 | PluginInfoChain .. Boost 22 | } 23 | } 24 | 25 | package robotchef_britishcuisine { 26 | Chef <|-- Fry 27 | Chef <|-- Bake 28 | PluginInfoChain .. Fry 29 | PluginInfoChain .. Bake 30 | } 31 | 32 | package robotchef_v2 { 33 | } 34 | 35 | robotchef_v2 +-- robotchef_api 36 | 37 | @enduml -------------------------------------------------------------------------------- /docs/source/uml/robotchef_crd.uml: -------------------------------------------------------------------------------- 1 | @startuml 2 | 3 | package lml { 4 | PluginManager .. PluginInfoChain : registers plugin info 5 | PluginManager o-- PluginInfo 6 | PluginInfoChain -right- PluginInfo 7 | } 8 | 9 | package robotchef { 10 | class CuisineManager { 11 | + get_a_plugin() 12 | + raise_exception() 13 | } 14 | interface Chef { 15 | + make() 16 | } 17 | PluginManager <|-- CuisineManager : cuisine 18 | package robotchef.robot_cuisine { 19 | Chef <|-- Boost 20 | PluginInfoChain .. Boost 21 | } 22 | } 23 | 24 | package robotchef_britishcuisine { 25 | Chef <|-- Fry 26 | Chef <|-- Bake 27 | PluginInfoChain .. Fry 28 | PluginInfoChain .. Bake 29 | } 30 | 31 | 32 | @enduml -------------------------------------------------------------------------------- /examples/README.rst: -------------------------------------------------------------------------------- 1 | READ ME 2 | ========= 3 | 4 | A robot chef was created to master the cuisines around the world. It learns faster 5 | than a human because it just needs a plugin to be installed. It consumes less 6 | memory as it load the cuisine knowlege on demand. 7 | 8 | Please note that there are two implementations of the robot chef. One is pure 9 | command line interface(CLI) using lml directly; the other one is CLI using 10 | robotchef_api package which uses lml. The former demonstrates how lml could 11 | be used in a CLI package. The latter illustrates how to construct a pure 12 | python library using lml. 13 | -------------------------------------------------------------------------------- /lml.yml: -------------------------------------------------------------------------------- 1 | name: "lml" 2 | full_name: "Load me later. A lazy plugin management system." 3 | organisation: "python-lml" 4 | author: "C.W." 5 | contact: "wangc_2011@hotmail.com" 6 | company: "C.W." 7 | version: "0.2.0" 8 | current_version: "0.2.0" 9 | release: "0.2.0" 10 | copyright_year: 2017-2025 11 | license: New BSD 12 | dependencies: [] 13 | test_dependencies: 14 | - lml 15 | - pytest 16 | - pytest-cov 17 | description: "Load me later. A lazy plugin management system." 18 | excluded_github_users: 19 | - chfw 20 | - gitter-badger 21 | sphinx_html_theme: sphinx_rtd_theme -------------------------------------------------------------------------------- /lml/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | lml 3 | ~~~~~~~~~~~~~~~~~~~ 4 | 5 | Load me later. A lazy loading plugin management system. 6 | 7 | :copyright: (c) 2017-2018 by Onni Software Ltd. 8 | :license: New BSD License, see LICENSE for more details 9 | """ 10 | import logging 11 | 12 | from lml._version import __author__ # noqa: F401 13 | from lml._version import __version__ # noqa: F401 14 | 15 | try: 16 | from logging import NullHandler 17 | except ImportError: 18 | 19 | class NullHandler(logging.Handler): 20 | """ 21 | Null handler for logging 22 | """ 23 | 24 | def emit(self, record): 25 | pass 26 | 27 | logging.getLogger(__name__).addHandler(NullHandler()) 28 | -------------------------------------------------------------------------------- /lml/_version.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.2.0" 2 | __author__ = "C.W." 3 | -------------------------------------------------------------------------------- /lml/loader.py: -------------------------------------------------------------------------------- 1 | """ 2 | lml.loader 3 | ~~~~~~~~~~~~~~~~~~~ 4 | 5 | Plugin discovery module. It supports plugins installed via pip tools 6 | and pyinstaller. :func:`~lml.loader.scan_plugins` is expected to be 7 | called in the main package of yours at an earliest time of convenience. 8 | 9 | :copyright: (c) 2017-2025 by C.W. 10 | :license: New BSD License, see LICENSE for more details 11 | """ 12 | import re 13 | import logging 14 | import pkgutil 15 | import warnings 16 | from itertools import chain 17 | 18 | from lml.utils import do_import 19 | 20 | log = logging.getLogger(__name__) 21 | 22 | 23 | def scan_plugins( 24 | prefix, 25 | pyinstaller_path, 26 | black_list=None, 27 | white_list=None, 28 | plugin_name_patterns=None, 29 | ): 30 | """ 31 | Implicitly discover plugins via pkgutil and pyinstaller path 32 | 33 | Parameters 34 | ----------------- 35 | 36 | prefix:string 37 | module prefix. This prefix should become the prefix of the module name 38 | of all plugins. 39 | 40 | In the tutorial, robotchef-britishcuisine is a plugin package 41 | of robotchef and its module name is 'robotchef_britishcuisine'. When 42 | robotchef call scan_plugins to load its cuisine plugins, it specifies 43 | its prefix as "robotchef_". All modules that starts with 'robotchef_' 44 | will be auto-loaded: robotchef_britishcuisine, robotchef_chinesecuisine, 45 | etc. 46 | 47 | pyinstaller_path:string 48 | used in pyinstaller only. When your end developer would package 49 | your main library and its plugins using pyinstaller, this path 50 | helps pyinstaller to find the plugins. 51 | 52 | black_list:list 53 | a list of module names that should be skipped. 54 | 55 | white_list:list 56 | a list of modules that comes with your main module. If you have a 57 | built-in module, the module name should be inserted into the list. 58 | 59 | For example, robot_cuisine is a built-in module inside robotchef. It 60 | is listed in white_list. 61 | """ 62 | __plugin_name_patterns = "^%s.+$" % prefix 63 | warnings.warn( 64 | "Deprecated! since version 0.0.3. Please use scan_plugins_regex!" 65 | ) 66 | scan_plugins_regex( 67 | plugin_name_patterns=__plugin_name_patterns, 68 | pyinstaller_path=pyinstaller_path, 69 | black_list=black_list, 70 | white_list=white_list, 71 | ) 72 | 73 | 74 | def scan_plugins_regex( 75 | plugin_name_patterns=None, 76 | pyinstaller_path=None, 77 | black_list=None, 78 | white_list=None, 79 | ): 80 | 81 | """ 82 | Implicitly discover plugins via pkgutil and pyinstaller path using 83 | regular expression 84 | 85 | Parameters 86 | ----------------- 87 | 88 | plugin_name_patterns: python regular expression 89 | it is used to match all your plugins, either it is a prefix, 90 | a suffix, some text in the middle or all. 91 | 92 | pyinstaller_path:string 93 | used in pyinstaller only. When your end developer would package 94 | your main library and its plugins using pyinstaller, this path 95 | helps pyinstaller to find the plugins. 96 | 97 | black_list:list 98 | a list of module names that should be skipped. 99 | 100 | white_list:list 101 | a list of modules that comes with your main module. If you have a 102 | built-in module, the module name should be inserted into the list. 103 | 104 | For example, robot_cuisine is a built-in module inside robotchef. It 105 | is listed in white_list. 106 | """ 107 | log.debug("scanning for plugins...") 108 | if black_list is None: 109 | black_list = [] 110 | 111 | if white_list is None: 112 | white_list = [] 113 | 114 | # scan pkgutil.iter_modules 115 | module_names = ( 116 | module_info[1] 117 | for module_info in pkgutil.iter_modules() 118 | if module_info[2] and re.match(plugin_name_patterns, module_info[1]) 119 | ) 120 | 121 | # scan pyinstaller 122 | module_names_from_pyinstaller = scan_from_pyinstaller( 123 | plugin_name_patterns, pyinstaller_path 124 | ) 125 | 126 | all_modules = chain( 127 | module_names, module_names_from_pyinstaller, white_list 128 | ) 129 | # loop through modules and find our plug ins 130 | for module_name in all_modules: 131 | 132 | if module_name in black_list: 133 | log.debug("ignored " + module_name) 134 | continue 135 | 136 | try: 137 | do_import(module_name) 138 | except ImportError as e: 139 | log.debug(module_name) 140 | log.debug(e) 141 | continue 142 | log.debug("scanning done") 143 | 144 | 145 | # load modules to work based with and without pyinstaller 146 | # from: https://github.com/webcomics/dosage/blob/master/dosagelib/loader.py 147 | # see: https://github.com/pyinstaller/pyinstaller/issues/1905 148 | # load modules using iter_modules() 149 | # (should find all plug ins in normal build, but not pyinstaller) 150 | def scan_from_pyinstaller(plugin_name_patterns, path): 151 | """ 152 | Discover plugins from pyinstaller 153 | """ 154 | table_of_content = set() 155 | for a_toc in ( 156 | importer.toc 157 | for importer in map(pkgutil.get_importer, path) 158 | if hasattr(importer, "toc") 159 | ): 160 | table_of_content |= a_toc 161 | 162 | for module_name in table_of_content: 163 | if "." in module_name: 164 | continue 165 | if re.match(plugin_name_patterns, module_name): 166 | yield module_name 167 | -------------------------------------------------------------------------------- /lml/plugin.py: -------------------------------------------------------------------------------- 1 | """ 2 | lml.plugin 3 | ~~~~~~~~~~~~~~~~~~~ 4 | 5 | lml divides the plugins into two category: load-me-later plugins and 6 | load-me-now ones. load-me-later plugins refer to the plugins were 7 | loaded when needed due its bulky and/or memory hungry dependencies. 8 | Those plugins has to use lml and respect lml's design principle. 9 | 10 | load-me-now plugins refer to the plugins are immediately imported. All 11 | conventional Python classes are by default immediately imported. 12 | 13 | :class:`~lml.plugin.PluginManager` should be inherited to form new 14 | plugin manager class. If you have more than one plugins in your 15 | architecture, it is advisable to have one class per plugin type. 16 | 17 | :class:`~lml.plugin.PluginInfoChain` helps the plugin module to 18 | declare the available plugins in the module. 19 | 20 | :class:`~lml.plugin.PluginInfo` can be subclassed to describe 21 | your plugin. Its method :meth:`~lml.plugin.PluginInfo.tags` 22 | can be overridden to help its matching :class:`~lml.plugin.PluginManager` 23 | to look itself up. 24 | 25 | :copyright: (c) 2017-2025 by C.W. 26 | :license: New BSD License, see LICENSE for more details 27 | """ 28 | import logging 29 | from collections import defaultdict 30 | 31 | from lml.utils import json_dumps, do_import_class 32 | 33 | PLUG_IN_MANAGERS = {} 34 | CACHED_PLUGIN_INFO = defaultdict(list) 35 | 36 | log = logging.getLogger(__name__) 37 | 38 | 39 | class PluginInfo(object): 40 | """ 41 | Information about the plugin. 42 | 43 | It is used together with PluginInfoChain to describe the plugins. 44 | Meanwhile, it is a class decorator and can be used to register a plugin 45 | immediately for use, in other words, the PluginInfo decorated plugin 46 | class is not loaded later. 47 | 48 | Parameters 49 | ------------- 50 | name: 51 | plugin name 52 | 53 | absolute_import_path: 54 | absolute import path from your plugin name space for your plugin class 55 | 56 | tags: 57 | a list of keywords help the plugin manager to retrieve your plugin 58 | 59 | keywords: 60 | Another custom properties. 61 | 62 | Examples 63 | ------------- 64 | 65 | For load-me-later plugins: 66 | 67 | >>> info = PluginInfo("sample", 68 | ... abs_class_path='lml.plugin.PluginInfo', # demonstration only. 69 | ... tags=['load-me-later'], 70 | ... custom_property = 'I am a custom property') 71 | >>> print(info.module_name) 72 | lml 73 | >>> print(info.custom_property) 74 | I am a custom property 75 | 76 | For load-me-now plugins: 77 | 78 | >>> @PluginInfo("sample", tags=['load-me-now']) 79 | ... class TestPlugin: 80 | ... def echo(self, words): 81 | ... print("echoing %s" % words) 82 | 83 | Now let's retrive the second plugin back: 84 | 85 | >>> class SamplePluginManager(PluginManager): 86 | ... def __init__(self): 87 | ... PluginManager.__init__(self, "sample") 88 | >>> sample_manager = SamplePluginManager() 89 | >>> test_plugin=sample_manager.get_a_plugin("load-me-now") 90 | >>> test_plugin.echo("hey..") 91 | echoing hey.. 92 | 93 | """ 94 | 95 | def __init__( 96 | self, plugin_type, abs_class_path=None, tags=None, **keywords 97 | ): 98 | self.plugin_type = plugin_type 99 | self.absolute_import_path = abs_class_path 100 | self.cls = None 101 | self.properties = keywords 102 | self.__tags = tags 103 | 104 | def __getattr__(self, name): 105 | if name == "module_name": 106 | if self.absolute_import_path: 107 | module_name = self.absolute_import_path.split(".")[0] 108 | else: 109 | module_name = self.cls.__module__ 110 | return module_name 111 | return self.properties.get(name) 112 | 113 | def tags(self): 114 | """ 115 | A list of tags for identifying the plugin class 116 | 117 | The plugin class is described at the absolute_import_path 118 | """ 119 | if self.__tags is None: 120 | yield self.plugin_type 121 | else: 122 | for tag in self.__tags: 123 | yield tag 124 | 125 | def __repr__(self): 126 | rep = { 127 | "plugin_type": self.plugin_type, 128 | "path": self.absolute_import_path, 129 | } 130 | rep.update(self.properties) 131 | return json_dumps(rep) 132 | 133 | def __call__(self, cls): 134 | self.cls = cls 135 | _register_a_plugin(self, cls) 136 | return cls 137 | 138 | 139 | class PluginInfoChain(object): 140 | """ 141 | Pandas style, chained list declaration 142 | 143 | It is used in the plugin packages to list all plugin classes 144 | """ 145 | 146 | def __init__(self, path): 147 | self._logger = logging.getLogger( 148 | self.__class__.__module__ + "." + self.__class__.__name__ 149 | ) 150 | self.module_name = path 151 | 152 | def add_a_plugin(self, plugin_type, submodule=None, **keywords): 153 | """ 154 | Add a plain plugin 155 | 156 | Parameters 157 | ------------- 158 | 159 | plugin_type: 160 | plugin manager name 161 | 162 | submodule: 163 | the relative import path to your plugin class 164 | """ 165 | a_plugin_info = PluginInfo( 166 | plugin_type, self._get_abs_path(submodule), **keywords 167 | ) 168 | 169 | self.add_a_plugin_instance(a_plugin_info) 170 | return self 171 | 172 | def add_a_plugin_instance(self, plugin_info_instance): 173 | """ 174 | Add a plain plugin 175 | 176 | Parameters 177 | ------------- 178 | 179 | plugin_info_instance: 180 | an instance of PluginInfo 181 | 182 | The developer has to specify the absolute import path 183 | """ 184 | self._logger.debug( 185 | "add %s as '%s' plugin", 186 | plugin_info_instance.absolute_import_path, 187 | plugin_info_instance.plugin_type, 188 | ) 189 | _load_me_later(plugin_info_instance) 190 | return self 191 | 192 | def _get_abs_path(self, submodule): 193 | return "%s.%s" % (self.module_name, submodule) 194 | 195 | 196 | class PluginManager(object): 197 | """ 198 | Load plugin info into in-memory dictionary for later import 199 | 200 | Parameters 201 | -------------- 202 | 203 | plugin_type: 204 | the plugin type. All plugins of this plugin type will be 205 | registered to it. 206 | """ 207 | 208 | def __init__(self, plugin_type): 209 | self.plugin_name = plugin_type 210 | self.registry = defaultdict(list) 211 | self.tag_groups = dict() 212 | self._logger = logging.getLogger( 213 | self.__class__.__module__ + "." + self.__class__.__name__ 214 | ) 215 | _register_class(self) 216 | 217 | def get_a_plugin(self, key, **keywords): 218 | """ Get a plugin 219 | 220 | Parameters 221 | --------------- 222 | 223 | key: 224 | the key to find the plugins 225 | 226 | keywords: 227 | additional parameters for help the retrieval of the plugins 228 | """ 229 | self._logger.debug("get a plugin called") 230 | plugin = self.load_me_now(key) 231 | return plugin() 232 | 233 | def raise_exception(self, key): 234 | """Raise plugin not found exception 235 | 236 | Override this method to raise custom exception 237 | 238 | Parameters 239 | ----------------- 240 | 241 | key: 242 | the key to find the plugin 243 | """ 244 | self._logger.debug(self.registry.keys()) 245 | raise Exception("No %s is found for %s" % (self.plugin_name, key)) 246 | 247 | def load_me_later(self, plugin_info): 248 | """ 249 | Register a plugin info for later loading 250 | 251 | Parameters 252 | -------------- 253 | 254 | plugin_info: 255 | a instance of plugin info 256 | """ 257 | self._logger.debug("load %s later", plugin_info.absolute_import_path) 258 | self._update_registry_and_expand_tag_groups(plugin_info) 259 | 260 | def load_me_now(self, key, library=None, **keywords): 261 | """ 262 | Import a plugin from plugin registry 263 | 264 | Parameters 265 | ----------------- 266 | 267 | key: 268 | the key to find the plugin 269 | 270 | library: 271 | to use a specific plugin module 272 | """ 273 | if keywords: 274 | self._logger.debug(keywords) 275 | __key = key.lower() 276 | 277 | if __key in self.registry: 278 | for plugin_info in self.registry[__key]: 279 | cls = self.dynamic_load_library(plugin_info) 280 | module_name = _get_me_pypi_package_name(cls) 281 | if library and module_name != library: 282 | continue 283 | else: 284 | break 285 | else: 286 | # only library condition could raise an exception 287 | self._logger.debug("%s is not installed" % library) 288 | self.raise_exception(key) 289 | self._logger.debug("load %s now for '%s'", cls, key) 290 | return cls 291 | else: 292 | self.raise_exception(key) 293 | 294 | def dynamic_load_library(self, a_plugin_info): 295 | """Dynamically load the plugin info if not loaded 296 | 297 | 298 | Parameters 299 | -------------- 300 | 301 | a_plugin_info: 302 | a instance of plugin info 303 | """ 304 | if a_plugin_info.cls is None: 305 | self._logger.debug("import " + a_plugin_info.absolute_import_path) 306 | cls = do_import_class(a_plugin_info.absolute_import_path) 307 | a_plugin_info.cls = cls 308 | return a_plugin_info.cls 309 | 310 | def register_a_plugin(self, plugin_cls, plugin_info): 311 | """ for dynamically loaded plugin during runtime 312 | 313 | Parameters 314 | -------------- 315 | 316 | plugin_cls: 317 | the actual plugin class refered to by the second parameter 318 | 319 | plugin_info: 320 | a instance of plugin info 321 | """ 322 | self._logger.debug("register %s", _show_me_your_name(plugin_cls)) 323 | plugin_info.cls = plugin_cls 324 | self._update_registry_and_expand_tag_groups(plugin_info) 325 | 326 | def get_primary_key(self, key): 327 | __key = key.lower() 328 | return self.tag_groups.get(__key, None) 329 | 330 | def _update_registry_and_expand_tag_groups(self, plugin_info): 331 | primary_tag = None 332 | for index, key in enumerate(plugin_info.tags()): 333 | self.registry[key.lower()].append(plugin_info) 334 | if index == 0: 335 | primary_tag = key.lower() 336 | self.tag_groups[key.lower()] = primary_tag 337 | 338 | 339 | def _register_class(cls): 340 | """Reigister a newly created plugin manager""" 341 | log.debug("declare '%s' plugin manager", cls.plugin_name) 342 | PLUG_IN_MANAGERS[cls.plugin_name] = cls 343 | if cls.plugin_name in CACHED_PLUGIN_INFO: 344 | # check if there is early registrations or not 345 | for plugin_info in CACHED_PLUGIN_INFO[cls.plugin_name]: 346 | if plugin_info.absolute_import_path: 347 | log.debug( 348 | "load cached plugin info: %s", 349 | plugin_info.absolute_import_path, 350 | ) 351 | else: 352 | log.debug( 353 | "load cached plugin info: %s", 354 | _show_me_your_name(plugin_info.cls), 355 | ) 356 | cls.load_me_later(plugin_info) 357 | 358 | del CACHED_PLUGIN_INFO[cls.plugin_name] 359 | 360 | 361 | def _register_a_plugin(plugin_info, plugin_cls): 362 | """module level function to register a plugin""" 363 | manager = PLUG_IN_MANAGERS.get(plugin_info.plugin_type) 364 | if manager: 365 | manager.register_a_plugin(plugin_cls, plugin_info) 366 | else: 367 | # let's cache it and wait the manager to be registered 368 | try: 369 | log.debug("caching %s", _show_me_your_name(plugin_cls.__name__)) 370 | except AttributeError: 371 | log.debug("caching %s", _show_me_your_name(plugin_cls)) 372 | CACHED_PLUGIN_INFO[plugin_info.plugin_type].append(plugin_info) 373 | 374 | 375 | def _load_me_later(plugin_info): 376 | """ module level function to load a plugin later""" 377 | manager = PLUG_IN_MANAGERS.get(plugin_info.plugin_type) 378 | if manager: 379 | manager.load_me_later(plugin_info) 380 | else: 381 | # let's cache it and wait the manager to be registered 382 | log.debug( 383 | "caching %s for %s", 384 | plugin_info.absolute_import_path, 385 | plugin_info.plugin_type, 386 | ) 387 | CACHED_PLUGIN_INFO[plugin_info.plugin_type].append(plugin_info) 388 | 389 | 390 | def _get_me_pypi_package_name(module): 391 | try: 392 | module_name = module.__module__ 393 | root_module_name = module_name.split(".")[0] 394 | return root_module_name.replace("_", "-") 395 | except AttributeError: 396 | return None 397 | 398 | 399 | def _show_me_your_name(cls_func_or_data_type): 400 | try: 401 | return cls_func_or_data_type.__name__ 402 | except AttributeError: 403 | return str(type(cls_func_or_data_type)) 404 | -------------------------------------------------------------------------------- /lml/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | lml.utils 3 | ~~~~~~~~~~~~~~~~~~~ 4 | 5 | json utils for dump plugin info class 6 | 7 | :copyright: (c) 2017-2025 by C.W. 8 | :license: New BSD License, see LICENSE for more details 9 | """ 10 | import logging 11 | import importlib 12 | from json import JSONEncoder, dumps 13 | 14 | log = logging.getLogger(__name__) 15 | 16 | 17 | class PythonObjectEncoder(JSONEncoder): 18 | """ 19 | Custom object encoder for json dump 20 | """ 21 | 22 | def default(self, obj): 23 | a_list_of_types = (list, dict, str, int, float, bool, type(None)) 24 | if isinstance(obj, a_list_of_types): 25 | return JSONEncoder.default(self, obj) 26 | return {"_python_object": str(obj)} 27 | 28 | 29 | def json_dumps(keywords): 30 | """ 31 | Dump function keywords in json 32 | """ 33 | return dumps(keywords, cls=PythonObjectEncoder) 34 | 35 | 36 | def do_import(plugin_module_name): 37 | """dynamically import a module""" 38 | try: 39 | return _do_import(plugin_module_name) 40 | except ImportError: 41 | log.exception("%s is absent or cannot be imported", plugin_module_name) 42 | raise 43 | 44 | 45 | def _do_import(plugin_module_name): 46 | plugin_module = importlib.import_module(plugin_module_name) 47 | log.debug("found " + plugin_module_name) 48 | return plugin_module 49 | 50 | 51 | def do_import_class(plugin_class): 52 | """dynamically import a class""" 53 | try: 54 | plugin_module_name, class_name = plugin_class.rsplit(".", 1) 55 | plugin_module = importlib.import_module(plugin_module_name) 56 | cls = getattr(plugin_module, class_name) 57 | return cls 58 | except ImportError: 59 | log.exception("Failed to import %s", plugin_module_name) 60 | raise 61 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/python-lml/lml/16198a29d818f5042d42431dcec7fcd73cd2ec56/requirements.txt -------------------------------------------------------------------------------- /rnd_requirements.txt: -------------------------------------------------------------------------------- 1 | https://github.com/pyexcel/pyexcel-xls/archive/v0.4.x.zip 2 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.rst 3 | [bdist_wheel] 4 | universal = 1 5 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | Template by pypi-mobans 5 | """ 6 | 7 | import os 8 | import sys 9 | import codecs 10 | import locale 11 | import platform 12 | from shutil import rmtree 13 | 14 | from setuptools import Command, setup, find_packages 15 | 16 | PY2 = sys.version_info[0] == 2 17 | PY26 = PY2 and sys.version_info[1] < 7 18 | PY33 = sys.version_info < (3, 4) 19 | 20 | # Work around mbcs bug in distutils. 21 | # http://bugs.python.org/issue10945 22 | # This work around is only if a project supports Python < 3.4 23 | 24 | # Work around for locale not being set 25 | try: 26 | lc = locale.getlocale() 27 | pf = platform.system() 28 | if pf != "Windows" and lc == (None, None): 29 | locale.setlocale(locale.LC_ALL, "C.UTF-8") 30 | except (ValueError, UnicodeError, locale.Error): 31 | locale.setlocale(locale.LC_ALL, "en_US.UTF-8") 32 | 33 | NAME = "lml" 34 | AUTHOR = "C.W." 35 | VERSION = "0.2.0" 36 | EMAIL = "wangc_2011@hotmail.com" 37 | LICENSE = "New BSD" 38 | DESCRIPTION = ( 39 | "Load me later. A lazy plugin management system." 40 | ) 41 | URL = "https://github.com/python-lml/lml" 42 | DOWNLOAD_URL = "%s/archive/0.2.0.tar.gz" % URL 43 | FILES = ["README.rst", "CHANGELOG.rst"] 44 | KEYWORDS = [ 45 | "python", 46 | ] 47 | 48 | CLASSIFIERS = [ 49 | "Topic :: Software Development :: Libraries", 50 | "Programming Language :: Python", 51 | "Intended Audience :: Developers", 52 | "Programming Language :: Python :: 2.6", 53 | "Programming Language :: Python :: 2.7", 54 | "Programming Language :: Python :: 3.3", 55 | "Programming Language :: Python :: 3.4", 56 | "Programming Language :: Python :: 3.5", 57 | "Programming Language :: Python :: 3.6", 58 | "Programming Language :: Python :: 3.7", 59 | "Programming Language :: Python :: 3.8", 60 | 61 | ] 62 | 63 | 64 | INSTALL_REQUIRES = [ 65 | ] 66 | SETUP_COMMANDS = {} 67 | 68 | PACKAGES = find_packages(exclude=["ez_setup", "examples", "tests", "tests.*"]) 69 | EXTRAS_REQUIRE = {} 70 | # You do not need to read beyond this line 71 | PUBLISH_COMMAND = "{0} setup.py sdist bdist_wheel upload -r pypi".format(sys.executable) 72 | HERE = os.path.abspath(os.path.dirname(__file__)) 73 | 74 | GS_COMMAND = ("gease lml v0.2.0 " + 75 | "Find 0.2.0 in changelog for more details") 76 | NO_GS_MESSAGE = ("Automatic github release is disabled. " + 77 | "Please install gease to enable it.") 78 | UPLOAD_FAILED_MSG = ( 79 | 'Upload failed. please run "%s" yourself.' % PUBLISH_COMMAND) 80 | 81 | 82 | class PublishCommand(Command): 83 | """Support setup.py upload.""" 84 | 85 | description = "Build and publish the package on github and pypi" 86 | user_options = [] 87 | 88 | @staticmethod 89 | def status(s): 90 | """Prints things in bold.""" 91 | print("\033[1m{0}\033[0m".format(s)) 92 | 93 | def initialize_options(self): 94 | pass 95 | 96 | def finalize_options(self): 97 | pass 98 | 99 | def run(self): 100 | try: 101 | self.status("Removing previous builds...") 102 | rmtree(os.path.join(HERE, "dist")) 103 | rmtree(os.path.join(HERE, "build")) 104 | rmtree(os.path.join(HERE, "lml.egg-info")) 105 | except OSError: 106 | pass 107 | 108 | self.status("Building Source and Wheel (universal) distribution...") 109 | run_status = True 110 | if has_gease(): 111 | run_status = os.system(GS_COMMAND) == 0 112 | else: 113 | self.status(NO_GS_MESSAGE) 114 | if run_status: 115 | if os.system(PUBLISH_COMMAND) != 0: 116 | self.status(UPLOAD_FAILED_MSG) 117 | 118 | sys.exit() 119 | 120 | 121 | SETUP_COMMANDS.update({ 122 | "publish": PublishCommand 123 | }) 124 | 125 | def has_gease(): 126 | """ 127 | test if github release command is installed 128 | 129 | visit http://github.com/moremoban/gease for more info 130 | """ 131 | try: 132 | import gease # noqa 133 | return True 134 | except ImportError: 135 | return False 136 | 137 | 138 | def read_files(*files): 139 | """Read files into setup""" 140 | text = "" 141 | for single_file in files: 142 | content = read(single_file) 143 | text = text + content + "\n" 144 | return text 145 | 146 | 147 | def read(afile): 148 | """Read a file into setup""" 149 | the_relative_file = os.path.join(HERE, afile) 150 | with codecs.open(the_relative_file, "r", "utf-8") as opened_file: 151 | content = filter_out_test_code(opened_file) 152 | content = "".join(list(content)) 153 | return content 154 | 155 | 156 | def filter_out_test_code(file_handle): 157 | found_test_code = False 158 | for line in file_handle.readlines(): 159 | if line.startswith(".. testcode:"): 160 | found_test_code = True 161 | continue 162 | if found_test_code is True: 163 | if line.startswith(" "): 164 | continue 165 | else: 166 | empty_line = line.strip() 167 | if len(empty_line) == 0: 168 | continue 169 | else: 170 | found_test_code = False 171 | yield line 172 | else: 173 | for keyword in ["|version|", "|today|"]: 174 | if keyword in line: 175 | break 176 | else: 177 | yield line 178 | 179 | 180 | if __name__ == "__main__": 181 | setup( 182 | test_suite="tests", 183 | name=NAME, 184 | author=AUTHOR, 185 | version=VERSION, 186 | author_email=EMAIL, 187 | description=DESCRIPTION, 188 | url=URL, 189 | download_url=DOWNLOAD_URL, 190 | long_description=read_files(*FILES), 191 | license=LICENSE, 192 | keywords=KEYWORDS, 193 | extras_require=EXTRAS_REQUIRE, 194 | tests_require=["nose"], 195 | install_requires=INSTALL_REQUIRES, 196 | packages=PACKAGES, 197 | include_package_data=True, 198 | zip_safe=False, 199 | classifiers=CLASSIFIERS, 200 | cmdclass=SETUP_COMMANDS 201 | ) 202 | -------------------------------------------------------------------------------- /test.sh: -------------------------------------------------------------------------------- 1 | pip freeze 2 | cd tests/test_plugin 3 | python setup.py install 4 | cd - 5 | pytest tests --verbosity=3 --cov=lml --doctest-glob=*.rst && flake8 . --exclude=.moban.d,docs,setup.py --builtins=unicode,xrange,long 6 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/python-lml/lml/16198a29d818f5042d42431dcec7fcd73cd2ec56/tests/__init__.py -------------------------------------------------------------------------------- /tests/requirements.txt: -------------------------------------------------------------------------------- 1 | pytest 2 | pytest-cov 3 | codecov 4 | coverage 5 | flake8 6 | black;python_version>="3.6" 7 | isort;python_version>="3.6" 8 | -------------------------------------------------------------------------------- /tests/sample_plugin/sample_plugin/__init__.py: -------------------------------------------------------------------------------- 1 | from lml.registry import PluginInfoChain 2 | 3 | __test_plugins__ = PluginInfoChain(__name__).add_a_plugin("test_io2", "reader") 4 | -------------------------------------------------------------------------------- /tests/sample_plugin/sample_plugin/manager.py: -------------------------------------------------------------------------------- 1 | from lml.plugin import PluginManager 2 | 3 | 4 | class TestPluginManager(PluginManager): 5 | def __init__(self): 6 | PluginManager.__init__(self, "test_io2") 7 | 8 | def load_me_later(self, plugin_info): 9 | PluginManager.load_me_later(self, plugin_info) 10 | 11 | def register_a_plugin(self, cls): 12 | PluginManager.register_a_plugin(self, cls) 13 | 14 | 15 | testmanager = TestPluginManager() 16 | -------------------------------------------------------------------------------- /tests/sample_plugin/sample_plugin/reader.py: -------------------------------------------------------------------------------- 1 | from lml.plugin import Plugin 2 | 3 | 4 | class TestPlugin(Plugin): 5 | def do_something(self): 6 | print("hello world") 7 | -------------------------------------------------------------------------------- /tests/sample_plugin/setup.py: -------------------------------------------------------------------------------- 1 | """ 2 | pyexcel-test_plugin 3 | ~~~~~~~~~~~~~~ 4 | 5 | It is a tes plugin 6 | """ 7 | 8 | try: 9 | from setuptools import setup, find_packages 10 | except ImportError: 11 | from ez_setup import use_setuptools 12 | 13 | use_setuptools() 14 | from setuptools import setup, find_packages 15 | 16 | setup( 17 | name="pyexcel-test_plugin2", 18 | author="C. W.", 19 | version="0.0.1", 20 | author_email="wangc_2011@hotmail.com", 21 | packages=find_packages(exclude=["ez_setup", "examples", "tests"]), 22 | include_package_data=True, 23 | long_description=__doc__, 24 | zip_safe=False, 25 | classifiers=[ 26 | "Development Status :: 3 - Alpha", 27 | "Topic :: Office/Business", 28 | "Topic :: Utilities", 29 | "Topic :: Software Development :: Libraries", 30 | "Programming Language :: Python", 31 | "License :: OSI Approved :: GNU General Public License v3", 32 | "Intended Audience :: Developers", 33 | "Programming Language :: Python :: 2.6", 34 | "Programming Language :: Python :: 2.7", 35 | ], 36 | ) 37 | -------------------------------------------------------------------------------- /tests/test_plugin/pyexcel_test/__init__.py: -------------------------------------------------------------------------------- 1 | from lml.plugin import PluginInfoChain 2 | 3 | __test_plugins__ = PluginInfoChain(__name__).add_a_plugin("test_io", "x") 4 | -------------------------------------------------------------------------------- /tests/test_plugin/setup.py: -------------------------------------------------------------------------------- 1 | """ 2 | pyexcel-test_plugin 3 | ~~~~~~~~~~~~~~ 4 | 5 | It is a tes plugin 6 | """ 7 | 8 | try: 9 | from setuptools import setup, find_packages 10 | except ImportError: 11 | from ez_setup import use_setuptools 12 | 13 | use_setuptools() 14 | from setuptools import setup, find_packages 15 | 16 | setup( 17 | name="pyexcel-test_plugin", 18 | author="C. W.", 19 | version="0.0.1", 20 | author_email="wangc_2011@hotmail.com", 21 | packages=find_packages(exclude=["ez_setup", "examples", "tests"]), 22 | include_package_data=True, 23 | long_description=__doc__, 24 | zip_safe=False, 25 | classifiers=[ 26 | "Development Status :: 3 - Alpha", 27 | "Topic :: Office/Business", 28 | "Topic :: Utilities", 29 | "Topic :: Software Development :: Libraries", 30 | "Programming Language :: Python", 31 | "License :: OSI Approved :: GNU General Public License v3", 32 | "Intended Audience :: Developers", 33 | "Programming Language :: Python :: 2.6", 34 | "Programming Language :: Python :: 2.7", 35 | ], 36 | ) 37 | -------------------------------------------------------------------------------- /tests/test_plugin_info.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from lml.plugin import PluginInfo 4 | 5 | 6 | def test_plugin_info(): 7 | info = PluginInfo( 8 | "renderer", abs_class_path="good.plugin.path", custom="property" 9 | ) 10 | assert info.custom == "property" 11 | keys = list(info.tags()) 12 | assert len(keys) == 1 13 | assert keys[0] == "renderer" 14 | assert info.module_name == "good" 15 | expected = { 16 | "path": "good.plugin.path", 17 | "plugin_type": "renderer", 18 | "custom": "property", 19 | } 20 | assert json.loads(info.__repr__()) == expected 21 | 22 | 23 | def test_module_name_scenario_2(): 24 | class TestClass2: 25 | pass 26 | 27 | info = PluginInfo("renderer", custom="property") 28 | info.cls = TestClass2 29 | assert info.module_name == "tests.test_plugin_info" 30 | -------------------------------------------------------------------------------- /tests/test_plugin_loader.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import patch 2 | 3 | 4 | @patch("pkgutil.get_importer") 5 | def test_load_from_pyinstaller(pkgutil_get_importer): 6 | sample_toc = set(["pyexcel_io", "pyexcel_xls", "blah", "test.dot.module"]) 7 | pkgutil_get_importer.return_value.toc = sample_toc 8 | from lml.loader import scan_from_pyinstaller 9 | 10 | module_names = scan_from_pyinstaller("pyexcel_", "path") 11 | expected = ["pyexcel_io", "pyexcel_xls"] 12 | assert sorted(list(module_names)) == sorted(expected) 13 | 14 | 15 | @patch("pkgutil.get_importer") 16 | def test_load_from_pyinstaller_with_regex(pkgutil_get_importer): 17 | sample_toc = set(["pyexcel_io", "pyexcel_xls", "blah"]) 18 | pkgutil_get_importer.return_value.toc = sample_toc 19 | from lml.loader import scan_from_pyinstaller 20 | 21 | module_names = scan_from_pyinstaller("^.+cel_.+$", "path") 22 | expected = ["pyexcel_io", "pyexcel_xls"] 23 | assert sorted(list(module_names)) == sorted(expected) 24 | 25 | 26 | @patch("pkgutil.get_importer") 27 | @patch("pkgutil.iter_modules") 28 | def test_load_plugins(pkgutil_iter_modules, pkgutil_get_importer): 29 | test_module_name = "pyexcel_test" 30 | sample_toc = set(["pyexcel_io"]) 31 | pkgutil_get_importer.return_value.toc = sample_toc 32 | pkgutil_iter_modules.return_value = [("not used", test_module_name, True)] 33 | from lml.loader import scan_plugins 34 | 35 | scan_plugins("pyexcel_", ".", ["pyexcel_io"]) 36 | from lml.plugin import CACHED_PLUGIN_INFO 37 | 38 | info = CACHED_PLUGIN_INFO["test_io"][0] 39 | assert info.plugin_type == "test_io" 40 | assert info.absolute_import_path == "pyexcel_test.x" 41 | 42 | 43 | @patch("pkgutil.get_importer") 44 | @patch("pkgutil.iter_modules") 45 | def test_load_plugins_without_pyinstaller( 46 | pkgutil_iter_modules, pkgutil_get_importer 47 | ): 48 | test_module_name = "pyexcel_test" 49 | sample_toc = set() 50 | pkgutil_get_importer.return_value.toc = sample_toc 51 | # mock iter modules 52 | pkgutil_iter_modules.return_value = [("not used", test_module_name, True)] 53 | from lml.loader import scan_plugins 54 | 55 | scan_plugins("pyexcel_", ".", ["pyexcel_io"]) 56 | from lml.plugin import CACHED_PLUGIN_INFO 57 | 58 | info = CACHED_PLUGIN_INFO["test_io"][0] 59 | assert info.plugin_type == "test_io" 60 | assert info.absolute_import_path == "pyexcel_test.x" 61 | 62 | 63 | @patch("pkgutil.get_importer") 64 | @patch("pkgutil.iter_modules") 65 | @patch("lml.plugin._load_me_later") 66 | def test_load_plugins_without_any_plugins( 67 | mocked_load_me_later, pkgutil_iter_modules, pkgutil_get_importer 68 | ): 69 | sample_toc = set() 70 | pkgutil_get_importer.return_value.toc = sample_toc 71 | pkgutil_iter_modules.return_value = [] 72 | from lml.loader import scan_plugins 73 | 74 | scan_plugins("pyexcel_", ".", ["pyexcel_io"]) 75 | assert not mocked_load_me_later.called 76 | 77 | 78 | @patch("pkgutil.get_importer") 79 | @patch("pkgutil.iter_modules") 80 | @patch("lml.plugin._load_me_later") 81 | def test_load_plugins_without_black_list( 82 | mocked_load_me_later, pkgutil_iter_modules, pkgutil_get_importer 83 | ): 84 | sample_toc = set() 85 | pkgutil_get_importer.return_value.toc = sample_toc 86 | pkgutil_iter_modules.return_value = [] 87 | from lml.loader import scan_plugins 88 | 89 | scan_plugins("pyexcel_", ".") 90 | assert not mocked_load_me_later.called 91 | 92 | 93 | @patch("pkgutil.get_importer") 94 | @patch("pkgutil.iter_modules") 95 | @patch("lml.plugin._load_me_later") 96 | def test_load_plugins_import_error( 97 | mocked_load_me_later, pkgutil_iter_modules, pkgutil_get_importer 98 | ): 99 | sample_toc = set(["test_non_existent_module"]) 100 | pkgutil_get_importer.return_value.toc = sample_toc 101 | pkgutil_iter_modules.return_value = [("not used", "pyexcel_xls", False)] 102 | from lml.loader import scan_plugins 103 | 104 | scan_plugins("test_", ".", ["pyexcel_io"]) 105 | assert not mocked_load_me_later.called 106 | -------------------------------------------------------------------------------- /tests/test_plugin_manager.py: -------------------------------------------------------------------------------- 1 | from lml.plugin import ( 2 | PLUG_IN_MANAGERS, 3 | CACHED_PLUGIN_INFO, 4 | PluginInfo, 5 | PluginManager, 6 | _show_me_your_name, 7 | ) 8 | 9 | from unittest.mock import patch 10 | from pytest import raises 11 | 12 | 13 | def test_plugin_manager(): 14 | test_plugin = "my plugin" 15 | manager = PluginManager(test_plugin) 16 | assert PLUG_IN_MANAGERS[test_plugin] == manager 17 | 18 | 19 | def test_load_me_later(): 20 | test_plugin = "my plugin" 21 | manager = PluginManager(test_plugin) 22 | plugin_info = make_me_a_plugin_info(test_plugin) 23 | manager.load_me_later(plugin_info) 24 | assert list(manager.registry.keys()) == [test_plugin] 25 | 26 | 27 | @patch("lml.plugin.do_import_class") 28 | def test_load_me_now(mock_import): 29 | custom_class = PluginInfo 30 | mock_import.return_value = custom_class 31 | test_plugin = "my plugin" 32 | manager = PluginManager(test_plugin) 33 | plugin_info = make_me_a_plugin_info(test_plugin) 34 | manager.load_me_later(plugin_info) 35 | actual = manager.load_me_now(test_plugin) 36 | assert actual == custom_class 37 | assert manager.tag_groups == {"my plugin": "my plugin"} 38 | assert plugin_info == manager.registry["my plugin"][0] 39 | 40 | 41 | @patch("lml.plugin.do_import_class") 42 | def test_load_me_now_with_known_missing_library(mock_import): 43 | custom_class = PluginInfo 44 | mock_import.return_value = custom_class 45 | test_plugin = "my plugin" 46 | manager = PluginManager(test_plugin) 47 | plugin_info = make_me_a_plugin_info(test_plugin) 48 | manager.load_me_later(plugin_info) 49 | with raises(Exception): 50 | manager.load_me_now(test_plugin, library='alien') 51 | 52 | 53 | @patch("lml.plugin.do_import_class") 54 | def test_load_me_now_exception(mock_import): 55 | custom_class = PluginInfo 56 | mock_import.return_value = custom_class 57 | test_plugin = "my plugin" 58 | with raises(Exception): 59 | manager = PluginManager(test_plugin) 60 | plugin_info = make_me_a_plugin_info("my") 61 | manager.load_me_later(plugin_info) 62 | manager.load_me_now("my", "my special library") 63 | 64 | 65 | def test_load_me_now_no_key_found(): 66 | test_plugin = "my plugin" 67 | with raises(Exception): 68 | manager = PluginManager(test_plugin) 69 | manager.load_me_now("my", custom_property="here") 70 | 71 | 72 | @patch("lml.plugin.do_import_class") 73 | def test_dynamic_load_library(mock_import): 74 | test_plugin = "test plugin" 75 | custom_obj = object() 76 | mock_import.return_value = custom_obj 77 | manager = PluginManager(test_plugin) 78 | plugin_info = make_me_a_plugin_info(test_plugin) 79 | manager.dynamic_load_library(plugin_info) 80 | assert custom_obj == plugin_info.cls 81 | 82 | 83 | @patch("lml.plugin.do_import_class") 84 | def test_dynamic_load_library_no_action(mock_import): 85 | test_plugin = "test plugin" 86 | manager = PluginManager(test_plugin) 87 | plugin_info = make_me_a_plugin_info(test_plugin) 88 | plugin_info.cls = object() 89 | manager.dynamic_load_library(plugin_info) 90 | assert mock_import.called is False 91 | 92 | 93 | class TestClass: 94 | pass 95 | 96 | 97 | def test_register_a_plugin(): 98 | test_plugin = "test plugin" 99 | 100 | manager = PluginManager(test_plugin) 101 | plugin_info = make_me_a_plugin_info("my") 102 | manager.register_a_plugin(TestClass, plugin_info) 103 | assert plugin_info.cls == TestClass 104 | assert manager.registry["my"][0] == plugin_info 105 | assert manager.tag_groups == {"my": "my"} 106 | 107 | 108 | def test_get_a_plugin(): 109 | test_plugin = "test plugin" 110 | 111 | manager = PluginManager(test_plugin) 112 | plugin_info = make_me_a_plugin_info("my") 113 | plugin_info.cls = TestClass 114 | manager.register_a_plugin(TestClass, plugin_info) 115 | the_plugin = manager.get_a_plugin("my") 116 | assert isinstance(the_plugin, TestClass) 117 | 118 | 119 | def test_register_class(): 120 | test_plugin = "test_plugin" 121 | plugin_info = make_me_a_plugin_info("my") 122 | CACHED_PLUGIN_INFO[test_plugin].append(plugin_info) 123 | manager = PluginManager(test_plugin) 124 | assert list(manager.registry.keys()) == ["my"] 125 | 126 | 127 | def test_load_me_later_function(): 128 | from lml.plugin import _load_me_later 129 | 130 | test_plugin = "my plugin" 131 | manager = PluginManager(test_plugin) 132 | plugin_info = make_me_a_plugin_info(test_plugin) 133 | _load_me_later(plugin_info) 134 | assert list(manager.registry.keys()) == [test_plugin] 135 | 136 | 137 | def test_do_import_cls_error(): 138 | from lml.plugin import do_import_class 139 | 140 | with raises(ImportError): 141 | do_import_class("non.exist.class") 142 | 143 | 144 | def test_register_a_plugin_function_1(): 145 | PluginManager("test plugin") 146 | 147 | @PluginInfo("test plugin", tags=["akey"]) 148 | class MyPlugin(object): 149 | pass 150 | 151 | MyPlugin() 152 | 153 | 154 | def test_register_a_plugin_function_2(): 155 | non_existent_plugin = "I have no plugin manager" 156 | 157 | @PluginInfo(non_existent_plugin, tags=["akey"]) 158 | class MyPlugin(object): 159 | pass 160 | 161 | MyPlugin() 162 | assert non_existent_plugin in CACHED_PLUGIN_INFO 163 | 164 | 165 | def test_primary_key(): 166 | manager = PluginManager("test plugin2") 167 | 168 | @PluginInfo("test plugin2", tags=["primary key", "key 1", "key 2"]) 169 | class MyPlugin(object): 170 | pass 171 | 172 | pk = manager.get_primary_key("key 1") 173 | assert pk == "primary key" 174 | 175 | 176 | def test_dict_as_plugin_payload(): 177 | manager = PluginManager("test plugin3") 178 | 179 | plugin = PluginInfo("test plugin3", tags=["primary key", "key 1", "key 2"]) 180 | plugin(dict(B=1)) 181 | 182 | instance = manager.load_me_now("key 1") 183 | assert instance == dict(B=1) 184 | 185 | 186 | def test_show_me_your_name(): 187 | class Test(object): 188 | pass 189 | 190 | name = _show_me_your_name(Test) 191 | assert name == "Test" 192 | 193 | name2 = _show_me_your_name(dict(A=1)) 194 | assert "dict" in name2 195 | 196 | 197 | def make_me_a_plugin_info(plugin_name): 198 | return PluginInfo(plugin_name, "abs_path", custom="property") 199 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | from lml.utils import do_import, json_dumps 2 | from lml.plugin import PluginManager 3 | 4 | from unittest.mock import patch 5 | from pytest import raises 6 | 7 | 8 | def test_json_dumps(): 9 | class TestClass: 10 | pass 11 | 12 | float_value = 1.3 13 | adict = dict(test=TestClass, normal=float_value) 14 | json_dumps(adict) 15 | 16 | 17 | def test_do_import(): 18 | import isort 19 | 20 | test_package = do_import("isort") 21 | assert test_package == isort 22 | 23 | 24 | def test_do_import_2(): 25 | import lml.plugin as plugin 26 | 27 | themodule = do_import("lml.plugin") 28 | assert plugin == themodule 29 | 30 | 31 | @patch("lml.utils.log.exception") 32 | def test_do_import_error(mock_exception): 33 | with raises(ImportError): 34 | do_import("non.exist") 35 | mock_exception.assert_called_with( 36 | "%s is absent or cannot be imported", "non.exist" 37 | ) 38 | 39 | 40 | def test_do_import_cls(): 41 | from lml.utils import do_import_class 42 | 43 | manager = do_import_class("lml.plugin.PluginManager") 44 | assert manager == PluginManager 45 | --------------------------------------------------------------------------------