├── .gitignore ├── LICENSE ├── README.rst ├── docs ├── Makefile ├── make.bat ├── outline.rst └── source │ ├── book │ ├── hub.rst │ ├── origin.rst │ ├── plugable.rst │ ├── problem.rst │ └── subs.rst │ ├── conf.py │ ├── index.rst │ ├── releases │ ├── 6.0.0.rst │ ├── 6.1.0.rst │ ├── 7.0.0.rst │ ├── 7.1.0.rst │ ├── 7.2.0.rst │ ├── 7.3.0.rst │ ├── 7.4.rst │ ├── 7.5.rst │ ├── 7.6.rst │ └── 8.rst │ ├── topics │ ├── app_merging.rst │ ├── conf.rst │ ├── conf_integrate.rst │ ├── contracts.rst │ ├── dyne_name.rst │ ├── glossary.rst │ ├── hub_overview.rst │ ├── ideas_that_were_not_used.rst │ ├── learning.rst │ ├── pop.rst │ ├── proc.rst │ ├── story.rst │ ├── sub_patterns.rst │ └── subs_overview.rst │ └── tutorial │ └── quickstart.rst ├── pop ├── __init__.py ├── contract.py ├── dirs.py ├── exc.py ├── hub.py ├── loader.py ├── mods │ ├── conf │ │ ├── args.py │ │ ├── dirs.py │ │ ├── file_parser.py │ │ ├── init.py │ │ ├── integrate.py │ │ ├── json_conf.py │ │ ├── log │ │ │ ├── basic.py │ │ │ └── init.py │ │ ├── nix_os.py │ │ ├── reader.py │ │ ├── toml_conf.py │ │ ├── version.py │ │ └── yaml_conf.py │ ├── pop │ │ ├── conf.py │ │ ├── dicts.py │ │ ├── input.py │ │ ├── loop.py │ │ ├── ref.py │ │ ├── seed.py │ │ ├── sub.py │ │ ├── testing.py │ │ └── verify.py │ └── proc │ │ ├── init.py │ │ ├── run.py │ │ └── worker.py ├── scanner.py ├── scripts.py ├── verify.py └── version.py ├── requirements-test.txt ├── requirements.txt ├── setup.py └── tests ├── __init__.py ├── cmods ├── contracts │ └── ctest.py └── ctest.py ├── conf1 └── conf.py ├── conf2 └── conf.py ├── conf3 └── conf.py ├── conftest.py ├── contracts ├── ctx.py ├── ctx_args.py ├── ctx_update.py ├── many.py ├── priv.py ├── test.py └── testing.py ├── csigs ├── contracts │ └── sigs.py └── sigs.py ├── integration └── contracted │ ├── mods │ ├── contracted_access.py │ └── contracts │ │ └── contracted_access.py │ └── test_contracted_access.py ├── mods ├── bad.py ├── bad_import │ └── bad_import.py ├── contract_ctx │ ├── ctx.py │ ├── ctx_args.py │ └── ctx_update.py ├── coro │ └── coro.py ├── foo.py ├── iter │ ├── bar.py │ ├── foo.py │ └── init.py ├── many.py ├── nest │ └── basic.py ├── packinit │ ├── init.py │ └── packinit.py ├── priv.py ├── proc.py ├── same_vname │ ├── will_load.py │ └── will_not_load.py ├── test.py ├── testing.py ├── vbad.py ├── virt.py └── vtrue.py ├── sdirs ├── l11 │ ├── l2 │ │ └── test.py │ └── test.py ├── l12 │ ├── l2 │ │ └── test.py │ └── test.py ├── l13 │ ├── l2 │ │ └── test.py │ └── test.py └── test.py ├── tpath ├── README.rst ├── dn1 │ ├── conf.py │ └── dn1 │ │ └── nest │ │ └── dn1.py ├── dn2 │ ├── conf.py │ └── dn1 │ │ └── nest │ │ └── dn2.py ├── dn3 │ ├── conf.py │ └── dn1 │ │ └── nest │ │ ├── dn3.py │ │ └── next │ │ ├── last │ │ └── test.py │ │ └── test.py ├── dyne1 │ ├── conf.py │ ├── dyne1 │ │ ├── init.py │ │ └── test.py │ └── nest │ │ └── dyne │ │ └── nest.py ├── dyne2 │ ├── conf.py │ ├── dyne2 │ │ ├── init.py │ │ └── test.py │ └── nest │ │ └── dyne │ │ └── nest.py └── dyne3 │ ├── conf.py │ ├── dyne3 │ ├── init.py │ └── test.py │ └── nest │ └── dyne │ └── nest.py └── unit ├── __init__.py ├── test_basic.py ├── test_bench.py ├── test_contract.py ├── test_contract_ctx.py ├── test_coro.py ├── test_fail.py ├── test_proc.py ├── test_reader.py └── test_testing.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[codx] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Ignore generated C files (Cython) 9 | *.c 10 | 11 | # iPython notebooks and checkpoints 12 | *.ipynb 13 | .ipynb_checkpoints 14 | 15 | # patch files 16 | *.orig 17 | 18 | # Distribution / packaging 19 | .Python 20 | env/ 21 | bin/ 22 | build/ 23 | develop-eggs/ 24 | dist/ 25 | eggs/ 26 | lib/ 27 | lib64/ 28 | parts/ 29 | sdist/ 30 | var/ 31 | *.egg-info/ 32 | .installed.cfg 33 | *.egg 34 | MANIFEST 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .coverage 44 | .cache 45 | .pytest_cache 46 | nosetests.xml 47 | coverage.xml 48 | 49 | # Pytest 50 | pytest.ini 51 | 52 | # Translations 53 | *.mo 54 | 55 | # macOS 56 | .DS_Store 57 | 58 | # Mr Developer 59 | .mr.developer.cfg 60 | .project 61 | .pydevproject 62 | 63 | # Rope 64 | .ropeproject 65 | 66 | # Django stuff: 67 | *.log 68 | *.pot 69 | 70 | # Sphinx documentation 71 | doc/_build/ 72 | doc/user/ubu/_build/ 73 | 74 | # Local VIM RC 75 | .lvimrc 76 | 77 | # Swap files 78 | .*.swp 79 | 80 | # nvim 81 | *.un~ 82 | *~ 83 | 84 | # setup compile left-overs 85 | *.py_orig 86 | 87 | # Nuitka build 88 | nuitka/ 89 | nuitka_standalone/ 90 | 91 | # Tags 92 | TAGS 93 | tags 94 | 95 | # kdevelop 96 | *.kdev4 97 | 98 | # pycharm 99 | .idea* 100 | 101 | # VSCode 102 | .vscode* 103 | 104 | # pyenv 105 | .python-version 106 | 107 | .ci/.rootdir 108 | 109 | # Coverage data files 110 | .coverage.* 111 | 112 | runpytest.sh 113 | 114 | # vscode 115 | .vscode 116 | .pytest_cache 117 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ==== 2 | POP 3 | ==== 4 | 5 | MOVED TO GITLAB 6 | =============== 7 | 8 | POP projects developed by Saltstack are being moved to Gitlab. 9 | 10 | The new location of idem is here: 11 | 12 | https://gitlab.com/saltstack/pop/pop 13 | 14 | Intro 15 | ===== 16 | 17 | Pop is used to express the Plugin Oriented Programming Paradigm. The Plugin 18 | Oriented Programming Paradigm has been designed to make pluggable software 19 | easy to write and easy to extend. 20 | 21 | Plugin Oriented Programming presents a new way to scale development teams 22 | and deliver complex software. This is done by making the applications entirely 23 | out of plugins, and also making the applications themselves natively pluggable 24 | with each other. 25 | 26 | Using Plugin Oriented Programming it then becomes easy to have the best of both 27 | worlds, software can be build in small pieces, making development easier to 28 | maintain. But the small pieces can then be merged and deployed in a single 29 | binary, making code deployment easy as well. 30 | 31 | All this using Python, one of the world's most popular and powerful programming 32 | language. 33 | 34 | Getting Started 35 | =============== 36 | 37 | A more complete Getting Started Guide is available inside of the documentation 38 | for `pop`. The best place to start is in the doc's Getting Started Guide found 39 | here: 40 | 41 | https://pop.readthedocs.io 42 | 43 | First off, install `pop` from pypi: 44 | 45 | .. code-block:: bash 46 | 47 | pip3 install pop 48 | 49 | Now that you have `pop`, use the tool that `pop` ships with to bootstrap your 50 | project! This tool is called `pop-seed` and it will make your Python project 51 | boiler plate for you! 52 | 53 | .. code-block:: bash 54 | 55 | mkdir poppy 56 | cd poppy 57 | pop-seed poppy 58 | 59 | Now you have a `setup.py` file will detect changes to you project and "Just Work". 60 | Feel free to open it up and fill in some of the blank places, like author name, 61 | description, etc. The `pop-seed` program also made your first directories, your 62 | `run.py` startup script, everything you need to install your project and the `pop` 63 | `conf.py` file used to load in configuration. Running `pop-seed` also made a few 64 | other files, but nothing to worry about now. 65 | 66 | Congratulations! You have a `pop` project! Now you can run the project: 67 | 68 | .. code-block:: bash 69 | 70 | python3 run.py 71 | 72 | With a project up and running you can now add more plugins, more code and more 73 | plugin subsystems! 74 | 75 | What Happened? 76 | ============== 77 | 78 | Take a look at the `poppy/poppy/init.py` file, your little `run.py` script 79 | created the `hub`, loaded your first plugin subsystem, `poppy` and called 80 | the run function therein. This is the starting point for your app. 81 | 82 | Next dive into the `pop` documentation, we will take you through how to 83 | think in Plugin Oriented Programming, helping you see a new way to write 84 | code that is more flexible and dynamic than anything you have seen before! 85 | 86 | Single Binary 87 | ============= 88 | 89 | In the first few sentences of this document I promised you a single binary! 90 | This is easy to do! Just pip install `pop-build`: 91 | 92 | .. code-block:: bash 93 | 94 | pip install pop-build 95 | pop-build -n poppy 96 | 97 | This will build a single binary of your program! If something goes wrong it 98 | is most likely because PyInstaller does not support the latest version of 99 | Python yet. To fix this you can usually just run `pop-build` with the 100 | `--dev-pyinst` to build the binary with the latest development snapshot 101 | of PyInstaller. 102 | 103 | Documentation 104 | ============= 105 | 106 | Check out the docs at: 107 | 108 | https://pop.readthedocs.io 109 | 110 | There is a much more in depth tutorial here, followed by documents on how to 111 | think in Plugin Oriented Programming. Take your time to read it, it is not long 112 | and can change how you look at writing software! 113 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = pop 8 | SOURCEDIR = source 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=_build 12 | set SPHINXPROJ=pop 13 | 14 | if "%1" == "" goto help 15 | 16 | %SPHINXBUILD% >NUL 2>NUL 17 | if errorlevel 9009 ( 18 | echo. 19 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 20 | echo.installed, then set the SPHINXBUILD environment variable to point 21 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 22 | echo.may add the Sphinx directory to PATH. 23 | echo. 24 | echo.If you don't have Sphinx installed, grab it from 25 | echo.http://sphinx-doc.org/ 26 | exit /b 1 27 | ) 28 | 29 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 30 | goto end 31 | 32 | :help 33 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 34 | 35 | :end 36 | popd 37 | -------------------------------------------------------------------------------- /docs/outline.rst: -------------------------------------------------------------------------------- 1 | ================ 2 | Pop Docs Outline 3 | ================ 4 | 5 | POP docs need to cover how to write and think in POP in a way that is consumabe, 6 | has many visuals and covers the basic concepts in an easy, follow along way. 7 | 8 | Quickstart 9 | ========== 10 | 11 | Get started with pop-seed to make a simple standalone project. This makes a project 12 | that can be executed easily. 13 | 14 | Go from pop-seed all the way to defining dynamic names and using pop-build. 15 | 16 | Quickstart Vertical 17 | =================== 18 | 19 | Cover how to make vertical app-merge projects using pop-seed and dyne name 20 | 21 | What is POP? 22 | ============ 23 | 24 | Declare the problem statement- 25 | Exlplain what POP is, and how it seeks to solve this problem. 26 | 27 | Origin 28 | ====== 29 | 30 | Where did POP come from, and how does that lend the concept credibility? 31 | 32 | What does Pluggable mean? 33 | ========================= 34 | 35 | Pluggable code means that your applications become legos, endlessly attatchable 36 | 37 | Intro the hub, app merging, plugins and why conf needs to be integrated 38 | 39 | Why Patterns are so critical 40 | 41 | The Hub 42 | ======= 43 | 44 | The root of the application and namespaces. Cover why namespaces are critical 45 | in POP and why they need to be shared 46 | 47 | App Merging 48 | =========== 49 | 50 | App Merging, the core feature and concept of POP 51 | 52 | Plugins, what this means 53 | ======================== 54 | 55 | Cover what Pluggins are and what it means to make pluggable software 56 | 57 | The Conf System 58 | =============== 59 | 60 | Why the conf system is needed, how it works and how it allows app merging to happen 61 | 62 | Thinking in POP 63 | =============== 64 | 65 | POP makes you think about code differently, learn how to think in plugin systems as 66 | I/O interfaces that expose consistent, pluggable systems. 67 | 68 | Structure concepts 69 | ================== 70 | 71 | Get familiar with the layout of pop projects, the hub and subs, how they lay out 72 | and communicate 73 | 74 | 75 | What Goes Where 76 | =============== 77 | 78 | What goes in __init__ 79 | --------------------- 80 | 81 | Define what goes in the __init__ func and what does not go in the __init__ func 82 | 83 | Vars and subs 84 | ------------- 85 | 86 | Expose the conventions about how to place variables on the hub, where they go and 87 | what they mean, what does read/write look like 88 | 89 | Using Sub subs 90 | -------------- 91 | 92 | When to use subs wihthin subs and hwo it relates to data access patterns 93 | 94 | Contracts For Enforcement 95 | ------------------------- 96 | 97 | Cover how and when to use contracts 98 | 99 | Understanding Patterns 100 | ====================== 101 | 102 | Pop is all about making patters inside of subs, if patterns are not being made then 103 | the code is not pop 104 | 105 | Define what a pattern is and how to think about making patterns 106 | 107 | -------------------------------------------------------------------------------- /docs/source/book/hub.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | The Hub 3 | ======= 4 | 5 | Plugin Oriented Programming is all about namespaces. Plugin interfaces are 6 | dynamic, otherwise they would not be plugins, therefore if they need to be 7 | stared with each other, they need to have a communication medium. Instead 8 | of making a complicated API interface for the plugins to communicate `pop` 9 | allows all of the plugins on the system to be accessed via a shared 10 | hierarchical namespace. 11 | 12 | All of the plugins in the application can be reached by traversing the 13 | namespace that is the `hub`. Plugin subsystems can exist on the hub and 14 | plugin subsystems can contain more plugins subsystems and plugins. 15 | 16 | But the `hub` is not just for plugins, we need to be able to store data 17 | associated with our functions. The data is stored on the `hub`, in locations 18 | in the namespace relative to where the functions that use that data reside. 19 | 20 | Getting Stated With the Hub 21 | =========================== 22 | 23 | The `hub` is the root of a `pop` project. Before we can start working with 24 | `pop` we must first create the `hub`. Normally, when using `pop-seed` you don't 25 | even need to consider where the `hub` comes from, as `pop-seed` creates the 26 | `hub` for you in the startup scripts. But this document is intended for 27 | understanding, so lets look at a `pop` app from the ground up. 28 | 29 | .. code-block:: python 30 | 31 | import pop.hub 32 | 33 | hub = pop.hub.Hub() 34 | 35 | Well, that was easy! Now we have a `hub`! When working on an existing codebase 36 | it is easy to wonder "Where did this hub come from?". The `hub` is created in 37 | the `run.py` file, and the first function is called from there as well. 38 | 39 | Once the hub is created, and the first function called, then the `hub` is passed 40 | to each function as the first argument. This is just like classes in python where 41 | each function receives the self variable as the first argument. 42 | 43 | The First Subsystem - pop 44 | ========================= 45 | 46 | When the `hub` is created it comes pre-loaded with the first plugin subsystem. A 47 | plugin subsystem is often referred to as just a `sub`. This first plugin subsystem 48 | is called `pop`. It contains the tools needed to extend the `hub` and add additional 49 | `subs`. 50 | 51 | When calling a reference on the `hub` the full path to the reference is called. This 52 | allows for all code to always exist in a finite, traceable location. 53 | 54 | .. code-block:: python 55 | 56 | import pop.hub 57 | 58 | hub = pop.hub.Hub() 59 | hub.pop.sub.add('poppy.poppy') 60 | hub.poppy.init.run() 61 | 62 | The `pop` `sub` contains a plugin called `sub` which is used for managing `subs`. 63 | Inside we find a simple function called `add`. This function allows for adding 64 | new plugin subsystems onto the `hub`. 65 | 66 | The `hub.pop.sub.add` function is very robust, allowing for new plugin subsystems 67 | to be added to the hub in many ways. The most common way is to use *Dynamic Names* 68 | which allows for *Vertical App Merging*. This is covered in much more depth in the 69 | section on *Subs*. 70 | 71 | Once the new `sub` has been added to the `hub` it can be referenced. The `hub` is 72 | not a complicated object, like everything in `pop` it is designed to be easily 73 | understood and simple. 74 | 75 | Now that you have a basic understanding of the `hub` we can move on to *Subs*. 76 | After you have a good understanding of *Subs* We can come back to the `hub` and 77 | go into more depth on how these critical components work together. 78 | -------------------------------------------------------------------------------- /docs/source/book/origin.rst: -------------------------------------------------------------------------------- 1 | ============= 2 | Origin of POP 3 | ============= 4 | 5 | If Plugin Oriented Programming claims to address difficult problems in modern software 6 | engineering, then it needs to have come from earlier attempts to solve these problems. 7 | Plugin Oriented Programming originates from development models that have been well 8 | established and is derived from successful software. 9 | 10 | Plugin oriented design aspects have long driven large scale development, most 11 | large software projects incorporate plugins and modular design. But plugins are almost 12 | always an afterthought, pushed into the codebase after the fact and end up exposing 13 | only a few plugin interfaces. 14 | 15 | Salt's Plugin System and POP 16 | ============================ 17 | 18 | The creator of Plugin Oriented Programming, Thomas Hatch, is also the creator of one of 19 | the world's largest open source projects "Salt". The design of Salt incorporated plugins 20 | very early in the project and build a reusable plugin system. 21 | 22 | The pluggable nature of Salt quickly became one of the main driving factors for the 23 | project, making contributions easy and the platform flexible. Salt's plugin system, called 24 | the "Salt Loader" was used to create many more plugin subsystems for Salt over the years 25 | and is now used for over 35 plugin systems in Salt. 26 | 27 | Adventures in Plugins 28 | ===================== 29 | 30 | Thomas realized the benefits of plugins as they existed in Salt and began to explore how 31 | to make a standalone plugin system. He made a plugin system which he called called "Pack", 32 | but felt that it still had many of the limitations that existed in the Salt Loader. So 33 | Thomas decided to attempt to make this plugin system in other languages. He made it in 34 | Go, Julia, Nim, and a few others, but came out of the experience with new ideas about how 35 | to approach the problem. 36 | 37 | Thomas then wrote Pack2, but this version of the plugin system had many new components. 38 | Pack2 was used by SaltStack to write a few software components and was eventually 39 | deeply tied to a number of software projects within SaltStack. 40 | 41 | But this experience also added to a deeper understanding about how plugin design should 42 | work. Thomas at this point realized that he had not just made a plugin library, but that 43 | he had create a programming paradigm. With this realization, Thomas renamed Pack to Pop 44 | to reflect that the implementation had grown into Plugin Oriented Programming. 45 | 46 | At this point Thomas changed how the plugin system worked, yet again, but his time his 47 | view of the problem set had grown a great deal. Now he was able to overcome many restrictions 48 | in the design and introduced the concepts around app merging and configuration merging. 49 | 50 | At this point Thomas had created Plugin Oriented Programming. The journey through writing 51 | mountains of code for many projects had led him to a much deeper understanding of programming 52 | problems and new ways to solve them. 53 | 54 | Emergence of POP 55 | ================ 56 | 57 | Plugin Oriented Programming was first released in late 2019 and began to gain a following. This 58 | book was written in response to more and more people showing interest in Plugin Oriented Programming 59 | and as the primary reference for the core concepts that make up Plugin Oriented Programming. 60 | -------------------------------------------------------------------------------- /docs/source/book/subs.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | Plugin Subsystems - Subs 3 | ======================== 4 | 5 | Plugin Subsystems - or *Subs* - are the basic containers for code in Plugin 6 | Oriented Programming. *Subs* contain both plugins and data, these *Subs* are 7 | available to the application via the `hub`. 8 | 9 | The concept of the *Sub* roughly as central and critical to Plugin Oriented 10 | Programming as the concept of a class is to Object Oriented Programming. 11 | This is therefore the central focus of how to create a plugin oriented 12 | application. 13 | 14 | Subs Contain Plugins 15 | ==================== 16 | 17 | On the surface, *Subs* appear to simply contain plugins, but this is similar 18 | to classes on the surface, as they appear to just contain data and functions. 19 | But like classes, *Subs* are the canvas for the developer, allowing for the 20 | complexity of the software to be simplified into reusable, flexible, compartments. 21 | While the *Sub* is comparible to criticality to classes in OOP, it should not be 22 | confused with classes! Plugin Oriented Programming does not have a concept 23 | inheritence for instance. 24 | 25 | When a *Sub* is created it gets placed on the higherarchical namespace - the `hub` - 26 | so that any application can call into functions and data exposed by the *Sub*. 27 | 28 | The *Sub's* primary purpose is the present plugins. The plugins are individual 29 | files containing tight code compartments that express an interface. 30 | 31 | Adding Subs to a hub 32 | ==================== 33 | 34 | Once a `hub` is available then it is easy to add a *Sub* to the `hub`. Keep in 35 | mind that there are many ways to construct a *Sub*. In the end a *Sub* 36 | only needs to be a directory with .py files in it, nothing more. With that 37 | said though, there are many ways to identify how to find the directory, or 38 | directories! 39 | 40 | But do not dispair, `pop` presents many flexible options, but retains a strong 41 | opinion about how to do things right. 42 | 43 | Using Dynamic Names 44 | ------------------- 45 | 46 | The best way to add a Sub to your `hub` is to use *Dynamic Names*. Using 47 | *Dynamic Names* is easy. It requires you to register your *Sub* with the `pop` 48 | configuration system. 49 | 50 | Assuming you started with a `pop-seed` project called "poppy", open up the 51 | project's `conf.py` file and add a *Sub* called `rpc`. Teh `conf.py` file can be 52 | found at `poppy/conf.py` AKA `/conf.py`: 53 | 54 | .. code-block:: python 55 | 56 | DYNE = { 57 | 'rpc': ['rpc'], 58 | } 59 | 60 | The `conf.py` will already exist, `pop-seed` creates it. When you add a **DYNE** 61 | entry to the `conf.py` it registers that under this project plugins for the *Sub* 62 | - in this case named `rpc` - can be found. 63 | 64 | This means that multiple codebases can contribute plugins to the *Sub*! This means 65 | that very easily third party developers can extend your *Sub* without modifying 66 | your code to do it! This is one of the most powerful aspects of Plugin Oriented 67 | Programming called *Vertical App Merging*. 68 | 69 | Now that your *Sub* is registered, it can be added to a `hub` by calling 70 | `hub.pop.sub.add` and passing in the `dyne_name` argument: 71 | 72 | .. code-block:: python 73 | 74 | hub.pop.sub.add(dyne_name='rpc') 75 | 76 | Since the `hub` is availabe anywhere in your application, this can be called from 77 | anywhere. A plugin can add a new *Sub*, this is a core design consideration of 78 | Plugin Oriented Programming, a plugin cannot be a dead end, it needs to be able 79 | to always add globally available *Subs* onto the `hub`. 80 | 81 | Using PyPath 82 | ------------ 83 | 84 | Using the `pypath` system in `pop` is the simplest way to add a *Sub* to your `hub`, 85 | but it does not activate *Vertical App Merging*. 86 | 87 | Using `pypath` is simple, just call `hub.pop.sub.add` and pass in the python import 88 | path that, when impoerted, will lead the way to the directory with plugins. 89 | 90 | .. code-block:: python 91 | 92 | hub.pop.sub.add('poppy.poppy') 93 | 94 | Thats it! Now you have a new *Sub* on your `hub` called `poppy` that will find 95 | your plugins by importing `poppy.poppy`, looking at the imported module's `path` 96 | variable and translating that into a directory which is then scaned for plugins. 97 | 98 | Recursive Sub Loading 99 | ===================== 100 | 101 | When a *Sub* gets added to the `hub` it can be desireable to find any directories 102 | under the found director(ies) and add them to the *Sub* as nested *Subs*. This can 103 | be an excellent way to organize code, allowing for *Subs* to exist nested within 104 | each other on the `hub`. 105 | 106 | To recursively scan for nested *Subs* just call 107 | `hub.pop.sub.load_subdirs(hub.subname, recurse=True)`. This will look in all of the 108 | directories defined in the loaded Sub and scan for any subdirectories. If those 109 | subdirectories exist they will be found and loaded onto the `hub` as nested *Subs*. 110 | 111 | Plugin Loading 112 | ============== 113 | 114 | By default plugins in `pop` are lazy loaded, they only get imported when they are 115 | first accesed. This gratly speeds up the creation of a *Sub* because if a *Sub* 116 | contained hundreds of Plugins then it would take as much as a few seconds to load 117 | all of the modules therein. 118 | 119 | Sometimes it may be desireable to pre-load all of your plugins, to do this just call 120 | `hub.pop.sub.load_all(hub.subname)`. 121 | 122 | Some events will trigger loading all plugins, in particular if you decide to iterate 123 | over a *Sub* then the `load_all` call will be executed on the *Sub* if it has not been 124 | already, therefore ensuring that all plugins are loaded and can be cleanly iterated 125 | over. 126 | 127 | The init System 128 | =============== 129 | 130 | Now that your *Sub* is on your hub, lets take a look at the *Init* system used 131 | by `pop`. This system allows you to initialize your new *Sub*. In a nutshell 132 | you can place an optional plugin in your *Sub* named `init.py` and this plugin 133 | will be automatically loaded when your *Sub* gets created. Think of the `init.py` 134 | as the plugin that defines how your *Sub* will function. 135 | 136 | The __init__ Function 137 | ===================== 138 | 139 | Just like Classes in Python, plugins in `pop` can be initialized. Just create an 140 | optional function called `__init__`. This function will be called when the plugin 141 | gets loaded. 142 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Configuration file for the Sphinx documentation builder. 4 | # 5 | # This file does only contain a selection of the most common options. For a 6 | # full list see the documentation: 7 | # http://www.sphinx-doc.org/en/master/config 8 | 9 | # -- Path setup -------------------------------------------------------------- 10 | 11 | # If extensions (or modules to document with autodoc) are in another directory, 12 | # add these directories to sys.path here. If the directory is relative to the 13 | # documentation root, use os.path.abspath to make it absolute, like shown here. 14 | # 15 | # import os 16 | # import sys 17 | # sys.path.insert(0, os.path.abspath('.')) 18 | 19 | 20 | # -- Project information ----------------------------------------------------- 21 | 22 | project = 'pop' 23 | copyright = '2020, Thomas S Hatch' 24 | author = 'Thomas S Hatch' 25 | 26 | # The short X.Y version 27 | version = '8' 28 | # The full version, including alpha/beta/rc tags 29 | release = version 30 | 31 | 32 | # -- General configuration --------------------------------------------------- 33 | 34 | # If your documentation needs a minimal Sphinx version, state it here. 35 | # 36 | # needs_sphinx = '1.0' 37 | 38 | # Add any Sphinx extension module names here, as strings. They can be 39 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 40 | # ones. 41 | extensions = [ 42 | 'sphinx.ext.githubpages', 43 | ] 44 | 45 | # Add any paths that contain templates here, relative to this directory. 46 | templates_path = ['_templates'] 47 | 48 | # The suffix(es) of source filenames. 49 | # You can specify multiple suffix as a list of string: 50 | # 51 | # source_suffix = ['.rst', '.md'] 52 | source_suffix = '.rst' 53 | 54 | # The master toctree document. 55 | master_doc = 'index' 56 | 57 | # The language for content autogenerated by Sphinx. Refer to documentation 58 | # for a list of supported languages. 59 | # 60 | # This is also used if you do content translation via gettext catalogs. 61 | # Usually you set "language" from the command line for these cases. 62 | language = None 63 | 64 | # List of patterns, relative to source directory, that match files and 65 | # directories to ignore when looking for source files. 66 | # This pattern also affects html_static_path and html_extra_path . 67 | exclude_patterns = [] 68 | 69 | # The name of the Pygments (syntax highlighting) style to use. 70 | pygments_style = 'sphinx' 71 | 72 | 73 | # -- Options for HTML output ------------------------------------------------- 74 | 75 | # The theme to use for HTML and HTML Help pages. See the documentation for 76 | # a list of builtin themes. 77 | # 78 | html_theme = 'alabaster' 79 | 80 | # Theme options are theme-specific and customize the look and feel of a theme 81 | # further. For a list of options available for each theme, see the 82 | # documentation. 83 | # 84 | # html_theme_options = {} 85 | 86 | # Add any paths that contain custom static files (such as style sheets) here, 87 | # relative to this directory. They are copied after the builtin static files, 88 | # so a file named "default.css" will overwrite the builtin "default.css". 89 | html_static_path = ['_static'] 90 | 91 | # Custom sidebar templates, must be a dictionary that maps document names 92 | # to template names. 93 | # 94 | # The default sidebars (for documents that don't match any pattern) are 95 | # defined by theme itself. Builtin themes are using these templates by 96 | # default: ``['localtoc.html', 'relations.html', 'sourcelink.html', 97 | # 'searchbox.html']``. 98 | # 99 | # html_sidebars = {} 100 | 101 | 102 | # -- Options for HTMLHelp output --------------------------------------------- 103 | 104 | # Output file base name for HTML help builder. 105 | htmlhelp_basename = 'popdoc' 106 | 107 | 108 | # -- Options for LaTeX output ------------------------------------------------ 109 | 110 | latex_elements = { 111 | # The paper size ('letterpaper' or 'a4paper'). 112 | # 113 | # 'papersize': 'letterpaper', 114 | 115 | # The font size ('10pt', '11pt' or '12pt'). 116 | # 117 | # 'pointsize': '10pt', 118 | 119 | # Additional stuff for the LaTeX preamble. 120 | # 121 | # 'preamble': '', 122 | 123 | # Latex figure (float) alignment 124 | # 125 | # 'figure_align': 'htbp', 126 | } 127 | 128 | # Grouping the document tree into LaTeX files. List of tuples 129 | # (source start file, target name, title, 130 | # author, documentclass [howto, manual, or own class]). 131 | latex_documents = [ 132 | (master_doc, 'pop.tex', 'pop Documentation', 133 | 'Thomas S Hatch', 'manual'), 134 | ] 135 | 136 | 137 | # -- Options for manual page output ------------------------------------------ 138 | 139 | # One entry per manual page. List of tuples 140 | # (source start file, name, description, authors, manual section). 141 | man_pages = [ 142 | (master_doc, 'pop', 'pop Documentation', 143 | [author], 1) 144 | ] 145 | 146 | 147 | # -- Options for Texinfo output ---------------------------------------------- 148 | 149 | # Grouping the document tree into Texinfo files. List of tuples 150 | # (source start file, target name, title, author, 151 | # dir menu entry, description, category) 152 | texinfo_documents = [ 153 | (master_doc, 'pop', 'pop Documentation', 154 | author, 'pop', 'One line description of project.', 155 | 'Miscellaneous'), 156 | ] 157 | 158 | 159 | # -- Extension configuration ------------------------------------------------- 160 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. pop documentation master file, created by 2 | sphinx-quickstart on Mon Jun 25 20:19:13 2018. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to pop's documentation! 7 | =============================== 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | :glob: 12 | 13 | tutorial/quickstart 14 | topics/learning 15 | topics/hub_overview 16 | topics/subs_overview 17 | topics/sub_patterns 18 | topics/conf 19 | topics/conf_integrate 20 | topics/app_merging 21 | topics/dyne_name 22 | topics/contracts 23 | topics/proc 24 | topics/story 25 | topics/pop 26 | topics/glossary 27 | topics/ideas_that_were_not_used 28 | releases/* 29 | 30 | 31 | Indices and tables 32 | ================== 33 | 34 | * :ref:`genindex` 35 | * :ref:`modindex` 36 | * :ref:`search` 37 | -------------------------------------------------------------------------------- /docs/source/releases/6.0.0.rst: -------------------------------------------------------------------------------- 1 | ================= 2 | Pop Release 6.0.0 3 | ================= 4 | 5 | The Plugin Oriented Programming implementation has reached version 6.0.0! 6 | 7 | This version introduces many changes to the system and established a long researched 8 | baseline of functionality. 9 | 10 | Why Version 6? 11 | ============== 12 | 13 | Pop is the 5th major iteration on the concept of Plugin Oriented Programming. It 14 | started as the Salt Loader System, then a project called pack, then pack in Julia, 15 | followed by another implementation in Python and then, finally, `pop`. 16 | 17 | This release cleans up many of the concepts that were hold overs from earlier 18 | systems and starts to add some of the long aspirational features to the platform. 19 | It is therefore a worthy version "6". 20 | 21 | Dynamic Names 22 | ============= 23 | 24 | Version 6 introduces Dynamic Names to `pop` and implements a critical component 25 | of Plugin Oriented Programming. 26 | 27 | Dynamic Names is part of the concept around App Merging. App Merging was introduced 28 | in `pop` 5, in that multiple applications can be merged onto the hub and configuration 29 | for said apps can also be merged onto the hub via the `conf.integrate` system. 30 | 31 | But Dynamic Names allows for a plugin subsystem to be dynamically expanded by additional 32 | apps being installed. With Dynamic Names a plugin subsystem can be dynamically extended 33 | by simply installing an additional python package that defines additions to the 34 | used Dynamic Name. This makes extending a project via external repos and installs 35 | not only easy, but completely transparent. 36 | 37 | hub._ and hub.__ 38 | ================ 39 | 40 | One of the issues with the `hub` is that all references need to fully qualified. Pop 6 41 | introduces `hub._` as a dynamic reference to the current module and `hub.__` as a dynamic 42 | reference to the current plugin subsystem. 43 | 44 | Move init.new and __mod_init__ to __init__ 45 | ========================================== 46 | 47 | To be more pythonic we have introduced the `__init__` function as a replacement 48 | for the `init.new` function and for the `__mod_init__` function. This helps to 49 | consolidate what these functions were used for into a single location. 50 | 51 | Contracts dir Autodetected 52 | ========================== 53 | 54 | In `pop` 5 it was suggested to make a `mods` and a `contracts` directory and 55 | statically direct the sub to both of them. In `pop` 6 this is still supported 56 | but now a `sub` can have a `contracts` directory inside and the contracts will 57 | be autoloaded. 58 | 59 | Add getattr to the hub 60 | ====================== 61 | 62 | Now getattr can look up nested refs on the hub: getattr(hub, 'pop.sub.add') 63 | 64 | Don't Need __init__.py files anymore 65 | ==================================== 66 | 67 | Python 3.3 deprecated the need to use *__init__.py* files. The old loading code 68 | was using these files to discern paths. Pop now uses the paths system introduced 69 | in Python 3.3. 70 | -------------------------------------------------------------------------------- /docs/source/releases/6.1.0.rst: -------------------------------------------------------------------------------- 1 | ================= 2 | Pop Release 6.1.0 3 | ================= 4 | 5 | The Plugin Oriented Programming implementation has reached version 6.1.0! 6 | 7 | This version has a few updates and fixes, nothing major apart from fixing 8 | a major issue in the `dyne_name` system introduced in 6.0.0 9 | 10 | Dyne Name Fix 11 | ============= 12 | 13 | When importing a module it needs to be given a reference name with the 14 | `sys.modules` dict as part of the import. There was in issue with how these 15 | names were being generated for the dyne_name system. It was repaired 16 | by making unknown path originating refs prefaced with a random 17 | string. This should likely be refined in the future to reference the 18 | actual file path. 19 | 20 | Remove Last imp Refs 21 | ==================== 22 | 23 | The imp library has been deprecated since python 3.3, but it was still 24 | being used, primarily out of old habits! All `imp` refs have now been 25 | removed. 26 | 27 | Refined pypath Inferred Names 28 | ============================= 29 | 30 | When pypath is used to add a new sub to your hub the name of the sub 31 | can be dynamically derived from the pypath. There was a bug where if the 32 | pypath was a list this would fail, it has been fixed. 33 | -------------------------------------------------------------------------------- /docs/source/releases/7.0.0.rst: -------------------------------------------------------------------------------- 1 | ================= 2 | Pop Release 7.0.0 3 | ================= 4 | 5 | The Plugin Oriented Programming implementation has reached version 7.0.0! 6 | 7 | The main highlight of this release is the addition of contract sigs. This 8 | new interface allows for contracts to enforce plugin implementation on 9 | a granular level. 10 | 11 | The main deprecation of this release is that the `tools` sub has been 12 | renamed to `pop`. This is a serious breaking change as if affects all 13 | pop based software, but since we are early on here, now is the time! 14 | 15 | Contract Signatures 16 | =================== 17 | 18 | Contract Signatures or `sigs` allow for plugin interfaces to be defined 19 | and enforced via the contract system. This makes it so that a plugin 20 | will fail to load if it does not implement the function signature that 21 | is defined in the contract. 22 | 23 | Contract Signatures also enforce python typing annotations. This 24 | allows for very granular typing to be mandated for defined 25 | interfaces. 26 | 27 | Take a look at the new contracts document for more details 28 | 29 | Removal of the functions Interface From Contracts 30 | ================================================= 31 | 32 | The original implementation of contracts looked for a function called 33 | `functions` which would return a list of required functions for a 34 | contracted plugin to implement. With the introduction of sigs this feature 35 | is completely superseded and is no longer needed. 36 | 37 | Change tools to pop 38 | =================== 39 | 40 | In older docs and releases the `tools` sub is automatically added when 41 | a new hub is created. We decided to change the name of `tools` to `pop`. 42 | This is a serious breaking change, but this is very early in the development 43 | of pop and so this is the time to do it! 44 | 45 | To fix your existing code just grep for `hub.tools` and replace them with 46 | refs to `hub.pop`. 47 | -------------------------------------------------------------------------------- /docs/source/releases/7.1.0.rst: -------------------------------------------------------------------------------- 1 | ================= 2 | Pop Release 7.1.0 3 | ================= 4 | 5 | This release comes with a few fixes and minor additions. 6 | 7 | Fix to Sigs System 8 | ================== 9 | 10 | This release fixes an oversight in the sigs system to allow for 11 | better enforcement of sigs. There was a situation with args that 12 | was being missed. 13 | 14 | Given the extensive possibilities when matching `*args` and 15 | `**kwargs` in options I suspect that there will be a couple more 16 | minor findings before sigs is perfected. But with that said I am 17 | not currently aware of any issues. 18 | 19 | Addition of Static Build System 20 | =============================== 21 | 22 | Also we have added a system to `pop-seed` that will build a single 23 | binary from a pop project. The new `build.py` script will be added 24 | to projects by the pop-seed command. 25 | 26 | This new script will download and install the needed tools to build 27 | the single binary into a venv. This system uses PyInstaller and 28 | compensates for some of the things that PyInstaller has trouble handling 29 | for pop projects where we manipulate the import system so much. 30 | 31 | Addition of Logging into the conf System 32 | ======================================== 33 | 34 | Now when you call `hub.pop.conf.integrate` there is a new option, `logs`. 35 | If `logs` is set to True then logging options will be added to your 36 | configuration options and a logger will be set up and made available to 37 | use just by adding `import logging; log = logging.getLogger(__name__)` to 38 | your plugins! 39 | 40 | The logging system in pop is of course... Pluggable! So it is easy to add 41 | more powerful plugins for kore advanced logging int he future. 42 | 43 | -------------------------------------------------------------------------------- /docs/source/releases/7.2.0.rst: -------------------------------------------------------------------------------- 1 | ================= 2 | Pop Release 7.2.0 3 | ================= 4 | 5 | This release comes with a few fixes and minor additions. 6 | 7 | Auto Load Plugins on Iteration 8 | ============================== 9 | 10 | Plugins are normally lazy loaded, but when iterating over a subsystem 11 | all plugins need to be loaded. Since one would expect to be able to 12 | iterate over a sub, it has been added to automatically load all 13 | of the plugins when starting to iterate. 14 | 15 | Add Callbacks For Signal Catches 16 | ================================ 17 | 18 | The loop inside of `pop` can now be started with functions to call when 19 | receiving signals on unix systems. This makes it easier to respond cleanly 20 | to interrupt and term signals. 21 | -------------------------------------------------------------------------------- /docs/source/releases/7.3.0.rst: -------------------------------------------------------------------------------- 1 | ================= 2 | Pop Release 7.3.0 3 | ================= 4 | 5 | This release fixes some bugs and makes the dyne loader more robust 6 | 7 | Recursive Subdirs 8 | ================= 9 | 10 | The hub.pop.sub.load_subdirs function got a new argument, `recurse`. 11 | This allows subdir loading to be recursive if set to True. 12 | 13 | This loading also allows for recursive subdirs to be loaded for 14 | disparate dyne_name loads. This means that if your subsystem is 15 | derived from multiple path sources the names get auto merged into 16 | the correct paths. 17 | 18 | Fix Issue With Class Loading 19 | ============================ 20 | 21 | We discovered that modules were getting reloaded too much and causing 22 | issues with loading classes onto the `hub`. This has been fixed 23 | 24 | Add DYNE to conf.py in pop-seed 25 | =============================== 26 | 27 | `pop-seed` continues to be refined, not the `DYNE` dict is added by default 28 | top the conf.py file created by `pop-seed` 29 | -------------------------------------------------------------------------------- /docs/source/releases/7.4.rst: -------------------------------------------------------------------------------- 1 | =============== 2 | Pop Release 7.4 3 | =============== 4 | 5 | This release fixes some bugs and adds initial Windows support 6 | 7 | Windows Support 8 | =============== 9 | 10 | POP now runs on Windows! The only things left to add is to make the 11 | conf system more robust on Windows. We add the ability 12 | for the conf system to get options from the registry in the same way that 13 | Linux can use environment variables. We also need to add automatic path 14 | management for Windows as it exists for root/user paths on Linux. 15 | 16 | Windows support for loops also changed, there were issues in using the 17 | Proactor loop and now we just use the default loop. This implies that 18 | Python 3.8 should be used for Windows as it uses the Proactor loop by 19 | default. 20 | 21 | Add pop-seed -t v 22 | ================= 23 | 24 | With the introduction of Idem it has become expedient to allow pop-seed to 25 | create POP trees that do not include the tools needed for a standalone 26 | project but just the structure for a vertical app-merge project. This 27 | can now be accomplished with the `pop-seed -t v ` command. 28 | 29 | Update Quickstart 30 | ================= 31 | 32 | The quickstart guide has been updated to reflect changes made in POP 7 33 | and made to be more fluid in exposing the RPC concept. It also got the 34 | addition of some curl commands to test poppy. 35 | -------------------------------------------------------------------------------- /docs/source/releases/7.5.rst: -------------------------------------------------------------------------------- 1 | =============== 2 | Pop Release 7.5 3 | =============== 4 | 5 | This release fixes a very minor memory issue and extends conf. 6 | 7 | Multiple subs In Conf 8 | ===================== 9 | 10 | While working on a new project I wanted to be able to have the same 11 | options available across multiple subcommands in conf. 7.5 adds this 12 | ability: 13 | 14 | .. code-block:: python 15 | 16 | CLI_CONFIG = { 17 | 'foo': { 18 | 'default': False}, 19 | 'action': 'store_true', 20 | 'subs': ['create', 'remove', 'edit'], 21 | }, 22 | } 23 | 24 | SUBS = { 25 | 'create': { 26 | 'help': 'Create some things', 27 | 'desc': Used to create the things, 28 | }, 29 | 'remove': { 30 | 'help': 'Remove some things', 31 | 'desc': Used to remove the things, 32 | }, 33 | 'edit': { 34 | 'help': 'Edit some things', 35 | 'desc': Used to edit the things, 36 | }, 37 | } 38 | 39 | Minor Memory Issue 40 | ================== 41 | 42 | While running memory and profiling tests we discovered that if the dyne system 43 | was called repeatedly that we leaked loading `conf.py` files. This has been fixed 44 | and the performance of the dyne system at scale has been greatly improved! 45 | -------------------------------------------------------------------------------- /docs/source/releases/7.6.rst: -------------------------------------------------------------------------------- 1 | =============== 2 | Pop Release 7.6 3 | =============== 4 | 5 | This release fixes a minor bug in contracts. 6 | 7 | Contract Direct Calls 8 | ===================== 9 | 10 | When executing a function with a pre contract but not a post 11 | contract, changes to the function args made in the pre 12 | contract did not carry to the called function. 13 | -------------------------------------------------------------------------------- /docs/source/releases/8.rst: -------------------------------------------------------------------------------- 1 | ============= 2 | Pop Release 8 3 | ============= 4 | 5 | Pop 8 introduces a few structural changes to the loader and some major additions 6 | to `pop-seed`. 7 | 8 | Function Variable Pass Through 9 | ============================== 10 | 11 | Pop wraps functions on the hub with contracted objects. This resulted in an issue 12 | where decorators that add variables to function objects did not always become 13 | visible to the contracted objects on the hub. This prevented decorators in 14 | a number of tools to not work with `pop`. 15 | 16 | To fix this we have made it so that function level variables are now available on 17 | the contracted object. 18 | 19 | Pop Seed Additions 20 | ================== 21 | 22 | Pop-seed now uses Dynamic Names by default when creating new projects. This moves 23 | the mindset towards making sure that developers are more app-merge centric. 24 | 25 | The ability to add dynamic names to the conf during the `pop-seed` call have also 26 | been added. Now it is possible to pass the -d option to `pop-seed` and list what 27 | dynamic names need to be created. Then those dynamic names get added to the `conf.py` 28 | and the needed directories are created. 29 | -------------------------------------------------------------------------------- /docs/source/topics/app_merging.rst: -------------------------------------------------------------------------------- 1 | .. _app_merging: 2 | 3 | ============================ 4 | Application Merging Overview 5 | ============================ 6 | 7 | Application Merging is one of the most powerful components inside of Plugin 8 | Oriented Programming. This is the idea that since an application is comprised 9 | entirely of plugins using a predictable memory namespace, one should be able 10 | to take full applications and merge them onto a single hub, and a single 11 | application. 12 | 13 | There are a number of goals and motivations for this concept. First off it 14 | enforces good software development, forcing that rules are followed in 15 | keeping the structure clean to avoid doing things that will violate app merging. 16 | 17 | Some things that violate app merging are creating multiple hubs in an application 18 | as the merged app can get confused on how to address namespaces. 19 | 20 | .. note:: 21 | 22 | The ability to make multiple hubs is deeply discouraged, but so is making 23 | singletons. The ability to have multiple hubs should always be supported 24 | to maintain a clean separation of the hub from the underlying programming 25 | language implementation. 26 | 27 | Another violation of app merging is to not follow the hub namespace rules, this 28 | creates opportunities for namespace collisions. 29 | 30 | Why is App Merging a Big Deal? 31 | ============================== 32 | 33 | The model inside of POP is not only built to make plugins easy to use, and make 34 | some cool namespaces, but to enable app merging. 35 | 36 | When looking at any idealized concept, social, political, theological, engineering, 37 | etc. we paint a picture of an ideal world. We say that "Only if people would follow 38 | this ideal, all would be well". 39 | 40 | This is a grossly naive assertion to make on humanity, that we as humans seem to 41 | never stop making. Even when a philosophy is presented to compensate for human 42 | weakness we tend to corrupt that philosophy. So instead of looking at a concept 43 | through the eyes of an idealized future we should look at concepts as ways to 44 | compensate for human weakness while looking to accomplish real goals. 45 | 46 | In application development we know that applications that are developed as collections 47 | of libraries present more reusable components. But in reality this ideal not often 48 | realized. This is because the creation of the application is the fundamental goal. 49 | Writing software is not about elegant engineering as much as it is about producing 50 | results. Any experienced software engineer has seen both sides of the spectrum, 51 | projects that are garbage because business goals drive poor engineering and projects 52 | that never see the light of day, or don't present a strong use case because so 53 | much energy has been put into clean engineering that the project becomes pure art, 54 | the only purpose it has is itself. 55 | 56 | POP presents interfaces to users in an attempt to solve this problem. Allow developers 57 | to natively and quickly create applications that solve busies needs but natively 58 | present themselves as clean, re-usable, library driven applications that can be 59 | easily re-used, in part or in whole. 60 | 61 | The hub, plugins and patterns enable this! If an application is comprised of plugins 62 | then interfaces that need extending or re-purposing can be easily added inside 63 | a merged app. Patterns present isolated processes that deliver standalone value. 64 | This allows us to extend many of the merits of OOP, now we have, not only data 65 | and functions brought together, but we have entire application workflows 66 | encapsulated in reusable patterns. 67 | 68 | App Merging Collision Points 69 | ============================ 70 | 71 | When following POP there should be few app merging collision points. If namespaces 72 | are kept clean, and if namespace rules are followed, then the result is one where 73 | collisions should rarely occur. 74 | 75 | The place where collisions are very like to occur though, is in configuration 76 | merging. This presents a truly serious issue. The conf system is built to resolve 77 | the collision problem to the smallest possible surface area. That surface area is 78 | the input level, where the configuration data is presented from the user. 79 | 80 | Once the configuration information is loaded into the hub it has been namespaced 81 | and made available on the respective namespace used by the subs. But before that 82 | it needs to be read in from the cli. This is what the `conf.integrate` system 83 | is designed to solve. The information read from the conf.py files can be modified 84 | to resolve collisions without having an effect on the operation of the underlying 85 | configured systems. 86 | 87 | How do I App Merge? 88 | =================== 89 | 90 | Acctually doing an app merge is simple, it is really done in 2 places. 91 | 92 | First, when you call `conf.integrate` the first option is a list of the apps you 93 | want to load configuration from. Just add the top level python import you want to 94 | use to the list of `conf.integrate` interfaces. 95 | 96 | Second, add the subs to your hub, you will likely need to start up the subs and make 97 | your high level app start using the interfaces exposed by the subs, but that 98 | should be it! You should be able to add subs onto your hub just like they are really 99 | powerful classes. 100 | -------------------------------------------------------------------------------- /docs/source/topics/conf_integrate.rst: -------------------------------------------------------------------------------- 1 | .. _conf_integrate_overview: 2 | 3 | ==================== 4 | The Integrate System 5 | ==================== 6 | 7 | Now that you have a clear view of the available constructs in configuration dicts used by 8 | the `conf` system we can talk about the `conf.integrate` module. By itself `conf` is a great 9 | tool to load configs, but `pop` is all about dynamically merging multiple plugin subsystems. 10 | Dynamically merging applications presents a significant issue when it comes to configuration, 11 | `conf` and the `conf.integrate` systems are designed to work together to solve this issue. 12 | 13 | Using `conf.integrate` is made to be as easy as possible, but it also means that the 14 | configuration system follows a clear model. 15 | 16 | When making a `pop` project, everything is a plugin, but you may have noticed that the 17 | `pop_seed` script makes two python files outside of the plugin system. These files are 18 | `version.py` and `conf.py`. The `version.py` file is hopefully self explanatory. But 19 | the `conf.py` file needs a little explanation. 20 | 21 | The Config Dicts 22 | ================ 23 | 24 | The integrate system uses this *conf.py* file to simply define CLI options, local config 25 | options, and options that we assume other systems would share. These types of 26 | configuration data are defined in configuration dicts in *conf.py*. 27 | 28 | Simply populate these dicts with configuration data and it can be easily 29 | and dynamically loaded by other `pop` projects. 30 | 31 | CONFIG 32 | ------ 33 | 34 | The `CONFIG` dict is where the configuration options used specifically by the subsystems 35 | defined in this project. 36 | 37 | GLOBAL 38 | ------ 39 | 40 | The `GLOBAL` dict is used for configuration that is likely shared with other projects. Like 41 | the root location of a cache directory. 42 | 43 | CLI_CONFIG 44 | ---------- 45 | 46 | The `CLI_CONFIG` dict is used for configuration data data specific to the command line. 47 | It is only used to set positional arguments, things that define the structure of how 48 | the CLI should be processed. 49 | 50 | When using `CLI_CONFIG` the options should still be defined in the `CONFIG` section. The 51 | top level key in the `CLI_CONFIG` will override the `CONFIG` values but having them set 52 | in the `CONFIG` section will allow for the values to be absorbed by plugin systems 53 | that are using your application. 54 | 55 | SUBS 56 | ---- 57 | 58 | The `SUBS` dict compliments the `CLI_CONFIG` dict in specifying what subparsers should be 59 | added to the cli when importing this config as the primary cli interface. 60 | 61 | Usage 62 | ===== 63 | 64 | Now, with the conf.py file in place loading the configuration data up is easier then ever! 65 | Just add this one line to your project: 66 | 67 | .. code-block:: python 68 | 69 | hub.pop.conf.integrate() 70 | 71 | The conf system will get loaded for you and hub.OPT will be populated with namespaced configuration 72 | data as defined in the configuration dicts. 73 | 74 | Multiple Projects 75 | ----------------- 76 | 77 | If multiple projects are used the first argument is a list of projects. The `CLI_CONFIG` 78 | will only be taken from one project. So when using multiple projects the `cli` option can be 79 | passed to specify which project to pull the CLI_CONFIG from: 80 | 81 | .. code-block:: python 82 | 83 | hub.pop.conf.integrate(['act', 'grains', 'rem'], cli='rem') 84 | 85 | Override Usage 86 | ============== 87 | 88 | Sometimes configuration options collide. Since the integrate system is used to dynamically merge 89 | multiple projects' configuration options we need to be able to handle these collisions. This 90 | is where the `override` setting comes into play. 91 | 92 | If there is a conflict in the configs, then the `conf` system will throw an exception listing 93 | the colliding options. These options will be shown as the package name followed by the config key. 94 | So if the project name passed into integrate is `poppy` and the configuration key is test, then 95 | the collision will be on key `poppy.test`. To overcome the collision we need to create a new 96 | key and potentially new options for the command. 97 | 98 | To use the override just define the override dict and pass it into `pop.conf.integrate`: 99 | 100 | .. code-block:: python 101 | 102 | override = {'poppy.test': {'key': 'test2', 'options': ['--test2', '-T']}} 103 | hub.pop.conf.integrate('poppy', override) 104 | 105 | Now the collisions are explicitly re-routed and fixed! 106 | 107 | Using The Roots System 108 | ====================== 109 | 110 | In many applications directories need to be created and verified. Often the directories also 111 | need to be set up for a specific user if not run with admin privileges. This is where the 112 | `roots` system comes into place. 113 | 114 | If you have set up any option(s) that ends in `_dir`, then you can have `conf.integrate` do 115 | some work for you! By setting the `roots` option `conf` will create the directory if it does 116 | not already exist and it will change the paths to those directories to be in a hidden directory 117 | in the user's home directory if not running as the root user. 118 | -------------------------------------------------------------------------------- /docs/source/topics/contracts.rst: -------------------------------------------------------------------------------- 1 | ========= 2 | Contracts 3 | ========= 4 | 5 | One of the core components of `POP` is the contract system. When everything 6 | is a plugin then these plugins need to be able to enforce the interface 7 | they belong to. `POP` provides contracts to enable these interfaces. 8 | Contracts can be used to define plugin interfaces, ensure that functions 9 | that are needed are available, and that needed functions follow argument 10 | signatures. Contracts can also define transparent wrappers, running 11 | pre or post functions around function calls, or replacing the actual 12 | function call. 13 | 14 | Signature System 15 | ================ 16 | 17 | The signature system allows for function signatures to be enforced in plugins 18 | via contracts. This means that a plugin the implements a contract will be 19 | forced to implement the named functions and follow the restrictions inside 20 | defined by the function signature. 21 | 22 | In a nutshell the signature system allows for contracts to define the 23 | implementation interface for plugins. 24 | 25 | If this file is defined as the contract `contracts/red.py` 26 | 27 | .. code-block:: python 28 | 29 | def sig_foo(hub, a, b: str): 30 | pass 31 | 32 | Then the file `red.py` is forced to create a function with a compatible signature 33 | 34 | .. code-block:: python 35 | 36 | def foo(hub, a, b:str): 37 | return a + b 38 | 39 | The signature system also verifies `*args` and `**kwargs` conditions, parameter 40 | names, types and annotations. 41 | 42 | So this `sig` contract: 43 | 44 | .. code-block:: python 45 | 46 | def sig_foo(hub, a, b, **args): 47 | pass 48 | 49 | Will work with this function: 50 | 51 | .. code-block:: python 52 | 53 | def foo(hub, a, b, c, d): 54 | return a + b + c + d 55 | 56 | Because the contract allows arbitrary `*args`, but in this example the contract 57 | will mandate that `a` and `b` are defined. 58 | 59 | Similarly `**kwargs` will pass through: 60 | 61 | .. code-block:: python 62 | 63 | def sig_foo(hub, a, b, c=4, **kwargs): 64 | pass 65 | 66 | Which allows a function like this: 67 | 68 | .. code-block:: python 69 | 70 | def foo(hub, a, b, c, d=5, e='foo'): 71 | return a 72 | 73 | Since the `sig` function in the contract allows `**kwargs`, the function can 74 | have `**kwargs`. 75 | 76 | Similarly, if the `sig` function does not have `**kwargs` then additional 77 | parameters are NOT allowed beyond what is defined in the `sig`. 78 | 79 | Wrappers 80 | ======== 81 | 82 | Contracts allow for functions to be wrapped. This allows for external 83 | validators to be enforced, for parameters to be validated, and data to 84 | me manipulated. 85 | 86 | The available wrappers are `pre`, `post`, and `call`. When these are included 87 | in a contract they will be called when the function is called. 88 | 89 | Module and Function Wrappers 90 | ---------------------------- 91 | 92 | When creating wrappers, they can be applied to all functions in a module 93 | or they can be applied to specific functions. To make a module level 94 | wrapper, just make a single function with the wrapper type name: 95 | 96 | .. code-block:: python 97 | 98 | def pre(hub, ctx): 99 | pass 100 | 101 | This function will now be executed for every function called in the 102 | corresponding plugin. 103 | 104 | A wrapper can also be made to be specific to a function by using the 105 | same function name, just prepend the function name with the name of the 106 | wrapper to use, as in `pre_`: 107 | 108 | .. code-block:: python 109 | 110 | def pre_foo(hub, ctx): 111 | pass 112 | 113 | Pre 114 | ---- 115 | 116 | When using `pre` the contract function will be executed before the module 117 | function. The `pre` function receives the hub and a `ctx` object. The `ctx` 118 | object is used to contain the context of the call. This `ctx` object has 119 | access to `args` and `kwargs` for the function call: 120 | 121 | .. code-block:: python 122 | 123 | def pre(hub, ctx): 124 | if len(ctx.args) > 1: 125 | raise ValueError('No can haz args!') 126 | if ctx.kwargs: 127 | raise ValueError('No can haz kwargs!') 128 | 129 | Call 130 | ---- 131 | 132 | The `call` wrapper can be used to replace the actual execution of the 133 | function. When call is used the underlying function is not called, it 134 | needs to be called inside of the call function. This function can be useful 135 | when you want to have conditions around weather to call a function, or to 136 | have a full context around the wrapping of the function. The function object 137 | is included in the `ctx`: 138 | 139 | .. code-block:: python 140 | 141 | def call(hub, ctx): 142 | return ctx.func(*ctx.args, **ctx.kwargs) 143 | 144 | Post 145 | ---- 146 | 147 | The `post` wrapper allows for the return data from the function to be handled. 148 | This can be useful if your function(s) need to modify or validate return data. 149 | The return data from the `post` function is the return data send back when the 150 | function is called. 151 | 152 | .. code-block:: python 153 | 154 | def post(hub, ctx): 155 | ret = ctx.ret 156 | if isinstance(ret, list): 157 | ret.append('post called') 158 | elif isinstance(ret, dict): 159 | ret['post'] = 'called' 160 | return ret 161 | 162 | Using the contracts Directory 163 | ============================= 164 | 165 | Contracts can be added to a `sub` by just adding a subdirectory called `contracts` 166 | into the directory containing the `sub` plugins. So if you have a sub called 167 | `rpc` then the contracts directory would be `rpc/contracts`. 168 | 169 | Inside the `contracts` directory the name of the modules will map to the name of 170 | the plugin in the corresponding `sub`. The `virtualname` of the contract module is 171 | also honored and will override the file name in the same way that `virtualname` 172 | will override the file name in standard plugin modules. 173 | 174 | This means that if you want a contract for a module called `red` then the file: 175 | `contracts/red.py` will apply for the module `red.py`. Similarly if you want a single 176 | contract to be applied to multiple plugins the implement the `red` interface just call 177 | the contract module `red` and then have the modules that implement the interface take 178 | the `red` `virtualname`. 179 | 180 | Using __contracts__ 181 | =================== 182 | 183 | A plugin can also volunteer itself to take on a specific contract or a list of 184 | contracts. This can be done with the `__contracts__` value at the top of a plugin 185 | module. 186 | 187 | .. code-block:: python 188 | 189 | __contracts__ = ['red', 'blue', 'green'] 190 | 191 | All of the contract wrappers and sigs will be enforced and called. If multiple wrappers 192 | are defined for a given function then they will be called in the order in which they 193 | are defined in the `__contracts__` variable. 194 | 195 | Subsystem Wide Contracts 196 | ======================== 197 | 198 | Sometimes it makes sense to enforce the same contract over an entire subsystem. This 199 | can be useful when the pattern you are using exposes many ways to accomplish the 200 | same task, like many back ends to a database, or many ways to read in different types 201 | of files. 202 | 203 | To make a subsystem wide contract just make an `init.py` file in your `contratcs` 204 | directory. That `init.py` contract will now be applied to all modules in the 205 | subsystem. 206 | -------------------------------------------------------------------------------- /docs/source/topics/dyne_name.rst: -------------------------------------------------------------------------------- 1 | .. _dyne_name: 2 | 3 | ============= 4 | Dynamic Names 5 | ============= 6 | 7 | The Dynamic Names system in `pop` is used to implement part of the app merging system. 8 | When you think of app merging you should think of it as coming from 2 separate directions, 9 | once direction is the ability to merge many apps together into a larger app, this is 10 | called *horizontal app merging*. But the other angle of app merging it to allow for external 11 | applications to extend your own subsystems, this is called *vertical app merging*. 12 | 13 | Think of it this way, you define a system that detects information about a server, but 14 | you don't want to have to build in support for all the specifics that could be discovered 15 | on multiple operating systems and platforms. Instead of trying to maintain support for 16 | 20 operating systems in one application, you can instead make the core of the application 17 | and then set up dynamic names, then you can have separate packages that gather data 18 | for each specific platform but the separate packages dynamically add their own plugins 19 | to your plugin subsystem. 20 | 21 | This is what Dynamic Names allows you to do! Using Dynamic names is very easy, first 22 | just define the `dyne_name` as the only option when you set up your new sub: 23 | 24 | .. code-block:: python 25 | 26 | hub.pop.sub.add(dyne_name='grains') 27 | 28 | Then in your project's *conf.py* file used by the `conf.integrate` system just add another 29 | dict called `DYNE`: 30 | 31 | .. code-block:: python 32 | 33 | DYNE = { 34 | 'grains':[ 35 | 'grains', 36 | ] 37 | } 38 | 39 | Inside your `DYNE` config you specify which `dyne_names` you want to add modules to and 40 | what the module path, relative to your project is. 41 | 42 | Now you can start up a new sub that will gather plugins from multiple systems that add 43 | the dyne_name that you have set up! 44 | -------------------------------------------------------------------------------- /docs/source/topics/glossary.rst: -------------------------------------------------------------------------------- 1 | ================= 2 | Glossary of Terms 3 | ================= 4 | 5 | Learning Plugin Oriented Programming requires understanding a number of terms. This glossary exists 6 | to make that easier. 7 | 8 | `pop`: Used to reference the `pop` Python project. When specifying the `pop` implementation of 9 | Plugin Oriented Programming use an all lowercase `pop` 10 | 11 | `POP`: The Plugin Oriented Programming concept. When specifying the Plugin Oriented Programing paradigm 12 | do so with all caps 13 | 14 | `hub`: The `hub` is the root of the hierarchical namespace. The hub represents both the concept of 15 | the hub in Plugin Oriented Programming and the hub as it is implemented in the `pop` project 16 | 17 | `sub`: The `sub` is the implementation of the Plugin Subsystem concept of Plugin Oriented Programing 18 | inside of `pop` 19 | 20 | `ref`: The `ref` is a string representation of a path that can be found on the `hub`. 21 | 22 | `pattern`: The Plugin Oriented Programming concept which defines how a Plugin Subsystem is 23 | implemented. :ref:`sub_patterns` 24 | 25 | `app-merging`: The ability to dynamically merge seperate POP codebases together. 26 | 27 | `Vertical App Merging`: Extending a single plugin subsystems by defining the sub in multiple codebases. 28 | 29 | `Horizontal App Merging`: Merging multiple subsystems together onto one hub. 30 | 31 | `Dyne Name`: The dynamic name is used to define what plugin sub that is being defined or 32 | extended. Dyne Names are used in Vertical App Merging. 33 | -------------------------------------------------------------------------------- /docs/source/topics/hub_overview.rst: -------------------------------------------------------------------------------- 1 | .. _hub_overview: 2 | 3 | ===================== 4 | Understanding The Hub 5 | ===================== 6 | 7 | The hub is the central feature of Plugin Oriented Programming. I first developed the idea 8 | of the hub when I implemented pop in Julia. I realized that I needed to pass in a namespace 9 | for all of the plugins to exist on. Before that I had been trying to maintain links between 10 | plugin systems on the python module level. Working in Julia forced me to think about the 11 | problem differently. The hub therefore allows for plugin systems to be separated from the 12 | module systems inside of (in this case) Python. 13 | 14 | Once I had a separate namespace to work with it became easy to decouple memory management 15 | as well. I had created a data hierarchy. 16 | 17 | This is a very critical part of POP, memory and variable management is very important to 18 | how software is written and I realized that a new exploration of how to approach the 19 | problem could yield new results. 20 | 21 | Namespaces Beyond Silos 22 | ======================= 23 | 24 | The first thing to think about the hub is that it defines the complete structure of your 25 | application while delivering a new way to think about memory management. Namespaces, beyond 26 | silos. 27 | 28 | In classical Object Oriented Programming we follow a model of variable namespaces where 29 | we isolate memory onto the stack and the heap. The clean, well defined stack of a function, 30 | and the broad mess of data, aptly called the heap. To deal with the open-ended flexibility 31 | of the heap, memory management evolved into an adjacency model. Data that is to be used by 32 | certain functions should be adjacent to said functions inside of class instances. This made 33 | memory management on the heap (arguably) more elegant. You don't need to manage a heap 34 | of variables, but instead a collection of class instances. This has generally been seen as 35 | an improvement, an evolution. 36 | 37 | But this created a reverse issue, data increasingly became isolated to silos, encapsulation 38 | offered control, but introduced over encapsulation. The idea that data became too isolated 39 | and hard to get access to. It is a common issue that data needs to be synced between classes 40 | and instances and that communication between classes and instances can be troublesome. There 41 | now exist many ways to solve this problem, but they often remain messy and convoluted. 42 | 43 | Object Oriented Programming also introduced the concept od public, protected and private 44 | variables and functions. This is an excellent tool for defining how the developer wants 45 | to scope the use of functions, variables and methods. 46 | 47 | POP pushes beyond OOP without sacrificing the benefits. The hub presents a namespace that 48 | can be used to place data onto a world readable medium, solving over encapsulation, while 49 | still communicating to the developer how to interact with data and still allowing data 50 | to be private to functions, modules, classes, and instances. 51 | 52 | Using Hub Namespaces 53 | ==================== 54 | 55 | The hub namespace is made to link to the physical layout of the code. Therefore the location 56 | of data on the hub also defines the location in code of where and how the data is to be used. 57 | Lets start by going over the hub's layout. 58 | 59 | hub 60 | --- 61 | 62 | The hub is the root object, if you decide to place variables on the hub, it is assumed that 63 | these variables are world read/writable. It is rarely, if ever, permissible to place variables 64 | directly on the hub. This is because it dirties the root namespace and creates issues 65 | for app merging because you have a higher likelihood of creating app merging conflicts. 66 | 67 | hub.sub{.sub.sub...} 68 | -------------------- 69 | 70 | Below the hub is where you get your subs. The subs are really the sole intended occupants of 71 | the hub. A sub is a named collection of plugins. In this case, variables that exist on the 72 | sub are intended to be writable only to plugins found within the sub. 73 | 74 | A sub can also have a nested sub. This is a sub within a sub that has its own plugins. 75 | A nested sub is intended to have write access to the parent sub's variables. This 76 | allows for communication dicts, queues, etc., to exist on a higher level sub to facilitate 77 | communication between lower level sub's modules. 78 | 79 | The sub level is a very common level to place variables. The nature of plugins is such that 80 | they need intercommunication, but that communication also needs to be limited! It should 81 | not experience interference from external plugin systems. 82 | 83 | Keep in mind that this is the data from external plugin systems. Many patterns are built 84 | to present external communication interfaces to other subs. This is a perfectly acceptable 85 | reason to have external subs send data into a sub. This is typically done by calling 86 | a function within the sub, so the data is still written from within the sub itself. This 87 | allows for the data input to be controlled and kept clean. 88 | 89 | hub.sub.mod 90 | ----------- 91 | 92 | The module layer allows for data that is isolated to the use of the specific module. This 93 | allows for patterns that call functions in modules repeatedly to persist data. 94 | 95 | Naming Conventions 96 | ================== 97 | 98 | Because the namespaces can collide it makes sense that objects on the hub should follow 99 | namespace rules. Use this document to define access and naming conventions. 100 | 101 | Variable Names 102 | -------------- 103 | 104 | Variable names stored on the hub in any location should always be all caps, following the 105 | Python convention of module level variables: 106 | 107 | hub.sub.INQUE = asyncio.Queue() 108 | 109 | Sub Names 110 | --------- 111 | 112 | The names of subs should always be underscore delimited and lower case: 113 | 114 | hub.sub 115 | 116 | Module Names 117 | ------------ 118 | 119 | The names of modules should always be underscore delimited and lower case: 120 | 121 | hub.sub 122 | 123 | Function Names 124 | -------------- 125 | 126 | The names of functions should always be underscore delimited and lower case: 127 | 128 | hub.sub.mod.function() 129 | 130 | Class Names 131 | ----------- 132 | 133 | Class names should always be CamelCase: 134 | 135 | hub.sub.mod.MyClass() 136 | -------------------------------------------------------------------------------- /docs/source/topics/ideas_that_were_not_used.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | Ideas that Were Not Used 3 | ======================== 4 | 5 | This doc is intended to track ideas that we thought were good at first but later 6 | decided against. This way we can remember why we are not doing specific things and 7 | track the the circumstances were at the time. 8 | 9 | Modification of the Imported Module 10 | =================================== 11 | 12 | When a module is imported Python loads the module object onto `sys.modules`. The 13 | object on the `hub` is not the module but an object filled with refs to classes 14 | functions etc. Early on we decided to make it easy to call functions inside the same 15 | module by taking the wrapped function and then overriding the function on the 16 | module object loaded into `sys.modules`. 17 | We ran into some issues, like the module being imported outside of `pop` and 18 | then getting changed by pop later on. We felt that it would be better to keep 19 | the modules clean. This allows us to have multiple hubs and is less surprising to 20 | users. The downside is that if you call a function locally in a module then 21 | it is not contracted and the hub needs to be passed in manually. -------------------------------------------------------------------------------- /docs/source/topics/learning.rst: -------------------------------------------------------------------------------- 1 | .. _learning_POP: 2 | 3 | ============ 4 | Learning POP 5 | ============ 6 | 7 | Learning POP means thinking about programming differently, like any 8 | programming paradigm. Start by letting go of how you think about programming. 9 | 10 | Rule 1 - Memory Management 11 | ========================== 12 | 13 | Classes are useful, but they are not the end all model for programming. Instead 14 | of thinking about classes as the ultimate container of code and data think about 15 | them as an interface to make types. This is the origin of classes and they are 16 | very good about it. The problem with Classes has to do with data access 17 | and over-encapsulation. When data is tied to a class it becomes isolated, and 18 | so does the functionality. Instances are like little data prisons. Data should 19 | be available to the entire program. 20 | 21 | Woah you will say! Globals are bad! 22 | 23 | You are right! Arbitrary globals are bad! But namespaces are not. The first 24 | thing to accept with POP is the global namespace, the `Hub`. The Hub is the 25 | shared root object in POP. Everything that you think of as being on the 26 | heap, now goes on the Hub. The problem with the heap is that stuff gets lost. 27 | This is where memory leaks come from. Instead of putting data on the heap, 28 | put the data on the Hub. That way you can locate and track all of your data. 29 | 30 | Coming to grips with the Hub is critical, then you can start to think about 31 | your program like the spokes of a great wheel, spokes that hold data and functions. 32 | 33 | So here is Rule 1: 34 | 35 | If you want to use a class ask yourself, "Do I need a type"? 36 | If not, don't make a class! If so, add a type to a plugin, but keep instances 37 | either on the stack in a function or on the Hub. 38 | 39 | Rule 2 - Dealing with Complexity 40 | ================================ 41 | 42 | The main idea of classes is that that they encapsulate functions and data, together! 43 | Classes then can be manipulated, inherited, morphed. These concepts are beautiful, 44 | elegant and useful. But in POP we can begin to break up the concepts of classes. 45 | 46 | In a type, having this connection makes a lot of sense, but when it comes to programming 47 | interfaces it becomes convoluted. We take a beautiful concept and force it to expand 48 | beyond its bounds. This is where software complexity comes from. 49 | 50 | All well designed code works well when it is small, it is traceable, elegant, algorithmic. 51 | The aspects of software development is still clean and tight. We subsequently look at 52 | code that is small and contained to be good, and monolithic, oversized and unwieldy code 53 | to be bad. 54 | 55 | But as complexity grows, we can't have our clean small applications, they grow with 56 | lives of their own into sprawling and often unmaintainable codebases. 57 | 58 | So in POP we break apart code differently, instead of encapsulating code in classes 59 | and instances, we encapsulate functionality inside of plugin subsystems, or Subs. 60 | 61 | Subs allow you to create functional interfaces where plugins can define functionality. 62 | If we look at this from just a functional perspective it falls down. If we look at it 63 | from a Flow Programming perspective it makes more sense. 64 | 65 | Flow Programming defines functions to traverse over data, the nature of the program 66 | transforms with the data that is sent into it. 67 | 68 | Subs merges concepts of OOP and Flow. The Subs get created against data, system data, 69 | configuration data. Instead of creating instances bound to data, you store data with 70 | functions. 71 | 72 | Rule 2 - Complexity is dealt with through breaking apart Classes. Functions work 73 | on their own, morphed with data stored alongside them on the namespace. Instead of 74 | encapsulation, namespace isolation. 75 | 76 | Rule 3 - Subs and Patterns 77 | ========================== 78 | 79 | Now that we have broken apart classes, and introduced Subs, the spokes to our Hub, 80 | we can cover Sub Patterns. 81 | 82 | Plugin Subs allow for the creation of new programming patterns. These patterns make 83 | Subs amazingly flexible. Now the Subs can define programming interfaces that are 84 | as diverse as programming itself. 85 | 86 | Patterns include, RPC Interfaces, Data collection, Scanners, Event gatherers, 87 | iterative processing, and simple functionality buckets and many more, some defined, 88 | most yet to be discovered. 89 | 90 | Rule 3 - Think of programming through exposing patterns. Patterns define the nature 91 | of code and allow for Subs to be easy to grok, manipulate and expand. Patterns 92 | allow for code to follow the rule of functional isolation, extending the concept 93 | of namespace isolation. 94 | 95 | Rule 4 - Public Vs Private 96 | ========================== 97 | 98 | Since all of this functionality and data exist on the Hub, any other user of the 99 | application can access it. This makes the public vs private question very serious! 100 | When making functions, make sure they follow the chosen pattern, but maintain 101 | all public code as if it is being used by others. 102 | 103 | This is a good thing! Writing code that follows the rules of libraries has always 104 | been good! So often we make private methods and data out of laziness rather than 105 | purpose. When you choose to make something private, it should be logically only 106 | applicable to the namespace that it is private to. Otherwise it should be public, 107 | and developed like public code. 108 | 109 | Rule 4 - Make functions public, unless they are truly private, and maintain them 110 | as such. If your subs follow patterns, then they can be easily re-used, reusable 111 | code should expose simple interfaces and follow good library development practices. 112 | 113 | Rule 5 - App Merging 114 | ==================== 115 | 116 | Software is easier to develop and manage when it is composed of many smaller 117 | applications. But software is easier to distribute and use when it is a single large 118 | application. Take a Linux Distro for example, they are made of thousands of small 119 | software packages, but these packages would be unusable alone, they need to be 120 | glued together and distributed. 121 | 122 | Plugin Oriented Programming is all about tearing down the walls between apps and 123 | libs. It is designed to make apps mergeable, but also standalone and useful. 124 | 125 | This solves the problem of communicating large codebases! Now small codebases can 126 | be created and iterated on quickly, but still merged into a larger whole. 127 | 128 | Rule 5 - Split your code into many smaller projects and use app merging inside of 129 | POP to bring the small projects together into a larger merged project. 130 | -------------------------------------------------------------------------------- /docs/source/topics/pop.rst: -------------------------------------------------------------------------------- 1 | =========================== 2 | Plugin Oriented Programming 3 | =========================== 4 | 5 | Plugin Oriented Programming, or POP, is a new programming paradigm targeting a number of 6 | modern software engineering problems. POP presents a new way of looking at programming, 7 | merging concepts from OOP, Functional, DataFlow, Configuration Management concepts and 8 | more. 9 | 10 | POP puts weight on plugins, namespaces, modularity and isolation of development and testing. 11 | 12 | The Components of POP 13 | ===================== 14 | 15 | Everything is a Plugin 16 | ---------------------- 17 | 18 | When developing large codebases it is extremely common to need to move code into a modular 19 | design over time. This typically means that the application needs to be overhauled after 20 | the first few years of development to be modular and pluggable. So why not just start with 21 | a plugin design to begin with? 22 | 23 | POP is turtles all the way down. Using the POP design forces you to make your code pluggable. 24 | The unique design allows for all of the code components to be easily replaced so long 25 | as the exposed interfaces remain the same. 26 | 27 | When everything is a plugin from the beginning it becomes easy to replace individual 28 | components and compartments of an application. When the application gets split up cleanly 29 | into plugin subsystems then entire plugin subsystems can be replaced or updated 30 | wholesale. When needed, instead of being stuck with code not intended to scale to the 31 | scope a project so often finds itself in. 32 | 33 | Global Namespace - The Hub 34 | -------------------------- 35 | 36 | The first thing you will notice in POP is the `hub` object. The hub is passed automatically 37 | as the first argument to all functions, much like `self` inside a python class. This hub 38 | can be infinitely extended to include new plugin subsystems as well as namespaces and 39 | variables. 40 | 41 | The hub is critical as it serves as a vehicle for accessing all of the plugin subsystems 42 | that are made available to the application. 43 | 44 | Plugin Subsystems 45 | ----------------- 46 | 47 | Plugin Subsystems, which are simply referred to as `subs` allow for new plugin systems to be added 48 | to the hub. This makes merging codebases easy. Other applications can be merged together by 49 | including their plugin subsystems. For instance an application can be written that creates 50 | an authentication system, and then the entire structure of that application can then be 51 | added to another application as a subsystem. 52 | 53 | Contracts 54 | --------- 55 | 56 | Plugin systems need to be able to support interfaces. In fact interfaces as a programming 57 | construct become more important than ever. Instead of having the overhead of class inheritance, 58 | contracts can server and transparent interfaces to enforce and guide plugin developers. 59 | 60 | Contracts get executed transparently, this allows for a developer to simply implement 61 | the interface without needing to also inherit an Abstract Base Class. Beyond this, 62 | contracts allow for pre and post hooks to be applied to functions in contracted plugins. 63 | 64 | This allows for things like interface input validation. Built in pre and post hooks. As 65 | well as load time enforcement of the validity of the interface. 66 | 67 | App Merging 68 | ----------- 69 | 70 | App merging allows for full applications to be merged into each other. This means that 71 | a large application can be developed as many small applications that can be merged together. 72 | 73 | Since any single application is comprised of `subs` it becomes easy to merge multiple subs 74 | together from multiple apps into a new larger app. 75 | 76 | Horizontal App Merging 77 | ~~~~~~~~~~~~~~~~~~~~~~ 78 | 79 | Horizontal app merging means that you take the `subs` from multiple applications and merge 80 | them together into a new larger application. For instance lets say that you have a process 81 | that exposes an rpc interface, but you want to add system data gathering to your rpc system. 82 | Just use horizontal app merging to bring in the functionality from another application. 83 | In a nutshell, Horizontal App Merging allows for functionality from multiple apps to 84 | be mereged together by adding more `subs` onto your `hub`. 85 | 86 | Vertical App Merging 87 | ~~~~~~~~~~~~~~~~~~~~ 88 | 89 | Vertical App merging happens when you have a `sub` but you want to extend that `sub` to 90 | support more plugins that folow your `sub`'s pattern and contracts. This can be 91 | useful when you want to add additional database support, or support interacting with more 92 | Operating Systems. Or when you want to add support for interacting with more apis. 93 | 94 | Challenges to Face 95 | ================== 96 | 97 | POP is designed to address a number of challenges in modern software engineering. These challenges 98 | include, but are not limited to... 99 | 100 | Distributed Development 101 | ----------------------- 102 | 103 | One of the best aspects of modern open source software is the distributed nature of development. 104 | Having many people working on a large, important piece of software is a serious challenge, not 105 | only in co-ordination of development, but also in the maintainability of said software stack. 106 | As many developers come together to push forward a project many problems also occur. Projects 107 | originally designed to be small and lean become bloated and feature laden. Systems that once 108 | could be tested in seconds grow to tests that take hours. Feature creep takes over and the 109 | beautiful concepts of clean software design get swept up in the excitement of a large scale 110 | project. 111 | 112 | POP is designed to make software development compartmentalizable. This idea is that entire 113 | functional components of a larger applications can be developed reliably in isolation. 114 | If the functional components of an application can be compartmentalized, that means they 115 | can be tested standalone and then merged back into a greater whole. This takes the main 116 | benefits of a microservices design and applies to to a cleanly ordered application 117 | development model. 118 | 119 | Concurrency Models 120 | ------------------ 121 | 122 | Over the last few years new concepts about concurrency have emerged, primarily in the 123 | sense of modern co-routines. Coroutines allow for concurrent processing without threading 124 | but they also impose unique challenges. The POP model presents a way for coroutines to 125 | cross communicate in clean and reliable ways, and allow for coroutines to be run without 126 | the headaches that so often occur using things like callbacks and having multiple 127 | coroutine streams running. 128 | 129 | This is accomplished using the `Hub`. The hub in POP allows for clean, globally accessible memory 130 | to be accessed in a safe, namespaced way, while still honoring valuable OOP concepts of 131 | encapsulation, without over encapsulating data that is so often useful in the broader application. 132 | This namespaced approach to development makes data sharing between coroutines safe, 133 | easy and reliable. 134 | -------------------------------------------------------------------------------- /docs/source/topics/proc.rst: -------------------------------------------------------------------------------- 1 | ======================= 2 | Proc Process Management 3 | ======================= 4 | 5 | The proc system is unlike other process management systems. It is inspired by 6 | the process systems found in Julia and allows for functions to be completely 7 | farmed out to async executions. The proc processes are not forked but are 8 | fresh python executions. These new processes will execute many async python 9 | functions simultaneously. 10 | 11 | When sending a fresh command into a proc process just send in the pop reference 12 | and the kwargs, then the function will be started up and the return will be 13 | sent back out to the calling function. 14 | 15 | Usage 16 | ===== 17 | 18 | Start by adding the proc subsystem to your hub: 19 | 20 | .. code-block:: python 21 | 22 | hub.pop.sub.add('pop.mods.proc') 23 | 24 | Now the proc subsystem is available. Create a new process pool: 25 | 26 | .. code-block:: python 27 | 28 | await hub.proc.init.pool(3, 'Workers', sock_dir='/tmp') 29 | 30 | You now have a worker pool named `Workers` with 3 processes and the unix sockets 31 | for communication will be placed in the `/tmp` directory. 32 | 33 | Before sending function calls to the pool add a new subsystem to the workers. 34 | Remember that these processes are not forks, they need to have the subsystems 35 | loaded! 36 | 37 | Just call `hub.proc.run.add_sub` with the name of the pool as the first argument 38 | followed by the arguments to `pop.sub.add`. Lets add the actor system so we 39 | can get a nice battery of functions to call: 40 | 41 | .. code-block:: python 42 | 43 | await hub.proc.run.add_sub('Workers', 'act.mods.actor') 44 | 45 | The pool can now be sent functions to be run and awaited for. The functions 46 | can be either async functions or just plain python functions. But the real power 47 | of the system is found in sending in async functions. 48 | 49 | .. code-block:: python 50 | 51 | ret = await hub.proc.run.func('Workers', 'act.test.ping') 52 | 53 | Any args or kwargs passed after the first 2 arguments to hub.proc.run.func will be 54 | passed to the called function. 55 | 56 | Generators 57 | ========== 58 | 59 | Generators and async generators are also supported, but you need to call a different 60 | function with `proc.run` to return a generator. The function to call is `proc.run.gen`. 61 | 62 | Calling this function will always return an async generator, even if the function 63 | called in the proc process is a classic generator, so remember to `async for`, not 64 | just `for`: 65 | 66 | .. code-block:: python 67 | 68 | async for ind in hub.proc.run.gen('Workers', 'act.test.iterate'): 69 | print(ind) 70 | 71 | Tracking Calls 72 | ============== 73 | 74 | Lets say you want the same worker function to be called repeatedly, perhaps 75 | you have a long running async task running that you want to communicate with 76 | by calling more functions. Proc can return the staged coroutine and index of 77 | the intended process to run on! 78 | 79 | .. code-block:: python 80 | 81 | ind, coro = await hub.proc.run.track_func('Workers', 'act.test.ping') 82 | await coro 83 | 84 | Now you can send another function in that you know will be run on the same 85 | process as the previous call: 86 | 87 | .. code-block:: python 88 | 89 | ret = await hub.proc.run.ind_func('Workers', 1, 'act.test.ping') 90 | 91 | 92 | Async Callback Server 93 | ===================== 94 | 95 | Sometimes it may be required to call a function that will return multiple times. 96 | This can be done using a callback function. 97 | -------------------------------------------------------------------------------- /docs/source/topics/story.rst: -------------------------------------------------------------------------------- 1 | .. _story_of_pop: 2 | 3 | ============== 4 | Origins of POP 5 | ============== 6 | 7 | I (Thomas Hatch) am the creator of a major systems management platform called Salt and 8 | the founder and CTO of SaltStack. As I developed Salt I found that a plugin approach 9 | suited our needs so well that I became increasingly enamored with plugins. 10 | 11 | Salt was originally developed with and for plugins, but only in isolated areas. But 12 | over time I added more and more plugin subsystems. As things moved forward I discovered 13 | that major components of Salt we being defined entirely within plugin systems. It 14 | also became a major desire to make more systems within Salt pluggable. We found that 15 | this made Salt more approachable by developers and amazingly extensible. But we also 16 | found a number of shortcomings in the design. When we have plugin systems we wanted 17 | to reach out to those plugins more and more from other parts of the code. We also 18 | found that we were dealing with such vast quantities of plugins that maintainability 19 | became an issue. 20 | 21 | In response to this I started to develop the POP concept. I did not have a lot of spare 22 | time, so it took me a few years, but I was able to find elegant solutions to many 23 | of these problems. Over time I became tied to the idea of POP. Over and over again I found 24 | myself rapidly developing software platforms using POP as an amazing shortcut. 25 | 26 | Subsequently I was able to find shortcomings in my designs and I repeatedly rewrote 27 | the POP platform. The platform you now see is actually the fifth iteration and something 28 | that I feel is finally ready to be shared with the world. 29 | 30 | I hope that this iteration will allow people to make the most of the POP paradigm and 31 | to be able to help fill out how to use POP to solve more software engineering problems 32 | that we face today. 33 | 34 | Software today has evolved into an incredibly distributed affair, with global teams 35 | often from many companies with innumerable goals and objectives. The POP system is 36 | an attempt to find a better way to help these teams collaborate and interface while 37 | still enabling large scale development. 38 | -------------------------------------------------------------------------------- /docs/source/topics/subs_overview.rst: -------------------------------------------------------------------------------- 1 | .. _subs_overview: 2 | 3 | ========================== 4 | Plugin Subsystems Overview 5 | ========================== 6 | 7 | Plugin subsystems, or `subs`, are the main container type used inside of `pop`. 8 | These `subs` contains the collections of plugins, patterns, contracts, and interfaces 9 | that drive your application. Fortunately adding `subs` to your `hub` is easy to do and 10 | adding plugins to your `subs` is equally as easy. All of the namespaces, tricky plugin 11 | loading, and tracking are all taken care of in `pop`. 12 | 13 | When you add a `sub` you have many options. Most of the demo subs you see will be 14 | very simple, just a call to `hub.pop.sub.add` with only the pypath variable 15 | specified. This is all you need in most cases! But you can do much more powerful things 16 | when loading up a new `sub`! 17 | 18 | The init.__init__ Function 19 | ========================== 20 | 21 | The first thing to be aware of in `pop` is the `init.__init__` function. When you make a new `sub` 22 | the *init.py* file is treated as the initializer, or pattern definer of the `sub`. This file 23 | does not need to exist to have a sub, but it exposes extra functionality. The main thing to be 24 | aware of is that if the `__init__` function is defined inside the *init.py* file then it will 25 | be executed when the subsystem is loaded. The first argument passed to the `__init__` function 26 | is, as usual, the hub, and the sub object that the *init.py* has been loaded onto is available. 27 | This makes it easy to initialize any data structures that might be needed on the `sub`. 28 | 29 | Directories and How to Find Them 30 | ================================ 31 | 32 | Plugin loading is all based on the directories that contain the *.py* files that constitute 33 | plugins. When the `pypath` argument is passed in python imports that path, then it derives 34 | what directory that path is and adds it to the directories to be scanned. The directories 35 | used by the `sub` can be loaded via a number of options: 36 | 37 | pypath: A string or list of python imports that import a python package containing plugins 38 | 39 | static: A string or list of strings that are directory paths containing plugins. 40 | 41 | contracts_pypath: A string or list of strings that import a python package containing plugins to load contracts 42 | 43 | contracts_static: A string or list of things that are directory paths containing plugins to load contracts 44 | 45 | Dynamic Name 46 | ============ 47 | 48 | The Dynamic Name function is amazingly powerful. It allows you to specify a dynamic loader name 49 | that pop will detect in your Python path and auto load extra plugins from external Python 50 | packages that have defined them. This is an amazing way to dynamically make your plugin 51 | subsystem even more pluggable by allowing external applications to extend your system. 52 | 53 | The Dynamic Name system is used by adding the option `dyne_name`. It is the only required 54 | option when enabling dynamic name, but it also requires that your application adds the 55 | `DYNE` flag to the conf.py file in the root of your project. 56 | 57 | dyne_name: A string which defines the name of the subsystem, and how to map it using the 58 | Dynamic Name system 59 | 60 | For more information on Dynamic Names please see the doc outlining how the Dynamic Names system 61 | works and how to use it: :ref:`dyne_name` 62 | 63 | Omitting Components From the Sub 64 | ================================ 65 | 66 | By default, when modules are loaded, they omit objects that start with an underscore. This is set 67 | to allow for objects to be kept private to the module and not expose them. The character used 68 | to determine if the object should be omitted can be changed, or it can be set as an endwith char: 69 | 70 | omit_start: The char to look for at the start of all objects to determine if it should be omitted, defaults to '_' 71 | 72 | omit_end: The char to look for at the end of all objects to determine if it should be omitted, disabled by default 73 | 74 | omit_func: Set to True to omit all functions in the sub 75 | 76 | omit_class: Set to True to omit all classes in the sub 77 | 78 | omit_vars: Set to True to omit all vars from a sub 79 | 80 | If you choose to change any of these values in your default settings for your `sub` it should be heavily 81 | documented, as it will really confuse users of your sub. It is strongly discouraged!! 82 | 83 | Stopping on Load Failures 84 | ========================= 85 | 86 | It can be good to set the sub loading to traceback if a plugin fails to load. Because plugin 87 | interfaces allow end users to add plugins and potentially dirty up the code, by default 88 | if a plugin fails to load it does not stop the sub from loading. 89 | If you do want the sub to traceback set: 90 | 91 | stop_on_failures: Set to True to make the sub traceback on failures to load plugins propagate up 92 | 93 | Virtual Execution 94 | ================= 95 | 96 | When modules are loaded they execute the `__virtual__` function. The `__virtual__` function 97 | can be disabled for a sub when it is loaded. This is typically only used for debugging. 98 | 99 | Modify the Initializer 100 | ====================== 101 | 102 | By default the `__init__` function is run when the sub loads. This can be disabled by setting 103 | the `init` value to False: 104 | 105 | init: Set to False to disable running the `__init__` functions for all modules 106 | 107 | Multiple Python Module Objects 108 | ============================== 109 | 110 | When plugins are loaded they are imported into the python module tracking system in a specific 111 | module path. If you want to be able to load the plugins multiple times and have them exist 112 | in multiple namespaces then you can via `mod_basename`. You only need to do this if you are 113 | loading persisted data onto the module level. If you are doing this then move your data 114 | onto the `hub`: 115 | 116 | mod_basename: Pass a string to specify the Python sys.modules namespace to load the module onto 117 | -------------------------------------------------------------------------------- /pop/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ''' 3 | The pop top level module 4 | ''' 5 | 6 | # Import python libs 7 | import os 8 | 9 | INSTALL_DIR = os.path.dirname(os.path.realpath(__file__)) 10 | 11 | 12 | # ----- Detect System Encoding --------------------------------------------------------------------------------------> 13 | # This will install a globally available variable which will hold the detected encoding 14 | def __define_global_system_encoding_variable__(): 15 | import sys 16 | # sys.stdin.encoding is the most trustworthy source of the system encoding, though, if 17 | # pop is being imported after being daemonized, this information is lost and reset to None 18 | encoding = None 19 | 20 | if not sys.platform.startswith('win') and sys.stdin is not None: 21 | # On linux we can rely on sys.stdin for the encoding since it 22 | # most commonly matches the filesystem encoding. This however 23 | # does not apply to windows 24 | encoding = sys.stdin.encoding 25 | 26 | if not encoding: 27 | # If the system is properly configured this should return a valid 28 | # encoding. MS Windows has problems with this and reports the wrong 29 | # encoding 30 | import locale 31 | try: 32 | encoding = locale.getdefaultlocale()[-1] 33 | except ValueError: 34 | # A bad locale setting was most likely found: 35 | # https://github.com/saltstack/salt/issues/26063 36 | pass 37 | 38 | # This is now garbage collectible 39 | del locale 40 | 41 | if not encoding: 42 | if sys.platform.startswith('darwin'): 43 | # Mac OS X uses UTF-8 44 | encoding = 'utf-8' 45 | elif sys.platform.startswith('win'): 46 | # Windows uses a configurable encoding; on Windows, Python uses the name “mbcs” 47 | # to refer to whatever the currently configured encoding is. 48 | encoding = 'mbcs' 49 | else: 50 | # This is most likely ascii which is not the best but we were 51 | # unable to find a better encoding. If this fails, we fall all 52 | # the way back to ascii 53 | # On linux default to ascii as a last resort 54 | encoding = sys.getdefaultencoding() or 'ascii' 55 | 56 | try: 57 | # Return the detected encoding 58 | return encoding 59 | finally: 60 | # This is now garbage collectible 61 | del sys 62 | del encoding 63 | 64 | 65 | PACK_SYSTEM_ENCODING = __define_global_system_encoding_variable__() 66 | 67 | # This is now garbage collectible 68 | del __define_global_system_encoding_variable__ 69 | # <---- Detect System Encoding --------------------------------------------------------------------------------------- 70 | -------------------------------------------------------------------------------- /pop/contract.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ''' 3 | Contracts to enforce loader objects 4 | ''' 5 | 6 | # Import python libs 7 | import inspect 8 | import os 9 | from collections import namedtuple 10 | 11 | # Import pop libs 12 | import pop.exc 13 | import pop.verify 14 | 15 | 16 | class ContractedContext(namedtuple('ContractedContext', ('func', 'args', 'kwargs', 'signature', 'ret', 'cache'))): 17 | ''' 18 | Contracted function calling context 19 | ''' 20 | def __new__(cls, func, args, kwargs, signature, ret=None, cache=None): # pylint: disable=too-many-arguments 21 | if cache is None: 22 | cache = {} 23 | return super(ContractedContext, cls).__new__(cls, func, list(args), kwargs, signature, ret, cache) 24 | 25 | def get_argument(self, name): 26 | ''' 27 | Return the value corresponding to a function argument after binding the contract context 28 | argument and keyword arguments to the function signature. 29 | ''' 30 | return self.get_arguments()[name] 31 | 32 | def get_arguments(self): 33 | ''' 34 | Return a dictionary of all arguments that will be passed to the function and their 35 | values, including default arguments. 36 | ''' 37 | if '__bound_signature__' not in self.cache: 38 | try: 39 | self.cache['__bound_signature__'] = self.signature.bind(*self.args, **self.kwargs) 40 | except TypeError as e: 41 | for frame in inspect.trace(): 42 | if frame.function == 'bind' and frame.filename.endswith(os.sep+'inspect.py'): 43 | raise pop.exc.BindError(e) 44 | raise 45 | # Apply any default values from the signature 46 | self.cache['__bound_signature__'].apply_defaults() 47 | return self.cache['__bound_signature__'].arguments 48 | 49 | 50 | def load_contract(contracts, default_contracts, mod, name): 51 | ''' 52 | return a Contract object loaded up 53 | ''' 54 | raws = [] 55 | if not contracts: 56 | return raws 57 | loaded_contracts = [] 58 | if getattr(contracts, name): 59 | loaded_contracts.append(name) 60 | raws.append(getattr(contracts, name)) 61 | if getattr(contracts, 'init'): 62 | loaded_contracts.append('init') 63 | raws.append(getattr(contracts, 'init')) 64 | if default_contracts: 65 | for contract in default_contracts: 66 | if contract in loaded_contracts: 67 | continue 68 | loaded_contracts.append(contract) 69 | raws.append(getattr(contracts, contract)) 70 | if hasattr(mod, '__contracts__'): 71 | cnames = getattr(mod, '__contracts__') 72 | if not isinstance(cnames, (list, tuple)): 73 | cnames = cnames.split(',') 74 | for cname in cnames: 75 | if cname in contracts: 76 | if cname in loaded_contracts: 77 | continue 78 | loaded_contracts.append(cname) 79 | raws.append(getattr(contracts, cname)) 80 | return raws 81 | 82 | 83 | class Wrapper: # pylint: disable=too-few-public-methods 84 | def __init__(self, func, ref, name): 85 | self.__dict__.update(getattr(func, '__dict__', {})) # do this first so we later overwrite any conflicts 86 | self.func = func 87 | self.ref = ref 88 | self.name = name 89 | self.signature = inspect.signature(self.func) 90 | self._sig_errors = [] 91 | 92 | def __call__(self, *args, **kwargs): 93 | self.func(*args, **kwargs) 94 | 95 | def __repr__(self): 96 | return '<{} func={}.{}>'.format(self.__class__.__name__, self.func.__module__, self.name) 97 | 98 | 99 | class Contracted(Wrapper): # pylint: disable=too-few-public-methods 100 | ''' 101 | This class wraps functions that have a contract associated with them 102 | and executes the contract routines 103 | ''' 104 | def __init__(self, hub, contracts, func, ref, name): 105 | super().__init__(func, ref, name) 106 | self.hub = hub 107 | self.contracts = contracts if contracts else [] 108 | self._load_contracts() 109 | 110 | def _get_contracts_by_type(self, contract_type='pre'): 111 | matches = [] 112 | fn_contract_name = '{}_{}'.format(contract_type, self.name) 113 | for contract in self.contracts: 114 | if hasattr(contract, fn_contract_name): 115 | matches.append(getattr(contract, fn_contract_name)) 116 | if hasattr(contract, contract_type): 117 | matches.append(getattr(contract, contract_type)) 118 | 119 | return matches 120 | 121 | def _load_contracts(self): 122 | self.contract_functions = {'pre': self._get_contracts_by_type('pre'), 123 | 'call': self._get_contracts_by_type('call')[:1], 124 | 'post': self._get_contracts_by_type('post'), 125 | } 126 | self._has_contracts = sum([len(l) for l in self.contract_functions.values()]) > 0 127 | 128 | def __call__(self, *args, **kwargs): 129 | args = (self.hub,) + args 130 | 131 | if not self._has_contracts: 132 | return self.func(*args, **kwargs) 133 | contract_context = ContractedContext(self.func, args, kwargs, self.signature) 134 | 135 | for fn in self.contract_functions['pre']: 136 | fn(contract_context) 137 | if self.contract_functions['call']: 138 | ret = self.contract_functions['call'][0](contract_context) 139 | else: 140 | ret = self.func(*contract_context.args, **contract_context.kwargs) 141 | for fn in self.contract_functions['post']: 142 | post_ret = fn(contract_context._replace(ret=ret)) 143 | if post_ret is not None: 144 | ret = post_ret 145 | 146 | return ret 147 | -------------------------------------------------------------------------------- /pop/dirs.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ''' 3 | Find directories 4 | ''' 5 | # Import python libs 6 | import os 7 | import sys 8 | import importlib 9 | 10 | 11 | def dir_list(subname, p_name, pypath=None, static=None): 12 | ''' 13 | Return the directories to look for modules in, pypath specifies files 14 | relative to an installed python package, static is for static dirs 15 | ''' 16 | ret = [] 17 | for path in pypath: 18 | mod = importlib.import_module(path) 19 | for m_path in mod.__path__: 20 | # If we are inside of an executable the path will be different 21 | ret.append(m_path) 22 | ret.extend(static) 23 | return ret 24 | 25 | 26 | def inline_dirs(dirs, subdir): 27 | ''' 28 | Look for the named subdir in the list of dirs 29 | ''' 30 | ret = [] 31 | for dir_ in dirs: 32 | check = os.path.join(dir_, subdir) 33 | if os.path.isdir(check): 34 | ret.append(check) 35 | return ret 36 | 37 | 38 | def dynamic_dirs(): 39 | ''' 40 | Iterate over the available python package imports and look for configured 41 | dynamic dirs 42 | ''' 43 | dirs = [] 44 | ret = {} 45 | for dir_ in sys.path: 46 | if not os.path.isdir(dir_): 47 | continue 48 | for sub in os.listdir(dir_): 49 | full = os.path.join(dir_, sub) 50 | if full.endswith('.egg-link'): 51 | with open(full) as rfh: 52 | dirs.append(rfh.read().strip()) 53 | if os.path.isdir(full): 54 | dirs.append(full) 55 | for dir_ in dirs: 56 | conf = os.path.join(dir_, 'conf.py') 57 | context = {} 58 | if not os.path.isfile(conf): 59 | continue 60 | try: 61 | with open(conf) as f: 62 | code = f.read() 63 | if 'DYNE' in code: 64 | exec(code, context) 65 | else: 66 | continue 67 | except Exception: 68 | continue 69 | if 'DYNE' in context: 70 | if not isinstance(context['DYNE'], dict): 71 | continue 72 | for name, paths in context['DYNE'].items(): 73 | if not isinstance(paths, list): 74 | continue 75 | if name not in ret: 76 | ret[name] = [] 77 | for path in paths: 78 | ret[name].append(os.path.join(dir_, path.replace('.', os.sep))) 79 | return ret 80 | -------------------------------------------------------------------------------- /pop/exc.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ''' 3 | Pop related exceptions 4 | ''' 5 | 6 | 7 | class PopBaseException(Exception): 8 | ''' 9 | Base exception where all of Pop's exceptions derive 10 | ''' 11 | 12 | 13 | class PopError(PopBaseException): 14 | ''' 15 | General purpose pack exception to signal an error 16 | ''' 17 | 18 | 19 | class PopLoadError(PopBaseException): 20 | ''' 21 | Exception raised when a pack module fails to load 22 | ''' 23 | 24 | 25 | class PopLookupError(PopBaseException): 26 | ''' 27 | Exception raised when a pack module lookup fails 28 | ''' 29 | 30 | 31 | class ContractModuleException(PopBaseException): 32 | ''' 33 | Exception raised when a function specified in a contract as required 34 | to exist is not found in the loaded module 35 | ''' 36 | 37 | 38 | class ContractFuncException(PopBaseException): 39 | ''' 40 | Exception raised when a function specified in a contract as required 41 | to exist is found on the module but it's not function 42 | ''' 43 | 44 | class ContractSigException(PopBaseException): 45 | ''' 46 | Exception raised when a function signature is not compatible with the 47 | coresponding function signature found in the contract. 48 | ''' 49 | 50 | 51 | class ProcessNotStarted(PopBaseException): 52 | ''' 53 | Exception raised when failing to start a process on the process manager 54 | ''' 55 | 56 | 57 | class BindError(PopBaseException): 58 | ''' 59 | Exception raised when arguments for a function in a ContractedContext cannot be bound 60 | Indicates invalid function arguments. 61 | ''' 62 | -------------------------------------------------------------------------------- /pop/mods/conf/dirs.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Used to take care of the options that end in `_dir`. The assumption is that 3 | `_dir` options need to be treated differently. They need to verified to exist 4 | and they need to be rooted based on the user, root option etc. 5 | ''' 6 | # Import python libs 7 | import os 8 | 9 | 10 | def roots(hub, default_root, f_opts, root_dir): 11 | ''' 12 | Detect the root dir data and apply it 13 | ''' 14 | # TODO: Make this safe for Windows 15 | os_root = '/' 16 | root = os_root 17 | change = False 18 | non_priv = False 19 | if hasattr(os, 'geteuid'): 20 | if not os.geteuid() == 0: 21 | change = True 22 | non_priv = True 23 | if root_dir and root_dir != default_root: 24 | root = root_dir 25 | change = True 26 | if not root.endswith(os.sep): 27 | root = f'{root}{os.sep}' 28 | if change: 29 | for imp in f_opts: 30 | for key in f_opts[imp]: 31 | if key == 'root_dir': 32 | continue 33 | if key.endswith('_dir'): 34 | if non_priv: 35 | root = os.path.join(os.environ['HOME'], f'.{imp}{os.sep}') 36 | if imp in f_opts[imp][key]: 37 | a_len = len(imp) + 1 38 | f_opts[imp][key] = f'{os_root}{f_opts[imp][key][f_opts[imp][key].index(imp)+a_len:]}' 39 | f_opts[imp][key] = f_opts[imp][key].replace( 40 | os_root, root, 1) 41 | 42 | 43 | def verify(hub, opts): 44 | ''' 45 | Verify that the environment and all named directories in the 46 | configuration exist 47 | ''' 48 | for key in opts: 49 | if key == 'root_dir': 50 | continue 51 | if key == 'config_dir': 52 | continue 53 | if key.endswith('_dir'): 54 | if not os.path.isdir(opts[key]): 55 | os.makedirs(opts[key]) 56 | -------------------------------------------------------------------------------- /pop/mods/conf/file_parser.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ''' 3 | Configuration file core loading functions 4 | ''' 5 | 6 | # Import python libs 7 | import os 8 | import glob 9 | import fnmatch 10 | 11 | __virtualname__ = 'file' 12 | __contracts__ = [__virtualname__] 13 | 14 | 15 | def load_file(hub, paths, defaults=None, overrides=None, includes=True): 16 | ''' 17 | Load a single configuration file 18 | ''' 19 | opts = {} 20 | if isinstance(defaults, dict): 21 | opts.update(defaults) 22 | if not isinstance(paths, list): 23 | paths = paths.split(',') 24 | add = [] 25 | for fn_ in paths: 26 | add.extend(glob.glob(fn_)) 27 | paths.extend(add) 28 | for fn_ in paths: 29 | if hub.conf._loader == 'yaml': 30 | opts.update(hub.conf.yaml.load(fn_)) 31 | elif hub.conf._loader == 'json': 32 | opts.update(hub.conf.json.load(fn_)) 33 | elif hub.conf._loader == 'toml': 34 | opts.update(hub.conf.toml.load(fn_)) 35 | if includes: 36 | hub.conf.file.proc_include(opts) 37 | if isinstance(overrides, dict): 38 | opts.update(overrides) 39 | return opts 40 | 41 | 42 | def load_dir(hub, 43 | confdir, 44 | defaults=None, 45 | overrides=None, 46 | includes=True, 47 | recurse=False, 48 | pattern=None): 49 | ''' 50 | Load takes a directory location to scan for configuration files. These 51 | files will be read in. The defaults dict defines what 52 | configuration options should exist if not found in the confdir. Overrides 53 | are configuration options which should be included regardless of whether 54 | those options existed before. If includes is set to True, then the 55 | statements 'include' and 'include_dir' found in either the defaults or 56 | in configuration files. 57 | ''' 58 | opts = {} 59 | if not isinstance(confdir, list): 60 | confdir = confdir.split(',') 61 | confdirs = [] 62 | for dirs in confdir: 63 | if not isinstance(dirs, (list, tuple)): 64 | dirs = [dirs] 65 | for dir_ in dirs: 66 | confdirs.extend(glob.glob(dir_)) 67 | if isinstance(defaults, dict): 68 | opts.update(defaults) 69 | paths = [] 70 | for dir_ in confdirs: 71 | dirpaths = [] 72 | if os.path.isdir(dir_): 73 | if not recurse: 74 | for fn_ in os.listdir(dir_): 75 | path = os.path.join(dir_, fn_) 76 | if os.path.isdir(path): 77 | # Don't process directories 78 | continue 79 | if pattern and not fnmatch.fnmatch(fn_, pattern): 80 | continue 81 | dirpaths.append(path) 82 | else: 83 | for root, dirs, files in os.walk(dir_): 84 | for fn_ in files: 85 | path = os.path.join(root, fn_) 86 | if pattern and not fnmatch.fnmatch(fn_, pattern): 87 | continue 88 | dirpaths.append(path) 89 | 90 | # Sort confdir directory paths like: 91 | # /b.txt 92 | # /c.txt 93 | # /a/x.txt 94 | # /b/x.txt 95 | paths.extend(sorted(dirpaths, key=lambda p: (p.count(os.path.sep), p))) 96 | opts.update(hub.conf.file.load_file(paths, includes)) 97 | if isinstance(overrides, dict): 98 | opts.update(overrides) 99 | return opts 100 | 101 | 102 | def proc_include(hub, opts): 103 | ''' 104 | process include and include_dir 105 | ''' 106 | rec = False 107 | if opts.get('include_dir'): 108 | idir = opts.pop('include_dir') 109 | opts.update(hub.conf.file.load_dir(idir)) 110 | rec = True 111 | if opts.get('include'): 112 | ifn = opts.pop('include') 113 | opts.update(hub.conf.file.load_file(ifn)) 114 | rec = True 115 | if rec: 116 | hub.conf.file.proc_include(opts) 117 | return opts 118 | -------------------------------------------------------------------------------- /pop/mods/conf/init.py: -------------------------------------------------------------------------------- 1 | def __init__(hub): 2 | ''' 3 | Load the subdirs for conf 4 | ''' 5 | hub.pop.sub.load_subdirs(hub.conf) 6 | -------------------------------------------------------------------------------- /pop/mods/conf/integrate.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Integrate is used to pull config data from multiple sources and merge it into 3 | the hub. Once it is merged then when a sub is loaded the respective config data 4 | is loaded into the sub as `OPTS` 5 | ''' 6 | # Take an *args list of modules to import and look for conf.py 7 | # Import conf.py if present 8 | # After gathering all dicts, modify them to merge CLI options 9 | # 10 | # Import python libs 11 | import importlib 12 | import copy 13 | 14 | 15 | def _ex_final(confs, final, override, key_to_ref, ops_to_ref, globe=False): 16 | ''' 17 | Scan the configuration datasets, create the final config 18 | value, and detect collisions 19 | ''' 20 | for arg in confs: 21 | for key in confs[arg]: 22 | ref = f'global.{key}' if globe else f'{arg}.{key}' 23 | if ref in override: 24 | s_key = override[ref]['key'] 25 | s_opts = override[ref]['options'] 26 | else: 27 | s_key = key 28 | s_opts = confs[arg][key].get('options', []) 29 | s_opts.append(f'--{s_key}') 30 | final[s_key] = confs[arg][key] 31 | if s_opts: 32 | final[s_key]['options'] = s_opts 33 | if s_key in key_to_ref: 34 | key_to_ref[s_key].append(ref) 35 | else: 36 | key_to_ref[s_key] = [ref] 37 | for opt in s_opts: 38 | if opt in ops_to_ref: 39 | ops_to_ref[opt].append(ref) 40 | else: 41 | ops_to_ref = [ref] 42 | 43 | 44 | def load( 45 | hub, 46 | imports, 47 | override=None, 48 | cli=None, 49 | roots=False, 50 | loader='json', 51 | logs=True, 52 | version=True, 53 | ): 54 | ''' 55 | This function takes a list of python packages to load and look for 56 | respective configs. The configs are then loaded in a non-collision 57 | way modifying the cli options dynamically. 58 | The args look for the named .conf python module and then 59 | looks for dictionaries named after the following convention: 60 | 61 | override = {'.key': 'key': 'new_key', 'options': ['--option1', '--option2']} 62 | 63 | CONFIG: The main configuration for this package - loads to hub.OPT[''] 64 | GLOBAL: Global configs to be used by other packages - loads to hub.OPT['global] 65 | CLI_CONFIG: Loaded only if this is the only import or if specified in the cli option 66 | SUBS: Used to define the subcommands, only loaded if this is the cli config 67 | ''' 68 | if override is None: 69 | override = {} 70 | if isinstance(imports, str): 71 | if cli is None: 72 | cli = imports 73 | imports = [imports] 74 | primary = imports[0] if cli is None else cli 75 | confs = {} 76 | globe = {} 77 | final = {} 78 | collides = [] 79 | key_to_ref = {} 80 | ops_to_ref = {} 81 | subs = {} 82 | for imp in imports: 83 | cmod = importlib.import_module(f'{imp}.conf') 84 | if hasattr(cmod, 'CONFIG'): 85 | confs[imp] = copy.deepcopy(cmod.CONFIG) 86 | if cli == imp: 87 | if hasattr(cmod, 'CLI_CONFIG'): 88 | confs[imp].update(copy.deepcopy(cmod.CLI_CONFIG)) 89 | if hasattr(cmod, 'SUBS'): 90 | subs = copy.deepcopy(cmod.SUBS) 91 | if hasattr(cmod, 'GLOBAL'): 92 | globe[imp] = copy.deepcopy(cmod.GLOBAL) 93 | if logs: 94 | lconf = hub.conf.log.init.conf(primary) 95 | lconf.update(confs[primary]) 96 | confs[primary] = lconf 97 | if version: 98 | vconf = hub.conf.version.CONFIG 99 | vconf.update(confs[primary]) 100 | confs[primary] = vconf 101 | _ex_final(confs, final, override, key_to_ref, ops_to_ref) 102 | _ex_final(globe, final, override, key_to_ref, ops_to_ref, True) 103 | for opt in ops_to_ref: 104 | g_count = 0 105 | if len(ops_to_ref[opt]) > 1: 106 | collides.append({opt: ops_to_ref[opt]}) 107 | for key in key_to_ref: 108 | col = [] 109 | for ref in key_to_ref[key]: 110 | if not ref.startswith('global.'): 111 | col.append(ref) 112 | if len(col) > 1: 113 | collides.append({key: key_to_ref[key]}) 114 | if collides: 115 | raise KeyError(collides) 116 | opts = hub.conf.reader.read(final, subs, loader=loader) 117 | f_opts = {} # I don't want this to be a defaultdict, 118 | # if someone tries to add a key willy nilly it should fail 119 | for key in opts: 120 | if key == '_subparser_': 121 | f_opts['_subparser_'] = opts['_subparser_'] 122 | continue 123 | for ref in key_to_ref[key]: 124 | imp = ref[:ref.rindex('.')] 125 | if imp not in f_opts: 126 | f_opts[imp] = {} 127 | f_opts[imp][key] = opts[key] 128 | if roots: 129 | root_dir = f_opts.get(cli, {}).get('root_dir') 130 | hub.conf.dirs.roots(final.get('root_dir', {}).get('default', '/'), f_opts, root_dir) 131 | for imp in f_opts: 132 | hub.conf.dirs.verify(f_opts[imp]) 133 | hub.OPT = f_opts 134 | if logs: 135 | log_plugin = hub.OPT[primary].get('log_plugin') 136 | getattr(hub, f'conf.log.{log_plugin}.setup')(hub.OPT[primary]) 137 | if hub.OPT[primary].get('version'): 138 | hub.conf.version.run(primary) 139 | -------------------------------------------------------------------------------- /pop/mods/conf/json_conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ''' 3 | Define the JSON loader interface 4 | ''' 5 | 6 | # Import python libs 7 | import json 8 | 9 | __virtualname__ = 'json' 10 | __contracts__ = [__virtualname__] 11 | 12 | 13 | def __virtual__(hub): 14 | return True 15 | 16 | 17 | def load(hub, path): 18 | ''' 19 | Use json to read in a file 20 | ''' 21 | try: 22 | with open(path, 'r') as fp_: 23 | ret = json.loads(fp_.read()) 24 | return ret 25 | except FileNotFoundError: 26 | return {} 27 | 28 | 29 | def render(hub, val): 30 | ''' 31 | Take the string and render it in json 32 | ''' 33 | return json.loads(val) -------------------------------------------------------------------------------- /pop/mods/conf/log/basic.py: -------------------------------------------------------------------------------- 1 | # Import python libs 2 | import logging 3 | 4 | 5 | def setup(hub, conf): 6 | ''' 7 | Given the configuration data set up the logger 8 | ''' 9 | level = hub.conf.log.LEVELS.get(conf['log_level'], logging.INFO) 10 | root = logging.getLogger('') 11 | root.setLevel(level) 12 | cf = logging.Formatter(fmt=conf['log_fmt_console'], datefmt=conf['log_datefmt']) 13 | ch = logging.StreamHandler() 14 | ch.setLevel(level) 15 | ch.setFormatter(cf) 16 | root.addHandler(ch) 17 | ff = logging.Formatter(fmt=conf['log_fmt_console'], datefmt=conf['log_datefmt']) 18 | fh = logging.FileHandler(conf['log_file']) 19 | fh.setLevel(level) 20 | fh.setFormatter(ff) 21 | root.addHandler(fh) 22 | -------------------------------------------------------------------------------- /pop/mods/conf/log/init.py: -------------------------------------------------------------------------------- 1 | ''' 2 | This sub is used to set up logging for pop projects and injects logging 3 | options into conf making it easy to add robust logging 4 | ''' 5 | # Import python libs 6 | import logging 7 | 8 | 9 | def __init__(hub): 10 | ''' 11 | Set up variables used by the log subsystem 12 | ''' 13 | hub.conf.log.LEVELS = { 14 | 'debug': logging.DEBUG, 15 | 'info': logging.INFO, 16 | 'warning': logging.WARNING, 17 | 'error': logging.ERROR, 18 | 'critical': logging.CRITICAL, 19 | } 20 | 21 | 22 | def conf(hub, name): 23 | ''' 24 | Return the conf dict for logging, this should be merged OVER by the loaded 25 | config dict(s) 26 | ''' 27 | #TODO: Make this more robust to handle more logging interfaces 28 | ldict = { 29 | 'log_file': 30 | { 31 | 'default': f'{name}.log', 32 | 'help': 'The location of the log file', 33 | 'group': 'Logging Options' 34 | }, 35 | 'log_level': 36 | { 37 | 'default': 'info', 38 | 'help': 'Set the log level, either quiet, info, warning, or error', 39 | 'group': 'Logging Options' 40 | }, 41 | 'log_fmt_logfile': 42 | { 43 | 'default': '%(asctime)s,%(msecs)03d [%(name)-17s][%(levelname)-8s] %(message)s', 44 | 'help': 'The format to be given to log file messages', 45 | 'group': 'Logging Options' 46 | }, 47 | 'log_fmt_console': 48 | { 49 | 'default': '[%(levelname)-8s] %(message)s', 50 | 'help': 'The log formatting used in the console', 51 | 'group': 'Logging Options' 52 | }, 53 | 'log_datefmt': 54 | { 55 | 'default': '%H:%M:%S', 56 | 'help': 'The date format to display in the logs', 57 | 'group': 'Logging Options' 58 | }, 59 | 'log_plugin': 60 | { 61 | 'default': 'basic', 62 | 'help': 'The logging plugin to use', 63 | 'group': 'Logging Options' 64 | }, 65 | } 66 | return ldict 67 | -------------------------------------------------------------------------------- /pop/mods/conf/nix_os.py: -------------------------------------------------------------------------------- 1 | ''' 2 | The os module is used to gather configuration options from the OS facility 3 | to send configuration options into applications. In the case of Unix like 4 | systems this translates to the environment variables. On Windows systems 5 | this translates to the registry. 6 | ''' 7 | # Import python libs 8 | import os 9 | 10 | __virtualname__ = 'os' 11 | 12 | 13 | def __virtual__(hub): 14 | ''' 15 | Don't load on Windows, this is for *nix style platforms 16 | ''' 17 | # TODO: detect if windows 18 | return True 19 | 20 | 21 | def gather(hub, defaults): 22 | ''' 23 | Iterate over the default config data and look for os: True/str options. When set 24 | gather the option from environment variables is present 25 | ''' 26 | ret = {} 27 | for key in defaults: 28 | if not 'os' in defaults[key]: 29 | continue 30 | os_var = defaults[key]['os'] 31 | if os_var is True: 32 | os_var = key 33 | os_var = os_var.upper() 34 | if os_var in os.environ: 35 | ret[key] = os.environ[os_var] 36 | return ret 37 | -------------------------------------------------------------------------------- /pop/mods/conf/reader.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ''' 3 | The reader module is used to read the config data. This will read in cli 4 | arguments and merge them with config fie arguments. 5 | ''' 6 | 7 | # Priority order: cli, config, cli_defaults 8 | 9 | __virtualname__ = 'reader' 10 | __contracts__ = [__virtualname__] 11 | 12 | 13 | def _merge_dicts(opts, updates, os_opts, explicit_cli_args): 14 | ''' 15 | recursively merge updates into opts 16 | ''' 17 | for key, val in os_opts.items(): 18 | if not val: 19 | # Don't use empty os vals 20 | continue 21 | if key in opts: 22 | opts[key] = val 23 | for key, val in updates.items(): 24 | if isinstance(val, dict) and isinstance(opts.get(key), dict): 25 | _merge_dicts(opts.get(key, {}), val, os_opts, explicit_cli_args) 26 | elif val is not None: 27 | if key not in opts: 28 | # The key is not in opts(from config file), let's add it 29 | opts[key] = val 30 | continue 31 | 32 | # We already have a value for the key in opts 33 | if opts[key] == val: 34 | # The value is the same, carry on 35 | continue 36 | 37 | if key in explicit_cli_args: 38 | # We have a value for the key in opts(from config file) but 39 | # this option was explicitly passed on the CLI, ie, it's not 40 | # a default value. 41 | # Overwrite what's in opts 42 | opts[key] = val 43 | continue 44 | return opts 45 | 46 | 47 | def read(hub, 48 | defaults, 49 | subs=None, 50 | loader='json', 51 | process_cli=True, 52 | process_cli_known_args_only=False, 53 | args=None, 54 | namespace=None): 55 | ''' 56 | Pass in the default options dict to use 57 | :param opts: 58 | :param process_cli: Process the passed args or sys.argv 59 | :param process_cli_known_args_only: Tells the ArgumentParser to only process known arguments 60 | :param args: Arguments to pass to ArgumentParser 61 | :param namespace: argparse.Namespace to pass to ArgumentParser 62 | :return: options 63 | ''' 64 | hub.conf._loader = loader 65 | if subs: 66 | hub.conf.args.subs(subs) 67 | opts = hub.conf.args.setup(defaults)['return'] 68 | os_opts = hub.conf.os.gather(defaults) 69 | if process_cli is True: 70 | cli_opts = hub.conf.args.parse(args, namespace, process_cli_known_args_only)['return'] 71 | else: 72 | cli_opts = {} 73 | explicit_cli_args = cli_opts.pop('_explicit_cli_args_', set()) 74 | cli_opts = hub.conf.args.render(defaults, cli_opts, explicit_cli_args) 75 | kwargs = {} 76 | # Due to the order of priorities and the representation of defaults in the 77 | # Argparser we need to manually check if the config option values are from 78 | # the cli or from defaults 79 | f_func = False 80 | if 'config_dir' in cli_opts: 81 | if cli_opts['config_dir']: 82 | kwargs['confdir'] = cli_opts['config_dir'] 83 | else: 84 | kwargs['confdir'] = opts['config_dir'] 85 | if 'config_recurse' in cli_opts: 86 | if cli_opts['config_recurse']: 87 | kwargs['recurse'] = cli_opts['config_recurse'] 88 | else: 89 | kwargs['recurse'] = opts['config_recurse'] 90 | # If the config_dir configuration dictionary provides a configuration 91 | # file pattern to read, pass it along 92 | kwargs['pattern'] = defaults['config_dir'].get('pattern') 93 | f_func = hub.conf.file.load_dir 94 | elif 'config' in cli_opts: 95 | if cli_opts['config']: 96 | kwargs['paths'] = cli_opts['config'] 97 | else: 98 | kwargs['paths'] = opts['config'] 99 | f_func = hub.conf.file.load_file 100 | # Render args before config parsing 101 | if f_func: 102 | f_opts = f_func(**kwargs) 103 | opts.update(f_opts) 104 | return _merge_dicts(opts, cli_opts, os_opts, explicit_cli_args) 105 | else: 106 | return _merge_dicts(opts, cli_opts, os_opts, explicit_cli_args) 107 | -------------------------------------------------------------------------------- /pop/mods/conf/toml_conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ''' 3 | Define the yaml loader interface 4 | ''' 5 | 6 | # Import third party libs 7 | try: 8 | import toml 9 | HAS_TOML = True 10 | except ImportError: 11 | HAS_TOML = False 12 | 13 | __virtualname__ = 'toml' 14 | #__contracts__ = [__virtualname__] 15 | 16 | 17 | def __virtual__(hub): 18 | if HAS_TOML: 19 | return True 20 | return (False, 'TOML could not be loaded') 21 | 22 | 23 | def load(hub, path): 24 | ''' 25 | use toml to read in a file 26 | ''' 27 | try: 28 | with open(path, 'rb') as fp_: 29 | return toml.load(fp_.read()) 30 | except FileNotFoundError: 31 | pass 32 | return {} 33 | 34 | def render(hub, val): 35 | ''' 36 | Take the string and render it in json 37 | ''' 38 | return toml.loads(val) 39 | -------------------------------------------------------------------------------- /pop/mods/conf/version.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Support embedding version number lookup into cli 3 | ''' 4 | # IMport python libs 5 | import importlib 6 | import sys 7 | 8 | 9 | CONFIG = { 10 | 'version': { 11 | 'default': False, 12 | 'action': 'store_true', 13 | 'help': 'Display version information', 14 | } 15 | } 16 | 17 | 18 | def run(hub, primary): 19 | ''' 20 | Check the version number and then exit 21 | ''' 22 | mod = importlib.import_module(f'{primary}.version') 23 | print(f'{primary} {mod.version}') 24 | sys.exit(0) 25 | -------------------------------------------------------------------------------- /pop/mods/conf/yaml_conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ''' 3 | Define the yaml loader interface 4 | ''' 5 | 6 | # Import third party libs 7 | try: 8 | import yaml 9 | HAS_YAML = True 10 | except ImportError: 11 | HAS_YAML = False 12 | 13 | __virtualname__ = 'yaml' 14 | __contracts__ = [__virtualname__] 15 | 16 | 17 | def __virtual__(hub): 18 | if HAS_YAML: 19 | return True 20 | return (False, 'PyYaml could not be loaded') 21 | 22 | 23 | def load(hub, path): 24 | ''' 25 | use yaml to read in a file 26 | ''' 27 | try: 28 | with open(path, 'rb') as fp_: 29 | return yaml.safe_load(fp_.read()) 30 | except FileNotFoundError: 31 | pass 32 | return {} 33 | 34 | 35 | def render(hub, val): 36 | ''' 37 | Take the string and render it in json 38 | ''' 39 | return yaml.safe_load(val) 40 | -------------------------------------------------------------------------------- /pop/mods/pop/conf.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Convenience wrappers to make using the conf system as easy and seamless as possible 3 | ''' 4 | 5 | 6 | def integrate( 7 | hub, 8 | imports, 9 | override=None, 10 | cli=None, 11 | roots=None, 12 | loader='json', 13 | logs=True): 14 | ''' 15 | Load the conf sub and run the integrate sequence. 16 | ''' 17 | hub.pop.sub.add('pop.mods.conf') 18 | hub.conf.integrate.load( 19 | imports, 20 | override, 21 | cli=cli, 22 | roots=roots, 23 | loader=loader, 24 | logs=logs) 25 | -------------------------------------------------------------------------------- /pop/mods/pop/dicts.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ''' 3 | Tools to work with dicts 4 | ''' 5 | 6 | __virtualname__ = 'dicts' 7 | 8 | 9 | def traverse(hub, data, key, default=None, delimiter=':'): 10 | ''' 11 | traverse a dict or list using a colon-delimited (or otherwise delimited, 12 | using the 'delimiter' param) target string. the target 'foo:bar:0' will 13 | return data['foo']['bar'][0] if this value exists, and will otherwise 14 | return the dict in the default argument. 15 | function will automatically determine the target type. 16 | the target 'foo:bar:0' will return data['foo']['bar'][0] if data like 17 | {'foo':{'bar':['baz']}} , if data like {'foo':{'bar':{'0':'baz'}}} 18 | then return data['foo']['bar']['0'] 19 | ''' 20 | for each in key.split(delimiter): 21 | if isinstance(data, list): 22 | try: 23 | idx = int(each) 24 | except ValueError: 25 | embed_match = False 26 | # Index was not numeric, lets look at any embedded dicts 27 | for embedded in (x for x in data if isinstance(x, dict)): 28 | try: 29 | data = embedded[each] 30 | embed_match = True 31 | break 32 | except KeyError: 33 | pass 34 | if not embed_match: 35 | # No embedded dicts matched, return the default 36 | return default 37 | else: 38 | try: 39 | data = data[idx] 40 | except IndexError: 41 | return default 42 | else: 43 | try: 44 | data = data[each] 45 | except (KeyError, TypeError): 46 | return default 47 | return data 48 | -------------------------------------------------------------------------------- /pop/mods/pop/input.py: -------------------------------------------------------------------------------- 1 | ''' 2 | The input module is used to translate typical input strings into the 3 | ref/args/kwargs used by pop when forwarding data into functions. 4 | ''' 5 | # Import python libs 6 | import re 7 | # Import third party libs 8 | import yaml 9 | 10 | KWARG_REGEX = re.compile(r'^([^\d\W][\w.-]*)=(?!=)(.*)$', re.UNICODE) 11 | 12 | 13 | def parse(hub, args, condition=True, no_parse=None): 14 | ''' 15 | Parse out the args and kwargs from a list of input values. Optionally, 16 | return the args and kwargs without passing them to condition_input(). 17 | Don't pull args with key=val apart if it has a newline in it. 18 | ''' 19 | if no_parse is None: 20 | no_parse = () 21 | _args = [] 22 | _kwargs = {} 23 | for arg in args: 24 | if isinstance(arg, str): 25 | if '=' in arg: 26 | arg_name, arg_value = _parse_kwarg(arg) 27 | if arg_name: 28 | _kwargs[arg_name] = _yamlify_arg(arg_value) \ 29 | if arg_name not in no_parse \ 30 | else arg_value 31 | else: 32 | _args.append(_yamlify_arg(arg)) 33 | elif isinstance(arg, dict): 34 | # Yes, we're popping this key off and adding it back if 35 | # condition_input is called below, but this is the only way to 36 | # gracefully handle both CLI and API input. 37 | if arg.pop('__kwarg__', False) is True: 38 | _kwargs.update(arg) 39 | else: 40 | _args.append(arg) 41 | else: 42 | _args.append(arg) 43 | if condition: 44 | return _condition_input(_args, _kwargs) 45 | return _args, _kwargs 46 | 47 | 48 | def _yamlify_arg(arg): 49 | ''' 50 | yaml.safe_load the arg 51 | ''' 52 | if not isinstance(arg, str): 53 | return arg 54 | 55 | if arg.strip() == '': 56 | # Because YAML loads empty (or all whitespace) strings as None, we 57 | # return the original string 58 | # >>> import yaml 59 | # >>> yaml.load('') is None 60 | # True 61 | # >>> yaml.load(' ') is None 62 | # True 63 | return arg 64 | 65 | elif '_' in arg and all([x in '0123456789_' for x in arg.strip()]): 66 | # When the stripped string includes just digits and underscores, the 67 | # underscores are ignored and the digits are combined together and 68 | # loaded as an int. We don't want that, so return the original value. 69 | return arg 70 | 71 | try: 72 | original_arg = arg 73 | if '#' in arg: 74 | # Only yamlify if it parses into a non-string type, to prevent 75 | # loss of content due to # as comment character 76 | parsed_arg = yaml.safe_load(arg) 77 | if isinstance(parsed_arg, str) or parsed_arg is None: 78 | return arg 79 | return parsed_arg 80 | if arg == 'None': 81 | arg = None 82 | else: 83 | arg = yaml.safe_load(arg) 84 | 85 | if isinstance(arg, dict): 86 | # dicts must be wrapped in curly braces 87 | if (isinstance(original_arg, str) and 88 | not original_arg.startswith('{')): 89 | return original_arg 90 | else: 91 | return arg 92 | 93 | elif isinstance(arg, list): 94 | # lists must be wrapped in brackets 95 | if (isinstance(original_arg, str) and 96 | not original_arg.startswith('[')): 97 | return original_arg 98 | else: 99 | return arg 100 | 101 | elif arg is None \ 102 | or isinstance(arg, (list, float, int, str)): 103 | # yaml.safe_load will load '|' as '', don't let it do that. 104 | if arg == '' and original_arg in ('|',): 105 | return original_arg 106 | # yaml.safe_load will treat '#' as a comment, so a value of '#' 107 | # will become None. Keep this value from being stomped as well. 108 | elif arg is None and original_arg.strip().startswith('#'): 109 | return original_arg 110 | else: 111 | return arg 112 | else: 113 | # we don't support this type 114 | return original_arg 115 | except Exception: 116 | # In case anything goes wrong... 117 | return original_arg 118 | 119 | 120 | def _parse_kwarg(string_): 121 | ''' 122 | Parses the string and looks for the following kwarg format: 123 | "{argument name}={argument value}" 124 | For example: "my_message=Hello world" 125 | Returns the kwarg name and value, or (None, None) if the regex was not 126 | matched. 127 | ''' 128 | try: 129 | return KWARG_REGEX.match(string_).groups() 130 | except AttributeError: 131 | return None, None 132 | 133 | 134 | def _condition_input(args, kwargs): 135 | ''' 136 | Return a single arg structure for the publisher to safely use 137 | ''' 138 | ret = [] 139 | for arg in args: 140 | if isinstance(arg, int): 141 | ret.append(str(arg)) 142 | else: 143 | ret.append(arg) 144 | if isinstance(kwargs, dict) and kwargs: 145 | kw_ = {'__kwarg__': True} 146 | for key, val in kwargs.items(): 147 | kw_[key] = val 148 | return ret + [kw_] 149 | return ret 150 | 151 | -------------------------------------------------------------------------------- /pop/mods/pop/loop.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ''' 3 | The main interface for management of the aio loop 4 | ''' 5 | # Import python libs 6 | import asyncio 7 | import os 8 | import sys 9 | import signal 10 | import functools 11 | 12 | __virtualname__ = 'loop' 13 | 14 | 15 | def __virtual__(hub): 16 | return True 17 | 18 | 19 | def create(hub): 20 | ''' 21 | Create the loop at hub.pop.Loop 22 | ''' 23 | if not hub.pop.Loop: 24 | hub.pop.loop.FUT_QUE = asyncio.Queue() 25 | hub.pop.Loop = asyncio.get_event_loop() 26 | 27 | 28 | def call_soon(hub, ref, *args, **kwargs): 29 | ''' 30 | Schedule a coroutine to be called when the loop has time. This needs 31 | to be called after the creation fo the loop 32 | ''' 33 | fun = hub.pop.ref.get_func(ref) 34 | hub.pop.Loop.call_soon(functools.partial(fun, *args, **kwargs)) 35 | 36 | 37 | def ensure_future(hub, ref, *args, **kwargs): 38 | ''' 39 | Schedule a coroutine to be called when the loop has time. This needs 40 | to be called after the creation fo the loop. This function also uses 41 | the hold system to await the future when it is done making it easy 42 | to create a future that will be cleanly awaited in the background. 43 | ''' 44 | fun = getattr(hub, ref) 45 | future = asyncio.ensure_future(fun(*args, **kwargs)) 46 | 47 | def callback(fut): 48 | hub.pop.loop.FUT_QUE.put_nowait(fut) 49 | future.add_done_callback(callback) 50 | return future 51 | 52 | 53 | def start(hub, *coros, hold=False, sigint=None, sigterm=None): 54 | ''' 55 | Start a loop that will run until complete 56 | ''' 57 | hub.pop.loop.create() 58 | if sigint: 59 | s = signal.SIGINT 60 | hub.pop.Loop.add_signal_handler( 61 | s, lambda s=s: asyncio.create_task(sigint(s)) 62 | ) 63 | if sigterm: 64 | s = signal.SIGTERM 65 | hub.pop.Loop.add_signal_handler( 66 | s, lambda s=s: asyncio.create_task(sigterm(s)) 67 | ) 68 | if hold: 69 | coros = list(coros) 70 | coros.append(_holder(hub)) 71 | # DO NOT CHANGE THIS CALL TO run_forever! If we do that then the tracebacks 72 | # do not get resolved. 73 | return hub.pop.Loop.run_until_complete( 74 | asyncio.gather(*coros) 75 | ) 76 | 77 | 78 | async def _holder(hub): 79 | ''' 80 | Just a sleeping while loop to hold the loop open while it runs until 81 | complete 82 | ''' 83 | while True: 84 | future = await hub.pop.loop.FUT_QUE.get() 85 | await future 86 | 87 | 88 | async def await_futures(hub): 89 | ''' 90 | Scan over the futures that have completed and manually await them. 91 | This function is used to clean up futures when the loop is not opened 92 | up with hold=True so that ensured futures can be cleaned up on demand 93 | ''' 94 | while not hub.pop.loop.FUT_QUE.empty(): 95 | future = await hub.pop.loop.FUT_QUE.get() 96 | await future 97 | 98 | 99 | async def kill(hub, wait=0): 100 | ''' 101 | Close out the loop 102 | ''' 103 | await asyncio.sleep(wait) 104 | hub.pop.Loop.stop() 105 | while True: 106 | if not hub.pop.Loop.is_running(): 107 | hub.pop.Loop.close() 108 | await asyncio.sleep(1) 109 | 110 | 111 | async def as_yielded(hub, gens): 112 | ''' 113 | Concurrently run multiple async generators and yield the next yielded 114 | value from the soonest yielded generator. 115 | 116 | async def many(): 117 | for n in range(10): 118 | yield os.urandom(6).hex() 119 | 120 | async def run(): 121 | gens = [] 122 | for n in range(10): 123 | gens.append(many()) 124 | async for y in as_yielded(gens): 125 | print(y) 126 | ''' 127 | fin = os.urandom(32) 128 | que = asyncio.Queue() 129 | fs = [] 130 | to_clean = [] 131 | async def _yield(gen): 132 | async for comp in gen: 133 | await que.put(comp) 134 | async def _ensure(coros): 135 | for f in asyncio.as_completed(coros): 136 | await f 137 | async def _set_done(): 138 | await que.put(fin) 139 | def _done(future): 140 | to_clean.append(asyncio.ensure_future(_set_done())) 141 | coros = [] 142 | for gen in gens: 143 | coros.append(_yield(gen)) 144 | f = asyncio.ensure_future(_ensure(coros)) 145 | f.add_done_callback(_done) 146 | while True: 147 | ret = await que.get() 148 | if ret == fin: 149 | break 150 | yield ret 151 | for c in to_clean: 152 | await c 153 | -------------------------------------------------------------------------------- /pop/mods/pop/ref.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Used to resolve resolutions to paths on the hub 3 | ''' 4 | 5 | 6 | def last(hub, ref): 7 | ''' 8 | Takes a string that references the desired ref and returns the last object 9 | called out in that ref 10 | ''' 11 | return hub.pop.ref.path(ref)[-1] 12 | 13 | 14 | def path(hub, ref): 15 | ''' 16 | Retuns a list of references up to the named ref 17 | ''' 18 | ret = [hub] 19 | if isinstance(ref, str): 20 | ref = ref.split('.') 21 | for chunk in ref: 22 | ret.append(getattr(ret[-1], chunk)) 23 | return ret 24 | 25 | 26 | def create(hub, ref, obj): 27 | ''' 28 | Create an attribute at a given target using just a ref string and the 29 | object to be saved at said location. The desired location must already 30 | exist! 31 | 32 | :param ref: The dot delimited string referencing the target location to 33 | create the given object on the hub 34 | :param obj: The object to store at the given reference point 35 | ''' 36 | if '.' not in ref: 37 | setattr(hub, ref, obj) 38 | return 39 | comps = ref.split('.') 40 | sub_ref = ref[:ref.rindex('.')] 41 | var = comps[-1] 42 | top = hub.pop.ref.last(sub_ref) 43 | setattr(top, var, obj) 44 | -------------------------------------------------------------------------------- /pop/mods/pop/sub.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ''' 3 | Control and add subsystems to the running daemon hub 4 | ''' 5 | # Import python libs 6 | import os 7 | # Import pop libs 8 | import pop.hub 9 | 10 | 11 | def add(hub, 12 | pypath=None, 13 | subname=None, 14 | sub=None, 15 | static=None, 16 | contracts_pypath=None, 17 | contracts_static=None, 18 | default_contracts=None, 19 | virtual=True, 20 | dyne_name=None, 21 | omit_start=('_'), 22 | omit_end=(), 23 | omit_func=False, 24 | omit_class=True, 25 | omit_vars=False, 26 | mod_basename='pop.sub', 27 | stop_on_failures=False, 28 | init=True, 29 | ): 30 | ''' 31 | Add a new subsystem to the hub 32 | ''' 33 | if pypath: 34 | pypath = pop.hub.ex_path(pypath) 35 | subname = subname if subname else pypath[0].split('.')[-1] 36 | elif static: 37 | subname = subname if subname else os.path.basename(static) 38 | if dyne_name: 39 | subname = subname if subname else dyne_name 40 | if sub: 41 | root = sub 42 | else: 43 | root = hub 44 | root._subs[subname] = pop.hub.Sub( 45 | hub, 46 | subname, 47 | pypath, 48 | static, 49 | contracts_pypath, 50 | contracts_static, 51 | default_contracts, 52 | virtual, 53 | dyne_name, 54 | omit_start, 55 | omit_end, 56 | omit_func, 57 | omit_class, 58 | omit_vars, 59 | mod_basename, 60 | stop_on_failures) 61 | root._subs[subname]._sub_init(init) 62 | root._iter_subs = sorted(root._subs.keys()) 63 | 64 | 65 | def remove(hub, subname): 66 | ''' 67 | Remove a pop from the hub, run the shutdown if needed 68 | ''' 69 | if hasattr(hub, subname): 70 | sub = getattr(hub, subname) 71 | if hasattr(sub, 'init'): 72 | mod = getattr(sub, 'init') 73 | if hasattr(mod, 'shutdown'): 74 | mod.shutdown() 75 | hub._remove_subsystem(subname) 76 | 77 | 78 | def load_all(hub, subname): 79 | ''' 80 | Load all modules under a given pop 81 | ''' 82 | if hasattr(hub, subname): 83 | sub = getattr(hub, subname) 84 | sub._load_all() 85 | return True 86 | else: 87 | return False 88 | 89 | 90 | def get_dirs(hub, sub): 91 | ''' 92 | Return a list of directories that contain the modules for this subname 93 | ''' 94 | return sub._dirs 95 | 96 | 97 | def iter_subs(hub, sub, recurse=False): 98 | ''' 99 | Return an iterator that will traverse just the subs. This is useful for 100 | nested subs 101 | ''' 102 | for name in sorted(sub._subs): 103 | ret = sub._subs[name] 104 | yield ret 105 | if recurse: 106 | if hasattr(ret, '_subs'): 107 | for nest in hub.pop.sub.iter_subs(ret, recurse): 108 | yield nest 109 | 110 | 111 | def load_subdirs(hub, sub, recurse=False): 112 | ''' 113 | Given a sub, load all subdirectories found under the sub into a lower namespace 114 | ''' 115 | dirs = hub.pop.sub.get_dirs(sub) 116 | roots = {} 117 | for dir_ in dirs: 118 | for fn in os.listdir(dir_): 119 | if fn.startswith('_'): 120 | continue 121 | if fn == 'contracts': 122 | continue 123 | full = os.path.join(dir_, fn) 124 | if not os.path.isdir(full): 125 | continue 126 | if fn not in roots: 127 | roots[fn] = [full] 128 | else: 129 | roots[fn].append(full) 130 | for name, sub_dirs in roots.items(): 131 | # Load er up! 132 | hub.pop.sub.add( 133 | subname=name, 134 | sub=sub, 135 | static=sub_dirs, 136 | virtual=sub._virtual, 137 | omit_start=sub._omit_start, 138 | omit_end=sub._omit_end, 139 | omit_func=sub._omit_func, 140 | omit_class=sub._omit_class, 141 | omit_vars=sub._omit_vars, 142 | mod_basename=sub._mod_basename, 143 | stop_on_failures=sub._stop_on_failures) 144 | if recurse: 145 | hub.pop.sub.load_subdirs(getattr(sub, name), recurse) 146 | 147 | 148 | def reload(hub, subname): 149 | ''' 150 | Instruct the hub to reload the modules for the given sub. This does not call 151 | the init.new function or remove sub level variables. But it does re-read the 152 | directory list and re-initialize the loader causing all modules to be re-evaluated 153 | when started. 154 | ''' 155 | if hasattr(hub, subname): 156 | sub = getattr(hub, subname) 157 | sub._prepare() 158 | return True 159 | else: 160 | return False 161 | 162 | 163 | def extend( 164 | hub, 165 | subname, 166 | pypath=None, 167 | static=None, 168 | contracts_pypath=None, 169 | contracts_static=None): 170 | ''' 171 | Extend the directory lookup for a given sub. Any of the directory lookup 172 | arguments can be passed. 173 | ''' 174 | if not hasattr(hub, subname): 175 | return False 176 | sub = getattr(hub, subname) 177 | if pypath: 178 | sub._pypath.extend(pop.hub.ex_path(pypath)) 179 | if static: 180 | sub._static.extend(pop.hub.ex_path(static)) 181 | if contracts_pypath: 182 | sub._contracts_pypath.extend(pop.hub.ex_path(contracts_pypath)) 183 | if contracts_static: 184 | sub._contracts_static.extend(pop.hub.ex_path(contracts_static)) 185 | sub._prepare() 186 | -------------------------------------------------------------------------------- /pop/mods/pop/verify.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Routines to verify the working environment etc. 3 | ''' 4 | # Import python libs 5 | import os 6 | 7 | 8 | def env(hub): 9 | ''' 10 | Verify that the directories specified in the system exist 11 | ''' 12 | for key in hub.opts: 13 | if key.endswith('_dir'): 14 | try: 15 | os.makedirs(hub.opts[key]) 16 | except OSError: 17 | pass 18 | 19 | -------------------------------------------------------------------------------- /pop/mods/proc/init.py: -------------------------------------------------------------------------------- 1 | ''' 2 | The Proc sub is used to spin up worker processes that run hub referenced 3 | coroutines. 4 | ''' 5 | # Import python libs 6 | import os 7 | import sys 8 | import atexit 9 | import itertools 10 | import asyncio 11 | import subprocess 12 | # Import third party libs 13 | import msgpack 14 | 15 | 16 | def __init__(hub): 17 | ''' 18 | Create constants used by the client and server side of procs 19 | ''' 20 | hub.proc.DELIM = b'd\xff\xcfCO)\xfe=' 21 | hub.proc.D_FLAG = b'D' 22 | hub.proc.I_FLAG = b'I' 23 | hub.proc.Workers = {} 24 | hub.proc.WorkersIter = {} 25 | hub.proc.WorkersTrack = {} 26 | 27 | 28 | def _get_cmd(hub, ind, ref, ret_ref, sock_dir): 29 | ''' 30 | Return the shell command to execute that will start up the worker 31 | ''' 32 | code = 'import sys; ' 33 | code += 'import pop.hub; ' 34 | code += 'hub = pop.hub.Hub(); ' 35 | code += 'hub.pop.sub.add("pop.mods.proc"); ' 36 | code += f'hub.proc.worker.start("{sock_dir}", "{ind}", "{ref}", "{ret_ref}")' 37 | cmd = f'{sys.executable} -c \'{code}\'' 38 | return cmd 39 | 40 | 41 | def mk_proc(hub, ind, workers, ret_ref, sock_dir): 42 | ''' 43 | Create the process and add it to the passed in workers dict at the 44 | specified index 45 | ''' 46 | ref = os.urandom(3).hex() + '.sock' 47 | workers[ind] = {'ref': ref} 48 | workers[ind]['path'] = os.path.join(sock_dir, ref) 49 | cmd = _get_cmd(hub, ind, ref, ret_ref, sock_dir) 50 | workers[ind]['proc'] = subprocess.Popen(cmd, shell=True) 51 | workers[ind]['pid'] = workers[ind]['proc'].pid 52 | 53 | 54 | async def pool(hub, num, name='Workers', callback=None, sock_dir=None): 55 | ''' 56 | Create a new local pool of process based workers 57 | 58 | :param num: The number of processes to add to this pool 59 | :param ref: The location on the hub to create the Workers dict used to 60 | store the worker pool, defaults to `hub.pop.proc.Workers` 61 | :param callback: The pop ref to call when the process communicates 62 | back 63 | ''' 64 | ret_ref = os.urandom(3).hex() + '.sock' 65 | ret_sock_path = os.path.join(sock_dir, ret_ref) 66 | if not hub.proc.Tracker: 67 | hub.proc.init.mk_tracker() 68 | workers = {} 69 | if callback: 70 | await asyncio.start_unix_server( 71 | hub.proc.init.ret_work(callback), 72 | path=ret_sock_path) 73 | for ind in range(num): 74 | hub.proc.init.mk_proc(ind, workers, ret_ref, sock_dir) 75 | w_iter = itertools.cycle(workers) 76 | hub.proc.Workers[name] = workers 77 | hub.proc.WorkersIter[name] = w_iter 78 | hub.proc.WorkersTrack[name] = { 79 | 'subs': [], 80 | 'ret_ref': ret_ref, 81 | 'sock_dir': sock_dir} 82 | up = set() 83 | while True: 84 | for ind in workers: 85 | if os.path.exists(workers[ind]['path']): 86 | up.add(ind) 87 | if len(up) == num: 88 | break 89 | await asyncio.sleep(0.01) 90 | # TODO: This seems to be spawning extra procs, this should be fixed 91 | #asyncio.ensure_future(hub.proc.init.maintain(name)) 92 | 93 | 94 | async def maintain(hub, name): 95 | ''' 96 | Keep an eye on these processes 97 | ''' 98 | workers = hub.proc.Workers[name] 99 | while True: 100 | for ind, data in workers.items(): 101 | if not data['proc'].poll(): 102 | hub.proc.init.mk_proc(ind, workers) 103 | await asyncio.sleep(2) 104 | 105 | 106 | def mk_tracker(hub): 107 | ''' 108 | Create the process tracker, this simply makes a data structure to hold 109 | process references and sets them to be terminated when the system is 110 | shutdown. 111 | ''' 112 | hub.proc.Tracker = True 113 | atexit.register(hub.proc.init.clean) 114 | 115 | 116 | def clean(hub): 117 | ''' 118 | Clean up the processes registered in the tracker 119 | ''' 120 | for name, workers in hub.proc.Workers.items(): 121 | for ind in workers: 122 | workers[ind]['proc'].terminate() 123 | 124 | 125 | def ret_work(hub, callback): 126 | async def work(reader, writer): 127 | ''' 128 | Process the incoming work 129 | ''' 130 | inbound = await reader.readuntil(hub.proc.DELIM) 131 | inbound = inbound[:-len(hub.proc.DELIM)] 132 | payload = msgpack.loads(inbound, raw=False) 133 | ret = await callback(payload) 134 | ret = msgpack.dumps(ret, use_bin_type=True) 135 | ret += hub.proc.DELIM 136 | writer.write(ret) 137 | await writer.drain() 138 | writer.close() 139 | return work 140 | -------------------------------------------------------------------------------- /pop/mods/proc/run.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Execute functions or load subs on the workers in the named worker pool 3 | ''' 4 | # import python libs 5 | import asyncio 6 | import os 7 | # Import third party libs 8 | import msgpack 9 | 10 | 11 | async def add_sub(hub, worker_name, *args, **kwargs): 12 | ''' 13 | Tell all of the worker in the named pool to load the given sub, 14 | 15 | This function takes all of the same arguments as hub.pop.sub.add 16 | ''' 17 | ret = {} 18 | workers = hub.proc.Workers[worker_name] 19 | for ind in workers: 20 | payload = {'fun': 'sub', 'args': args, 'kwargs': kwargs} 21 | # TODO: Make these futures to the run at the same time 22 | async for chunk in hub.proc.run.send(workers[ind], payload): 23 | ret[ind] = chunk 24 | hub.proc.WorkersTrack[worker_name]['subs'].append({'args': args, 'kwargs': kwargs}) 25 | return ret 26 | 27 | 28 | async def add_proc(hub, worker_name): 29 | ''' 30 | Add a single process to the worker pool, also make sure that 31 | ''' 32 | # grab and extrapolate the data we need 33 | ret_ref = hub.proc.WorkersTrack[worker_name]['ret_ref'] 34 | sock_dir = hub.proc.WorkersTrack[worker_name]['sock_dir'] 35 | workers = hub.proc.Workers[worker_name] 36 | ind = len(workers) + 1 37 | for s_ind in range(len(workers) + 1): 38 | if s_ind not in workers: 39 | ind = s_ind 40 | hub.proc.init.mk_proc(ind, workers, ret_ref, sock_dir) 41 | # Make sure the process is up with a live socket 42 | while True: 43 | if os.path.exists(workers[ind]['path']): 44 | break 45 | await asyncio.sleep(0.01) 46 | # Add all of the subs that have been added to processes in this pool 47 | for sub in hub.proc.WorkersTrack[worker_name]['subs']: 48 | payload = {'fun': 'sub', 'args': sub['args'], 'kwargs': sub['kwargs']} 49 | async for chunk in hub.proc.run.send(workers[ind], payload): 50 | pass 51 | return ind 52 | 53 | 54 | async def pub(hub, worker_name, func_ref, *args, **kwargs): 55 | ''' 56 | Execute the given function reference on ALL the workers in the given 57 | worker pool and return the return data from each. 58 | 59 | Pass in the arguments for the function, keep in mind that the sub needs 60 | to be loaded into the workers for a function to be available via 61 | hub.proc.run.add_sub 62 | ''' 63 | workers = hub.proc.Workers[worker_name] 64 | ret = {} 65 | for ind in workers: 66 | payload = {'fun': 'run', 'ref': func_ref, 'args': args, 'kwargs': kwargs} 67 | # TODO: Make these futures to the run at the same time 68 | async for chunk in hub.proc.run.send(workers[ind], payload): 69 | ret[ind] = chunk 70 | return ret 71 | 72 | 73 | async def set_attr(hub, worker_name, ref, value): 74 | ''' 75 | Set the given attribute to the given location on the hub of all 76 | worker procs 77 | ''' 78 | workers = hub.proc.Workers[worker_name] 79 | ret = {} 80 | for ind in workers: 81 | payload = {'fun': 'setattr', 'ref': ref, 'value': value} 82 | # TODO: Make these futures to the run at the same time 83 | async for chunk in hub.proc.run.send(workers[ind], payload): 84 | ret[ind] = chunk 85 | return ret 86 | 87 | 88 | async def ind_func(hub, worker_name, _ind, func_ref, *args, **kwargs): 89 | ''' 90 | Execute the function on the indexed process within the named worker pool 91 | ''' 92 | workers = hub.proc.Workers[worker_name] 93 | worker = workers[_ind] 94 | payload = {'fun': 'run', 'ref': func_ref, 'args': args, 'kwargs': kwargs} 95 | async for ret in hub.proc.run.send(worker, payload): 96 | return ret 97 | 98 | 99 | async def func(hub, worker_name, func_ref, *args, **kwargs): 100 | ''' 101 | Execute the given function reference on one worker in the given worker 102 | pool and return the return data. 103 | 104 | Pass in the arguments for the function, keep in mind that the sub needs 105 | to be loaded into the workers for a function to be available via 106 | hub.proc.run.add_sub 107 | ''' 108 | ind, coro = await hub.proc.run.track_func(worker_name, func_ref, *args, **kwargs) 109 | return await coro 110 | 111 | 112 | async def track_func(hub, worker_name, func_ref, *args, **kwargs): 113 | ''' 114 | Run a function and return the index of the worker that the function was 115 | executed on and a coroutine to track 116 | ''' 117 | w_iter = hub.proc.WorkersIter[worker_name] 118 | ind = next(w_iter) 119 | coro = hub.proc.run.ind_func(worker_name, ind, func_ref, *args, **kwargs) 120 | return ind, coro 121 | 122 | 123 | async def gen(hub, worker_name, func_ref, *args, **kwargs): 124 | ''' 125 | Execute a generator function reference within one worker within the given 126 | worker pool. 127 | 128 | Like `func` the sub needs to be made available to all workers first 129 | ''' 130 | ind, coro = await hub.proc.run.track_gen(worker_name, func_ref, *args, **kwargs) 131 | async for chunk in coro: 132 | yield chunk 133 | 134 | 135 | async def track_gen(hub, worker_name, func_ref, *args, **kwargs): 136 | ''' 137 | Return an iterable coroutine and the index executed on 138 | ''' 139 | w_iter = hub.proc.WorkersIter[worker_name] 140 | ind = next(w_iter) 141 | coro = hub.proc.run.ind_gen(worker_name, ind, func_ref, *args, **kwargs) 142 | return ind, coro 143 | 144 | 145 | async def ind_gen(hub, worker_name, _ind, func_ref, *args, **kwargs): 146 | ''' 147 | run the given iterator on the defined index 148 | ''' 149 | workers = hub.proc.Workers[worker_name] 150 | worker = workers[_ind] 151 | payload = {'fun': 'gen', 'ref': func_ref, 'args': args, 'kwargs': kwargs} 152 | async for chunk in hub.proc.run.send(worker, payload): 153 | yield chunk 154 | 155 | 156 | async def send(hub, worker, payload): 157 | ''' 158 | Send the given payload to the given worker, yield iterations based on the 159 | returns from the remote. 160 | ''' 161 | mp = msgpack.dumps(payload, use_bin_type=True) 162 | mp += hub.proc.DELIM 163 | reader, writer = await asyncio.open_unix_connection(path=worker['path']) 164 | writer.write(mp) 165 | await writer.drain() 166 | final_ret = True 167 | while True: 168 | ret = await reader.readuntil(hub.proc.DELIM) 169 | p_ret = ret[:-len(hub.proc.DELIM)] 170 | i_flag = p_ret[-1:] 171 | ret = msgpack.loads(p_ret[:-1], raw=False) 172 | if i_flag == hub.proc.D_FLAG: 173 | # break for the end of the sequence 174 | break 175 | yield ret 176 | final_ret = False 177 | if final_ret: 178 | yield ret 179 | -------------------------------------------------------------------------------- /pop/mods/proc/worker.py: -------------------------------------------------------------------------------- 1 | ''' 2 | This module is used to manage the process started up by the pool. Work in this 3 | module is used to manage the worker process itself and not other routines on 4 | the hub this process was derived from 5 | 6 | This is an exec, not a fork! This is a fresh memory space! 7 | ''' 8 | # Import python libs 9 | import os 10 | import types 11 | import asyncio 12 | # Import third party libs 13 | import msgpack 14 | # TODO: The workers should detect if their controlling process dies and terminate by themselves 15 | # The controlling process will kill them when it exists, but if it exists hard then the workers 16 | # Should be able to also clean themselves up 17 | 18 | 19 | def start(hub, sock_dir, ind, ref, ret_ref): 20 | ''' 21 | This function is called by the startup script to create a worker process 22 | 23 | :NOTE: This is a new process started from the shell, it does not have any 24 | of the process namespace from the creating process. 25 | This is an EXEC, NOT a FORK! 26 | ''' 27 | hub.proc.SOCK_DIR = sock_dir 28 | hub.proc.REF = ref 29 | hub.proc.SOCK_PATH = os.path.join(sock_dir, ref) 30 | hub.proc.RET_REF = ret_ref 31 | hub.proc.RET_SOCK_PATH = os.path.join(sock_dir, ret_ref) 32 | hub.proc.IND = ind 33 | hub.pop.loop.start(hub.proc.worker.hold(), hub.proc.worker.server()) 34 | 35 | 36 | async def hold(hub): 37 | ''' 38 | This function just holds the loop open by sleeping in a while loop 39 | ''' 40 | while True: 41 | await asyncio.sleep(60) 42 | 43 | 44 | async def server(hub): 45 | ''' 46 | Start the unix socket server to receive commands 47 | ''' 48 | await asyncio.start_unix_server( 49 | hub.proc.worker.work, 50 | path=hub.proc.SOCK_PATH) 51 | 52 | 53 | async def work(hub, reader, writer): 54 | ''' 55 | Process the incoming work 56 | ''' 57 | inbound = await reader.readuntil(hub.proc.DELIM) 58 | inbound = inbound[:-len(hub.proc.DELIM)] 59 | payload = msgpack.loads(inbound, encoding='utf8') 60 | ret = b'' 61 | if 'fun' not in payload: 62 | ret = {'err': 'Invalid format'} 63 | elif payload['fun'] == 'sub': 64 | # Time to add a sub to the hub! 65 | try: 66 | hub.proc.worker.add_sub(payload) 67 | ret = {'status': True} 68 | except Exception as exc: 69 | ret = {'status': False, 'exc': str(exc)} 70 | elif payload['fun'] == 'run': 71 | # Time to do some work! 72 | try: 73 | ret = await hub.proc.worker.run(payload) 74 | except Exception as exc: 75 | ret = {'status': False, 'exc': str(exc)} 76 | elif payload['fun'] == 'gen': 77 | ret = await hub.proc.worker.gen(payload, reader, writer) 78 | elif payload['fun'] == 'setattr': 79 | ret = await hub.proc.worker.set_attr(payload) 80 | ret = msgpack.dumps(ret, use_bin_type=True) 81 | ret += hub.proc.D_FLAG 82 | ret += hub.proc.DELIM 83 | writer.write(ret) 84 | await writer.drain() 85 | writer.close() 86 | 87 | 88 | def add_sub(hub, payload): 89 | ''' 90 | Add a new sub onto the hub for this worker 91 | ''' 92 | hub.pop.sub.add(*payload['args'], **payload['kwargs']) 93 | 94 | 95 | async def gen(hub, payload, reader, writer): 96 | ''' 97 | Run a generator and yield back the returns. Supports a generator and an 98 | async generator 99 | ''' 100 | ref = payload.get('ref') 101 | args = payload.get('args', []) 102 | kwargs = payload.get('kwargs', {}) 103 | ret = hub.pop.ref.last(ref)(*args, **kwargs) 104 | if isinstance(ret, types.AsyncGeneratorType): 105 | async for chunk in ret: 106 | rchunk = msgpack.dumps(chunk, use_bin_type=True) 107 | rchunk += hub.proc.I_FLAG 108 | rchunk += hub.proc.DELIM 109 | writer.write(rchunk) 110 | await writer.drain() 111 | elif isinstance(ret, types.GeneratorType): 112 | for chunk in ret: 113 | rchunk = msgpack.dumps(chunk, use_bin_type=True) 114 | rchunk += hub.proc.I_FLAG 115 | rchunk += hub.proc.DELIM 116 | writer.write(rchunk) 117 | await writer.drain() 118 | elif asyncio.iscoroutine(ret): 119 | return await ret 120 | else: 121 | return ret 122 | return '' 123 | 124 | 125 | async def run(hub, payload): 126 | ''' 127 | Execute the given payload 128 | ''' 129 | ref = payload.get('ref') 130 | args = payload.get('args', []) 131 | kwargs = payload.get('kwargs', {}) 132 | ret = hub.pop.ref.last(ref)(*args, **kwargs) 133 | if asyncio.iscoroutine(ret): 134 | return await ret 135 | return ret 136 | 137 | 138 | async def set_attr(hub, payload): 139 | ''' 140 | Set the named attribute to the hub 141 | ''' 142 | ref = payload.get('ref') 143 | value = payload.get('value') 144 | hub.pop.ref.create(ref, value) 145 | 146 | 147 | async def ret(hub, payload): 148 | ''' 149 | Send a return payload to the spawning process. This return will be tagged 150 | with the index of the process that returned it 151 | ''' 152 | payload = {'ind': hub.proc.IND, 'payload': payload} 153 | mp = msgpack.dumps(payload, use_bin_type=True) 154 | mp += hub.proc.DELIM 155 | reader, writer = await asyncio.open_unix_connection(path=hub.proc.RET_SOCK_PATH) 156 | writer.write(mp) 157 | await writer.drain() 158 | ret = await reader.readuntil(hub.proc.DELIM) 159 | ret = ret[:-len(hub.proc.DELIM)] 160 | writer.close() 161 | return msgpack.loads(ret, encoding='utf8') 162 | -------------------------------------------------------------------------------- /pop/scanner.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ''' 3 | Used to scan the given directories for loadable files 4 | ''' 5 | # Import python libs 6 | import os 7 | import importlib 8 | import collections 9 | 10 | PY_END = ('.py', '.pyc', '.pyo') 11 | PYEXT_END = tuple(importlib.machinery.EXTENSION_SUFFIXES) 12 | CYTHON_END = ('.pyx',) 13 | SKIP_DIRNAMES = ('__pycache__',) 14 | 15 | 16 | def scan(dirs): 17 | ''' 18 | Return a list of importable files 19 | ''' 20 | ret = collections.OrderedDict() 21 | ret['python'] = collections.OrderedDict() 22 | ret['cython'] = collections.OrderedDict() 23 | ret['ext'] = collections.OrderedDict() 24 | ret['imp'] = collections.OrderedDict() 25 | for dir_ in dirs: 26 | for fn_ in os.listdir(dir_): 27 | _apply_scan(ret, dir_, fn_) 28 | return ret 29 | 30 | 31 | def _apply_scan(ret, dir_, fn_): 32 | ''' 33 | Convert the scan data into 34 | ''' 35 | if fn_.startswith('_'): 36 | return 37 | if os.path.basename(dir_) in SKIP_DIRNAMES: 38 | return 39 | full = os.path.join(dir_, fn_) 40 | if '.' not in full: 41 | return 42 | bname = full[:full.rindex('.')] 43 | if fn_.endswith(PY_END): 44 | if bname not in ret['python']: 45 | ret['python'][bname] = {'path': full} 46 | if fn_.endswith(CYTHON_END): 47 | if bname not in ret['cython']: 48 | ret['cython'][bname] = {'path': full} 49 | if fn_.endswith(PYEXT_END): 50 | if bname not in ret['ext']: 51 | ret['ext'][bname] = {'path': full} 52 | -------------------------------------------------------------------------------- /pop/scripts.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import pop.hub 4 | 5 | 6 | def pop_seed(): 7 | CONFIG = { 8 | 'seed_name': { 9 | 'positional': True, 10 | 'help': 'The name of the project that is being created', 11 | }, 12 | 'type': { 13 | 'default': 'p', 14 | 'options': ['-t'], 15 | 'help': 'The type of project to build, by default make a standalone project, but for a vetical app project pass a "v"', 16 | }, 17 | 'dyne': { 18 | 'options': ['-d'], 19 | 'default': [], 20 | 'nargs': '*', 21 | 'help': 'A space delimited list of additional dynamic names for vertical app-merging', 22 | }, 23 | } 24 | 25 | hub = pop.hub.Hub() 26 | hub.pop.sub.add('pop.mods.conf') 27 | hub.opts = hub.conf.reader.read(CONFIG) 28 | hub.pop.seed.new() 29 | -------------------------------------------------------------------------------- /pop/verify.py: -------------------------------------------------------------------------------- 1 | # Import python libs 2 | import inspect 3 | # Import pop libs 4 | import pop.exc 5 | 6 | 7 | def contract(hub, raws, mod): # pylint: disable=unused-argument 8 | ''' 9 | Verify module level contract - functions only 10 | ''' 11 | sig_errs = [] 12 | sig_miss = [] 13 | mname = mod.__name__ 14 | for raw in raws: 15 | for fun in raw._funcs: 16 | if fun.startswith('sig_'): 17 | tfun = fun[4:] 18 | if tfun not in mod._funcs: 19 | sig_miss.append(tfun) 20 | continue 21 | sig_errs.extend(sig(mod._funcs[tfun].func, raw._funcs[fun].func)) 22 | if sig_errs or sig_miss: 23 | msg = '' 24 | if sig_errs: 25 | msg += f'Signature Errors in {mname}:\n' 26 | for err in sig_errs: 27 | msg += f'{err}\n' 28 | if sig_miss: 29 | msg += f'Signature Functions Missing in {mname}:\n' 30 | for err in sig_miss: 31 | msg += f'{err}\n' 32 | msg = msg.strip() 33 | raise pop.exc.ContractSigException(msg) 34 | 35 | 36 | def sig_map(ver): 37 | ''' 38 | Generates the map dict for the signature verification 39 | ''' 40 | vsig = inspect.signature(ver) 41 | vparams = list(vsig.parameters.values()) 42 | vdat = {'args': [], 'v_pos': -1, 'kw': [], 'kwargs': False, 'ann': {}} 43 | for ind in range(len(vparams)): 44 | param = vparams[ind] 45 | val = param.kind.value 46 | name = param.name 47 | if val == 0 or val == 1: 48 | vdat['args'].append(name) 49 | if param.default != inspect._empty: # Is a KW, can be inside of **kwargs 50 | vdat['kw'].append(name) 51 | elif val == 2: 52 | vdat['v_pos'] = ind 53 | elif val == 3: 54 | vdat['kw'].append(name) 55 | elif val == 4: 56 | vdat['kwargs'] = ind 57 | if param.annotation != inspect._empty: 58 | vdat['ann'][name] = param.annotation 59 | return vdat 60 | 61 | 62 | def sig(func, ver): 63 | ''' 64 | Takes 2 functions, the first function is verified to have a parameter signature 65 | compatible with the second function 66 | ''' 67 | errors = [] 68 | fsig = inspect.signature(func) 69 | fparams = list(fsig.parameters.values()) 70 | vdat = sig_map(ver) 71 | arg_len = len(vdat['args']) 72 | v_pos = False 73 | for ind in range(len(fparams)): 74 | param = fparams[ind] 75 | val = param.kind.value 76 | name = param.name 77 | has_default = param.default != inspect._empty 78 | ann = param.annotation 79 | vann = vdat['ann'].get(name, inspect._empty) 80 | if vann != ann: 81 | errors.append(f'Parameter, "{name}" is type "{str(ann)}" not "{str(vann)}"') 82 | if val == 2: 83 | v_pos = True 84 | if val == 0 or val == 1: 85 | if ind >= arg_len: # Past available positional args 86 | if not vdat['v_pos'] == -1: # Has a *args 87 | if ind >= vdat['v_pos'] and v_pos: 88 | # Invalid unless it is a kw 89 | if not name in vdat['kw']: 90 | # Is a kw 91 | errors.append(f'Parameter "{name}" is invalid') 92 | if vdat['kwargs'] is False: 93 | errors.append(f'Parameter "{name}" not defined as kw only') 94 | continue 95 | elif vdat['kwargs'] is not False and not has_default: 96 | errors.append(f'Parameter "{name}" is past available positional params') 97 | elif vdat['kwargs'] is False: 98 | errors.append(f'Parameter "{name}" is past available positional params') 99 | else: 100 | v_param = vdat['args'][ind] 101 | if v_param != name: 102 | errors.append(f'Parameter "{name}" does not have the correct name: {v_param}') 103 | if val == 2: 104 | if ind < vdat['v_pos']: 105 | errors.append(f'Parameter "{name}" is not in the correct position for *args') 106 | if val == 3: 107 | if name not in vdat['kw'] and not vdat['kwargs']: 108 | errors.append(f'Parameter "{name}" is not available as a kwarg') 109 | if val == 4: 110 | if vdat['kwargs'] is False: 111 | errors.append(f'Kwargs are not permitted as a parameter') 112 | return errors 113 | -------------------------------------------------------------------------------- /pop/version.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | version = '8' 3 | -------------------------------------------------------------------------------- /requirements-test.txt: -------------------------------------------------------------------------------- 1 | asynctest 2 | pytest 3 | pytest-asyncio 4 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | PyYAML 2 | msgpack -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | # Import python libs 5 | import os 6 | import sys 7 | import shutil 8 | from setuptools import setup, Command 9 | 10 | NAME = 'pop' 11 | DESC = ('The Plugin Oriented Programming System') 12 | 13 | # Version info -- read without importing 14 | _locals = {} 15 | with open('pop/version.py') as fp: 16 | exec(fp.read(), None, _locals) 17 | VERSION = _locals['version'] 18 | SETUP_DIRNAME = os.path.dirname(__file__) 19 | if not SETUP_DIRNAME: 20 | SETUP_DIRNAME = os.getcwd() 21 | 22 | 23 | class Clean(Command): 24 | user_options = [] 25 | def initialize_options(self): 26 | pass 27 | 28 | def finalize_options(self): 29 | pass 30 | 31 | def run(self): 32 | for subdir in ('pop', 'tests'): 33 | for root, dirs, files in os.walk(os.path.join(os.path.dirname(__file__), subdir)): 34 | for dir_ in dirs: 35 | if dir_ == '__pycache__': 36 | shutil.rmtree(os.path.join(root, dir_)) 37 | 38 | 39 | def discover_packages(): 40 | modules = [] 41 | for package in ('pop', ): 42 | for root, _, files in os.walk(os.path.join(SETUP_DIRNAME, package)): 43 | pdir = os.path.relpath(root, SETUP_DIRNAME) 44 | modname = pdir.replace(os.sep, '.') 45 | modules.append(modname) 46 | return modules 47 | 48 | 49 | setup(name=NAME, 50 | author='Thomas S Hatch', 51 | author_email='thatch@saltstack.com', 52 | url='https://saltstack.com', 53 | version=VERSION, 54 | description=DESC, 55 | python_requires='>=3.6', 56 | classifiers=[ 57 | 'Operating System :: OS Independent', 58 | 'Programming Language :: Python :: 3.6', 59 | 'Programming Language :: Python :: 3.7', 60 | 'Programming Language :: Python :: 3.8', 61 | 'Development Status :: 5 - Production/Stable', 62 | ], 63 | entry_points={ 64 | 'console_scripts': [ 65 | 'pop-seed = pop.scripts:pop_seed', 66 | ], 67 | }, 68 | packages=discover_packages(), 69 | cmdclass={'clean': Clean}, 70 | ) 71 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | -------------------------------------------------------------------------------- /tests/cmods/contracts/ctest.py: -------------------------------------------------------------------------------- 1 | def pre_cping(hub, ctx): 2 | hub.CPING = True 3 | -------------------------------------------------------------------------------- /tests/cmods/ctest.py: -------------------------------------------------------------------------------- 1 | def cping(hub): 2 | return True 3 | 4 | 5 | -------------------------------------------------------------------------------- /tests/conf1/conf.py: -------------------------------------------------------------------------------- 1 | CONFIG = { 2 | 'test': { 3 | 'default': False, 4 | 'action': 'store_true', 5 | 'help': 'Help, I need sombody!' 6 | }, 7 | 'stuff_dir': { 8 | 'default': '/tmp/tests.conf1/stuff', 9 | 'help': 'A directory dedicated to stuff', 10 | }, 11 | } 12 | 13 | 14 | GLOBAL = { 15 | 'cache_dir': { 16 | 'default': '/var/cache', 17 | 'help': 'A cachedir', 18 | }, 19 | } 20 | 21 | 22 | CLI_CONFIG = { 23 | 'someone': { 24 | 'default': 'Not just anybody!', 25 | 'help': 'Oh yes I need someone', 26 | }, 27 | } 28 | -------------------------------------------------------------------------------- /tests/conf2/conf.py: -------------------------------------------------------------------------------- 1 | CONFIG = { 2 | 'monty': { 3 | 'default': False, 4 | 'action': 'store_true', 5 | 'help': 'Its only a model.' 6 | }, 7 | } 8 | 9 | GLOBAL = { 10 | 'cache_dir': { 11 | 'default': '/var/cache', 12 | 'help': 'A cachedir', 13 | }, 14 | } 15 | 16 | 17 | CLI_CONFIG = { 18 | 'someone': { 19 | 'default': 'Not just anybody!', 20 | 'help': 'Oh yes I need someone', 21 | }, 22 | } 23 | -------------------------------------------------------------------------------- /tests/conf3/conf.py: -------------------------------------------------------------------------------- 1 | CONFIG = { 2 | 'test': { 3 | 'default': False, 4 | 'action': 'store_true', 5 | 'help': 'Help, I need sombody!' 6 | }, 7 | } 8 | 9 | GLOBAL = { 10 | 'cache_dir': { 11 | 'default': '/var/cache', 12 | 'help': 'A cachedir', 13 | }, 14 | } 15 | 16 | 17 | CLI_CONFIG = { 18 | 'someone': { 19 | 'default': 'Not just anybody!', 20 | 'help': 'Oh yes I need someone', 21 | }, 22 | } -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Import python libs 4 | import os 5 | import sys 6 | import glob 7 | import logging 8 | 9 | CODE_DIR = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) 10 | TPATH_DIR = os.path.join(os.path.dirname(__file__), 'tpath') 11 | 12 | if CODE_DIR in sys.path: 13 | sys.path.remove(CODE_DIR) 14 | sys.path.insert(0, CODE_DIR) 15 | sys.path.insert(0, TPATH_DIR) 16 | 17 | # Import 3rd-party libs 18 | import pytest 19 | 20 | 21 | log = logging.getLogger('pop.tests') 22 | 23 | def pytest_runtest_protocol(item, nextitem): 24 | ''' 25 | implements the runtest_setup/call/teardown protocol for 26 | the given test item, including capturing exceptions and calling 27 | reporting hooks. 28 | ''' 29 | log.debug('>>>>> START >>>>> {0}'.format(item.name)) 30 | 31 | 32 | def pytest_runtest_teardown(item): 33 | ''' 34 | called after ``pytest_runtest_call`` 35 | ''' 36 | log.debug('<<<<< END <<<<<<< {0}'.format(item.name)) 37 | 38 | 39 | @pytest.fixture 40 | def os_sleep_secs(): 41 | if 'CI_RUN' in os.environ: 42 | return 1.75 43 | return 0.5 44 | -------------------------------------------------------------------------------- /tests/contracts/ctx.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ''' 3 | Contract Context 4 | ''' 5 | 6 | __virtualname__ = 'ctx' 7 | 8 | 9 | def pre(hub, ctx): 10 | ''' 11 | ''' 12 | assert 'injected_data' not in ctx.cache 13 | ctx.cache['injected_data'] = 1 14 | 15 | 16 | def call(hub, ctx): 17 | ''' 18 | ''' 19 | if 'injected_data' not in ctx.cache: 20 | raise AssertionError('\'injected_data\' was not found in contract call \'ctx.cache\'') 21 | ctx.cache['injected_data'] += 1 22 | return 'contract executed' 23 | 24 | 25 | def post_ping(hub, ctx): 26 | ''' 27 | ''' 28 | if 'injected_data' not in ctx.cache: 29 | raise AssertionError('\'injected_data\' was not found in contract post \'ctx.cache\'') 30 | assert ctx.cache['injected_data'] == 2 31 | return ctx.ret 32 | -------------------------------------------------------------------------------- /tests/contracts/ctx_args.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ''' 3 | Contract Context 4 | ''' 5 | 6 | __virtualname__ = 'ctx_args' 7 | 8 | 9 | def call(hub, ctx): 10 | ''' 11 | ''' 12 | return ctx.get_argument(ctx.get_argument('value')) 13 | -------------------------------------------------------------------------------- /tests/contracts/ctx_update.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ''' 3 | Contract Context 4 | ''' 5 | 6 | __virtualname__ = 'ctx_update' 7 | 8 | 9 | def pre(hub, ctx): 10 | ''' 11 | ''' 12 | assert ctx.args == [hub, True] 13 | # Let's replace the context arguments 14 | ctx.args[1] = False 15 | assert ctx.args == [hub, False] 16 | 17 | 18 | def call_test_call(hub, ctx): 19 | ''' 20 | ''' 21 | assert ctx.args == [hub, False] 22 | return 'contract executed' 23 | 24 | 25 | def post(hub, ctx): 26 | ''' 27 | ''' 28 | assert ctx.args == [hub, False] 29 | return ctx.ret 30 | -------------------------------------------------------------------------------- /tests/contracts/many.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | __virtualname__ = 'all' 4 | 5 | 6 | def pre(hub, ctx): 7 | if len(ctx.args) > 1: 8 | raise ValueError('No can haz args!') 9 | if ctx.kwargs: 10 | raise ValueError('No can haz kwargs!') 11 | 12 | 13 | def call(hub, ctx): 14 | return ctx.func(*ctx.args, **ctx.kwargs) 15 | 16 | 17 | def call_list(hub, ctx): 18 | return ['override'] 19 | 20 | 21 | def post(hub, ctx): 22 | ret = ctx.ret 23 | if isinstance(ret, list): 24 | ret.append('post called') 25 | elif isinstance(ret, dict): 26 | ret['post'] = 'called' 27 | return ret 28 | -------------------------------------------------------------------------------- /tests/contracts/priv.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | __virtualname__ = 'priv' 4 | 5 | 6 | def call(hub, ctx): 7 | return ctx.func(*ctx.args, **ctx.kwargs) 8 | -------------------------------------------------------------------------------- /tests/contracts/test.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ''' 3 | Example contract 4 | ''' 5 | 6 | __virtualname__ = 'test' 7 | 8 | 9 | def functions(hub): 10 | ''' 11 | Return the functions 12 | ''' 13 | return ('ping', 'demo') 14 | 15 | 16 | def pre_ping(hub, ctx): 17 | ''' 18 | ''' 19 | if ctx.args: 20 | raise Exception('ping does not take args!') 21 | if ctx.kwargs: 22 | raise Exception('ping does not take kwargs!') 23 | 24 | 25 | def call_ping(hub, ctx): 26 | ''' 27 | ''' 28 | print('calling!') 29 | return ctx.func(*ctx.args, **ctx.kwargs) 30 | 31 | 32 | def post_ping(hub, ctx): 33 | ''' 34 | ''' 35 | print('Calling Post!') 36 | if not isinstance(ctx.ret, dict): 37 | raise Exception('MUST BE DICT!!') 38 | -------------------------------------------------------------------------------- /tests/contracts/testing.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | 4 | __virtualname__ = 'testing' 5 | 6 | 7 | def call(hub, ctx): 8 | return 'contract ' + ctx.func(*ctx.args, **ctx.kwargs) 9 | 10 | 11 | def call_signature_func(hub, ctx): 12 | args = ctx.get_arguments() 13 | assert args['param1'] == 'passed in' 14 | assert args['param2'] == 'default' 15 | 16 | 17 | async def call_async_echo(hub, ctx): 18 | return 'async contract ' + await ctx.func(*ctx.args, **ctx.kwargs) 19 | -------------------------------------------------------------------------------- /tests/csigs/contracts/sigs.py: -------------------------------------------------------------------------------- 1 | # Import python libs 2 | from typing import List 3 | 4 | def sig_first(hub, a: str, b, c: List): 5 | pass 6 | 7 | 8 | def sig_second(hub, **kwargs): 9 | pass 10 | 11 | 12 | def sig_third(hub, a, b, *args, **kwargs): 13 | pass 14 | 15 | 16 | def sig_four(hub, a, *args, e=7): 17 | pass 18 | 19 | 20 | def sig_five(hub, a: str, *args): 21 | pass 22 | 23 | 24 | def sig_six(hub, a, *args, **kwargs): 25 | pass 26 | 27 | 28 | def sig_seven(hub, foo): 29 | pass 30 | 31 | 32 | def sig_missing(): 33 | ''' 34 | This function is missing in the module to make sure it gets picked up 35 | ''' 36 | pass 37 | -------------------------------------------------------------------------------- /tests/csigs/sigs.py: -------------------------------------------------------------------------------- 1 | def first(hub, a, z, c: str, **kwargs): 2 | return a*b*c 3 | 4 | def second(hub, a, b=7, **kwargs): 5 | pass 6 | 7 | def third(hub, a, b, c, d=3): 8 | pass 9 | 10 | def four(hub, a, b, c, d, *args, e=4): 11 | pass 12 | 13 | def five(hub, a: str, d, *args): 14 | pass 15 | 16 | def six(hub, *args): 17 | pass 18 | 19 | def seven(hub, foo, bar): 20 | pass 21 | -------------------------------------------------------------------------------- /tests/integration/contracted/mods/contracted_access.py: -------------------------------------------------------------------------------- 1 | def two_hubs(hub1, hub2): 2 | return 3 | 4 | 5 | def hub2_dereferenced_call(hub1, hub2): 6 | hub1.mods.contracted_access.hub1_called = True 7 | hub2._.hub2_call() 8 | 9 | 10 | def hub2_direct_call(hub1, hub2): 11 | hub1.mods.contracted_access.hub1_called = True 12 | hub2_call(hub2) 13 | 14 | 15 | def hub2_direct_call_kwargs(hub1, hub2): 16 | hub1.mods.contracted_access.hub1_called = True 17 | hub2_call(h=hub2) 18 | 19 | 20 | def hub2_call(h): 21 | h.mods.contracted_access.hub2_called = True 22 | -------------------------------------------------------------------------------- /tests/integration/contracted/mods/contracts/contracted_access.py: -------------------------------------------------------------------------------- 1 | def post(hub, ctx): 2 | hub.contract_called = True 3 | -------------------------------------------------------------------------------- /tests/integration/contracted/test_contracted_access.py: -------------------------------------------------------------------------------- 1 | # import pop 2 | from pop.hub import Hub 3 | 4 | # Import pytest 5 | import pytest 6 | 7 | 8 | def hub(): 9 | hub = Hub() 10 | hub.pop.sub.add('tests.integration.contracted.mods') 11 | return hub 12 | 13 | 14 | def test_two_hubs(): 15 | h = hub() 16 | 17 | # we should be able to call a function with two hubs as parameters 18 | h.mods.contracted_access.two_hubs(h) 19 | 20 | 21 | def test_contracted_different_mod_dereferenced(): 22 | # create hub 23 | # call hub, 24 | # function on module calls another function with a different hub 25 | # Does that other function use the correct hub? 26 | # Does it call applicable contracts? 27 | # And does it work correctly with a MockHub? 28 | h1 = hub() 29 | h2 = hub() 30 | h1.mods.contracted_access.hub2_dereferenced_call(h2) 31 | 32 | assert h1.mods.contracted_access.hub1_called 33 | assert h2.mods.contracted_access.hub2_called 34 | 35 | assert h1.contract_called 36 | assert h2.contract_called 37 | 38 | # contract_hub = testing.ContractHub(hub) 39 | 40 | 41 | def test_contracted_different_mod_direct(): 42 | h1 = hub() 43 | h2 = hub() 44 | 45 | h1.mods.contracted_access.hub2_direct_call(h2) 46 | 47 | assert h1.mods.contracted_access.hub1_called 48 | assert h2.mods.contracted_access.hub2_called 49 | 50 | assert h1.contract_called 51 | with pytest.raises(AttributeError): 52 | assert h2.contract_called 53 | 54 | # TODO: how should '_' work on mock hubs? does it work correctly? 55 | 56 | 57 | def test_contracted_different_mod_direct_kwargs(): 58 | # create hub 59 | # call hub, 60 | # function on module calls another function with a different hub 61 | # Does that other function use the correct hub? 62 | # Does it call applicable contracts? 63 | # And does it work correctly with a MockHub? 64 | h1 = hub() 65 | h2 = hub() 66 | h1.mods.contracted_access.hub2_direct_call_kwargs(h2) 67 | 68 | assert h1.mods.contracted_access.hub1_called 69 | assert h2.mods.contracted_access.hub2_called 70 | 71 | assert h1.contract_called 72 | with pytest.raises(AttributeError): 73 | assert h2.contract_called 74 | 75 | # contract_hub = testing.ContractHub(hub) 76 | -------------------------------------------------------------------------------- /tests/mods/bad.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ''' 3 | Fail to load to test load errors 4 | ''' 5 | 6 | __virtualname__ = 'bad' 7 | 8 | 9 | def __virtual__(hub): 10 | return 'Failed to load bad' 11 | 12 | 13 | def func(hub): 14 | return 'wha?' 15 | -------------------------------------------------------------------------------- /tests/mods/bad_import/bad_import.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ''' 3 | Fail to load to test load errors 4 | ''' 5 | # pylint: disable=unused-import 6 | 7 | import foobar123456foobar 8 | 9 | 10 | def func(): 11 | return 'wha?' 12 | -------------------------------------------------------------------------------- /tests/mods/contract_ctx/ctx.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | __contracts__ = 'ctx' 4 | 5 | 6 | def test(): 7 | return True 8 | -------------------------------------------------------------------------------- /tests/mods/contract_ctx/ctx_args.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | __contracts__ = 'ctx_args' 4 | 5 | 6 | def test(hub, value, yes=True, no=False): 7 | return value 8 | -------------------------------------------------------------------------------- /tests/mods/contract_ctx/ctx_update.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | __contracts__ = 'ctx_update' 4 | 5 | 6 | def test_call(hub, value): 7 | return value 8 | 9 | 10 | def test_direct(hub, value): 11 | return value 12 | -------------------------------------------------------------------------------- /tests/mods/coro/coro.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | __virtualname__ = 'coro' 4 | 5 | # Import python libs 6 | import asyncio 7 | 8 | # Import third party libs 9 | try: 10 | import tornado.gen 11 | HAS_TORNADO = True 12 | except ImportError: 13 | HAS_TORNADO = False 14 | 15 | 16 | @asyncio.coroutine 17 | def asyncio_demo(): 18 | return True 19 | 20 | 21 | if HAS_TORNADO: 22 | @tornado.gen.coroutine 23 | def tornado_demo(hub): 24 | return False 25 | -------------------------------------------------------------------------------- /tests/mods/foo.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | 4 | def bar(hub): 5 | return hub.mods.test.ping() # pylint: disable=undefined-variable 6 | -------------------------------------------------------------------------------- /tests/mods/iter/bar.py: -------------------------------------------------------------------------------- 1 | def run(hub): 2 | hub.iter.DATA['bar'] = True 3 | -------------------------------------------------------------------------------- /tests/mods/iter/foo.py: -------------------------------------------------------------------------------- 1 | def run(hub): 2 | hub.iter.DATA['foo'] = True 3 | -------------------------------------------------------------------------------- /tests/mods/iter/init.py: -------------------------------------------------------------------------------- 1 | def __init__(hub): 2 | hub.iter.DATA = {} 3 | -------------------------------------------------------------------------------- /tests/mods/many.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | __virtualname__ = 'all' 4 | __contracts__ = 'all' 5 | __func_alias__ = { 6 | 'list_': 'list', 7 | 'dict_': 'dict', 8 | } 9 | 10 | 11 | def list_(hub): 12 | return ['list'] 13 | 14 | 15 | def dict_(hub): 16 | return {} 17 | -------------------------------------------------------------------------------- /tests/mods/nest/basic.py: -------------------------------------------------------------------------------- 1 | def ret_true(hub): 2 | return True 3 | -------------------------------------------------------------------------------- /tests/mods/packinit/init.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ''' 3 | used to test the pack_init system 4 | ''' 5 | # pylint: disable=undefined-variable 6 | 7 | 8 | def __init__(hub): 9 | ''' 10 | Add a value to the context 11 | ''' 12 | hub.context['NEW'] = True 13 | hub.mods._mem['new'] = True 14 | 15 | 16 | def check(hub): 17 | return hub.context.get('NEW') 18 | -------------------------------------------------------------------------------- /tests/mods/packinit/packinit.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # pylint: disable=undefined-variable 3 | 4 | 5 | def __init__(hub): 6 | hub.context['LOADED'] = True 7 | 8 | 9 | def loaded(hub): 10 | return 'LOADED' in hub.context 11 | -------------------------------------------------------------------------------- /tests/mods/priv.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | 4 | def public(hub): 5 | return _private(hub) 6 | 7 | 8 | def _private(hub): 9 | return hub.opts # pylint: disable=undefined-variable 10 | -------------------------------------------------------------------------------- /tests/mods/proc.py: -------------------------------------------------------------------------------- 1 | # Import python libs 2 | import random 3 | 4 | 5 | async def callback(hub, payload): 6 | if 'payload' in payload: 7 | hub.set_me = payload['payload']['ret'] 8 | return 'foo' 9 | 10 | 11 | async def ret(hub): 12 | await hub.proc.worker.ret({'ret': 'Returned'}) 13 | return 'inline' 14 | 15 | 16 | async def gen(hub, start, end): 17 | for x in range(start, end): 18 | yield x 19 | 20 | 21 | def simple_gen(hub, start, end): 22 | for x in range(start, end): 23 | yield x 24 | 25 | 26 | def init_lasts(hub): 27 | hub.LASTS = {} 28 | return True 29 | 30 | 31 | def echo_last(hub): 32 | ''' 33 | Return 2 numbers, the second number is new, the first number is the same that 34 | was the second number the last time it was called 35 | ''' 36 | next_ = random.randint(0, 50000) 37 | if 'last' not in hub.LASTS: 38 | hub.LASTS['last'] = 0 39 | last = hub.LASTS['last'] 40 | hub.LASTS['last'] = next_ 41 | return last, next_ 42 | 43 | 44 | def gen_last(hub, num=5): 45 | for _ in range(num): 46 | next_ = random.randint(0, 50000) 47 | if 'last' not in hub.LASTS: 48 | hub.LASTS['last'] = 0 49 | last = hub.LASTS['last'] 50 | hub.LASTS['last'] = next_ 51 | yield last, next_ 52 | -------------------------------------------------------------------------------- /tests/mods/same_vname/will_load.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | 4 | __virtualname__ = 'vname' 5 | 6 | 7 | def __virtual__(hub): 8 | return True 9 | 10 | 11 | def func(hub): 12 | return 'wha? Yep!' 13 | -------------------------------------------------------------------------------- /tests/mods/same_vname/will_not_load.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | __virtualname__ = 'vname' 4 | 5 | 6 | def __virtual__(hub): 7 | return 'Do not load!' 8 | 9 | 10 | def func(hub): 11 | return 'wha? No! No! No!!!!!' 12 | -------------------------------------------------------------------------------- /tests/mods/test.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Import python libs 4 | import os 5 | 6 | # Import pack libs 7 | import pytest 8 | 9 | from pop.scanner import scan # pylint: disable=unused-import 10 | 11 | __virtualname__ = 'test' 12 | __contracts__ = 'test' 13 | __func_alias__ = {'ping_': 'ping'} 14 | 15 | 16 | def ping_(hub): 17 | return {} 18 | 19 | 20 | def demo(hub): 21 | return False 22 | 23 | 24 | def this(hub): 25 | return hub._.ping() 26 | 27 | 28 | def fqn(hub): 29 | return hub.mods.test.ping() 30 | 31 | 32 | def module_level_non_aliased_ping_call(hub): 33 | return ping_() # pylint: disable=no-value-for-parameter 34 | 35 | 36 | def module_level_non_aliased_ping_call_fw_hub(hub): 37 | return ping_(hub) 38 | 39 | 40 | def attr(hub): 41 | return True 42 | 43 | 44 | attr.bar = True 45 | attr.func = True 46 | 47 | 48 | def call_scan(hub): 49 | # If scan has been packed(wrongly), the call below will throw a TypeError because 50 | # we'll also pass hub 51 | scan([os.path.dirname(__file__)]) 52 | return True 53 | 54 | 55 | def double_underscore(hub): 56 | assert hub.__ is hub.mods 57 | assert hub.___ is hub 58 | assert hub.______ is hub 59 | -------------------------------------------------------------------------------- /tests/mods/testing.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | ''' 4 | module used while testing mock hubs provided in 'testing'. 5 | ''' 6 | 7 | 8 | __contracts__ = ['testing'] 9 | 10 | 11 | def noparam(hub): 12 | pass 13 | 14 | 15 | def echo(hub, param): 16 | return param 17 | 18 | 19 | def signature_func(hub, param1, param2='default'): 20 | pass 21 | 22 | 23 | def attr_func(hub): 24 | pass 25 | 26 | 27 | attr_func.test = True 28 | attr_func.__test__ = True 29 | 30 | 31 | async def async_echo(hub, param): 32 | return param 33 | -------------------------------------------------------------------------------- /tests/mods/vbad.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ''' 3 | Fail to load to test load errors making sure to find the module under 4 | the defined __virtualname__ 5 | ''' 6 | 7 | __virtualname__ = 'virtual_bad' 8 | 9 | 10 | def __virtual__(hub): 11 | return 'Failed to load virtual bad' 12 | 13 | 14 | def func(hub): 15 | return 'wha?' 16 | -------------------------------------------------------------------------------- /tests/mods/virt.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | 4 | def __virtual__(hub): 5 | ''' 6 | ''' 7 | try: 8 | hub.opts # pylint: disable=undefined-variable 9 | except Exception: # pylint: disable=broad-except 10 | return False 11 | return True 12 | 13 | 14 | def present(hub): 15 | return True 16 | -------------------------------------------------------------------------------- /tests/mods/vtrue.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | __virtualname__ = 'truev' 4 | 5 | 6 | def __virtual__(hub): 7 | return True 8 | 9 | 10 | def present(hub): 11 | return True 12 | -------------------------------------------------------------------------------- /tests/sdirs/l11/l2/test.py: -------------------------------------------------------------------------------- 1 | def ping(hub): 2 | return True 3 | -------------------------------------------------------------------------------- /tests/sdirs/l11/test.py: -------------------------------------------------------------------------------- 1 | def ping(hub): 2 | return True 3 | -------------------------------------------------------------------------------- /tests/sdirs/l12/l2/test.py: -------------------------------------------------------------------------------- 1 | def ping(hub): 2 | return True 3 | -------------------------------------------------------------------------------- /tests/sdirs/l12/test.py: -------------------------------------------------------------------------------- 1 | def ping(hub): 2 | return True 3 | -------------------------------------------------------------------------------- /tests/sdirs/l13/l2/test.py: -------------------------------------------------------------------------------- 1 | def ping(hub): 2 | return True 3 | -------------------------------------------------------------------------------- /tests/sdirs/l13/test.py: -------------------------------------------------------------------------------- 1 | def ping(hub): 2 | return True 3 | -------------------------------------------------------------------------------- /tests/sdirs/test.py: -------------------------------------------------------------------------------- 1 | def ping(hub): 2 | return True 3 | -------------------------------------------------------------------------------- /tests/tpath/README.rst: -------------------------------------------------------------------------------- 1 | ====================== 2 | The TPATH testing path 3 | ====================== 4 | 5 | Some features inside of pop can scan the available paths in python and find extra 6 | plugins. This directory gets added to the sys.path in the test run so we can validate 7 | these types of scans -------------------------------------------------------------------------------- /tests/tpath/dn1/conf.py: -------------------------------------------------------------------------------- 1 | DYNE = {'dn1': ['dn1']} 2 | -------------------------------------------------------------------------------- /tests/tpath/dn1/dn1/nest/dn1.py: -------------------------------------------------------------------------------- 1 | def ping(hub): 2 | return True 3 | -------------------------------------------------------------------------------- /tests/tpath/dn2/conf.py: -------------------------------------------------------------------------------- 1 | DYNE = {'dn1': ['dn1']} 2 | -------------------------------------------------------------------------------- /tests/tpath/dn2/dn1/nest/dn2.py: -------------------------------------------------------------------------------- 1 | def ping(hub): 2 | return True 3 | -------------------------------------------------------------------------------- /tests/tpath/dn3/conf.py: -------------------------------------------------------------------------------- 1 | DYNE = {'dn1': ['dn1']} 2 | -------------------------------------------------------------------------------- /tests/tpath/dn3/dn1/nest/dn3.py: -------------------------------------------------------------------------------- 1 | def ping(hub): 2 | return True 3 | -------------------------------------------------------------------------------- /tests/tpath/dn3/dn1/nest/next/last/test.py: -------------------------------------------------------------------------------- 1 | def ping(hub): 2 | return True 3 | -------------------------------------------------------------------------------- /tests/tpath/dn3/dn1/nest/next/test.py: -------------------------------------------------------------------------------- 1 | def ping(hub): 2 | return True 3 | -------------------------------------------------------------------------------- /tests/tpath/dyne1/conf.py: -------------------------------------------------------------------------------- 1 | DYNE = { 2 | 'dyne1': [ 3 | 'dyne1', 4 | 'nest.dyne', 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /tests/tpath/dyne1/dyne1/init.py: -------------------------------------------------------------------------------- 1 | def __init__(hub): 2 | hub.dyne1.INIT = True 3 | hub.pop.sub.add(dyne_name='dyne2') 4 | hub.pop.sub.add(dyne_name='dyne3') 5 | -------------------------------------------------------------------------------- /tests/tpath/dyne1/dyne1/test.py: -------------------------------------------------------------------------------- 1 | def dyne_ping(hub): 2 | return True 3 | -------------------------------------------------------------------------------- /tests/tpath/dyne1/nest/dyne/nest.py: -------------------------------------------------------------------------------- 1 | def nest_dyne_ping(hub): 2 | return True 3 | -------------------------------------------------------------------------------- /tests/tpath/dyne2/conf.py: -------------------------------------------------------------------------------- 1 | DYNE = { 2 | 'dyne2': [ 3 | 'dyne2', 4 | 'nest.dyne', 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /tests/tpath/dyne2/dyne2/init.py: -------------------------------------------------------------------------------- 1 | def __init__(hub): 2 | hub.dyne2.INIT = True 3 | -------------------------------------------------------------------------------- /tests/tpath/dyne2/dyne2/test.py: -------------------------------------------------------------------------------- 1 | def dyne_ping(hub): 2 | return True 3 | -------------------------------------------------------------------------------- /tests/tpath/dyne2/nest/dyne/nest.py: -------------------------------------------------------------------------------- 1 | def nest_dyne_ping(hub): 2 | return True 3 | -------------------------------------------------------------------------------- /tests/tpath/dyne3/conf.py: -------------------------------------------------------------------------------- 1 | DYNE = { 2 | 'dyne3': [ 3 | 'dyne3', 4 | 'nest.dyne', 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /tests/tpath/dyne3/dyne3/init.py: -------------------------------------------------------------------------------- 1 | def __init__(hub): 2 | hub.dyne3.INIT = True 3 | 4 | 5 | def mod_name(hub): 6 | return __name__ 7 | -------------------------------------------------------------------------------- /tests/tpath/dyne3/dyne3/test.py: -------------------------------------------------------------------------------- 1 | def dyne_ping(hub): 2 | return True 3 | -------------------------------------------------------------------------------- /tests/tpath/dyne3/nest/dyne/nest.py: -------------------------------------------------------------------------------- 1 | def nest_dyne_ping(hub): 2 | return True 3 | -------------------------------------------------------------------------------- /tests/unit/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | -------------------------------------------------------------------------------- /tests/unit/test_bench.py: -------------------------------------------------------------------------------- 1 | import pop.hub 2 | repeats = 10000 3 | 4 | 5 | def test_direct(): 6 | hub = pop.hub.Hub() 7 | hub.pop.sub.add('tests.mods') 8 | for i in range(repeats): 9 | hub.mods.test.ping() 10 | 11 | 12 | def test_contract(): 13 | hub = pop.hub.Hub() 14 | hub.pop.sub.add('tests.cmods') 15 | for i in range(repeats): 16 | hub.cmods.ctest.cping() 17 | 18 | 19 | def test_via_underscore(): 20 | hub = pop.hub.Hub() 21 | hub.pop.sub.add('tests.mods') 22 | for i in range(repeats): 23 | hub.mods.test.this() 24 | 25 | 26 | def test_via_fqn(): 27 | hub = pop.hub.Hub() 28 | hub.pop.sub.add('tests.mods') 29 | for i in range(repeats): 30 | hub.mods.test.fqn() 31 | -------------------------------------------------------------------------------- /tests/unit/test_contract.py: -------------------------------------------------------------------------------- 1 | from pop.contract import Contracted 2 | 3 | 4 | def test_contracted_shortcut(): 5 | def f(hub): 6 | pass 7 | 8 | c = Contracted(hub="a hub", contracts=[], func=f, ref=None, name=None) 9 | c.contract_functions['pre'] = [None] # add some garbage so we raise if we try to evaluate contracts 10 | 11 | c() 12 | -------------------------------------------------------------------------------- /tests/unit/test_contract_ctx.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Import pop 4 | import pop.hub 5 | import pytest 6 | 7 | 8 | def test_contract_context(): 9 | hub = pop.hub.Hub() 10 | hub.pop.sub.add( 11 | pypath='tests.mods.contract_ctx', 12 | subname='mods', 13 | contracts_pypath='tests.contracts' 14 | ) 15 | assert hub.mods.ctx.test() == 'contract executed' 16 | # Multiple calls have the same outcome 17 | assert hub.mods.ctx.test() == 'contract executed' 18 | 19 | 20 | def test_contract_context_update_call(): 21 | # if a pre modifies args, make sure they persist when called via 'call' function 22 | hub = pop.hub.Hub() 23 | hub.pop.sub.add( 24 | pypath='tests.mods.contract_ctx', 25 | subname='mods', 26 | contracts_pypath='tests.contracts' 27 | ) 28 | assert hub.mods.ctx_update.test_call(True) == 'contract executed' 29 | # Multiple calls have the same outcome 30 | assert hub.mods.ctx_update.test_call(True) == 'contract executed' 31 | 32 | 33 | def test_contract_context_update_direct(): 34 | # if a pre modifies args, make sure they persist when called directly 35 | hub = pop.hub.Hub() 36 | hub.pop.sub.add( 37 | pypath='tests.mods.contract_ctx', 38 | subname='mods', 39 | contracts_pypath='tests.contracts' 40 | ) 41 | assert hub.mods.ctx_update.test_direct(True) is False 42 | assert hub.mods.ctx_update.test_direct(True) is False 43 | 44 | 45 | def test_contract_ctx_argument_retrieval(): 46 | hub = pop.hub.Hub() 47 | hub.pop.sub.add( 48 | pypath='tests.mods.contract_ctx', 49 | subname='mods', 50 | contracts_pypath='tests.contracts' 51 | ) 52 | assert hub.mods.ctx_args.test('yes', yes=True) is True 53 | assert hub.mods.ctx_args.test('yes', yes=False) is False 54 | assert hub.mods.ctx_args.test('no', no=False) is False 55 | assert hub.mods.ctx_args.test('no', no=True) is True 56 | 57 | with pytest.raises(pop.exc.BindError, 58 | match="got an unexpected keyword argument 'garbage'"): 59 | hub.mods.ctx_args.test('', garbage=True) 60 | -------------------------------------------------------------------------------- /tests/unit/test_coro.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # pylint: disable=expression-not-assigned 3 | 4 | # Import third party libs 5 | try: 6 | import tornado # pylint: disable=unused-import 7 | HAS_TORNADO = True 8 | except ImportError: 9 | HAS_TORNADO = False 10 | 11 | # Import pop 12 | import pop.hub 13 | 14 | 15 | def test_asyncio_coro(): 16 | hub = pop.hub.Hub() 17 | hub.pop.sub.add('tests.mods.coro', 'mods') 18 | assert 'coro' in hub.mods 19 | assert 'asyncio_demo' in dir(hub.mods.coro) 20 | try: 21 | hub.mods.coro.asyncio_demo() 22 | except Exception: 23 | raise 24 | 25 | 26 | if HAS_TORNADO: 27 | def test_tornado_coro(): 28 | hub = pop.hub.Hub() 29 | hub.pop.sub.add('tests.mods.coro', 'mods') 30 | assert 'coro' in hub.mods 31 | assert 'tornado_demo' in dir(hub.mods.coro) 32 | try: 33 | hub.mods.coro.tornado_demo() 34 | except Exception: 35 | raise 36 | -------------------------------------------------------------------------------- /tests/unit/test_fail.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # pylint: disable=expression-not-assigned 3 | 4 | # Import pytest 5 | import pytest 6 | 7 | # Import pop 8 | import pop.exc 9 | import pop.hub 10 | 11 | 12 | def test_load_error(): 13 | ''' 14 | In this test, pop will continue loading although, when trying to 15 | access a functions which should be accessible on the module, a 16 | PopError is raised. 17 | ''' 18 | hub = pop.hub.Hub() 19 | hub.pop.sub.add('tests.mods') 20 | with pytest.raises(pop.exc.PopError, 21 | match='Failed to load bad'): 22 | hub.mods.bad.func() 23 | 24 | 25 | def test_load_error_stop_on_failures(): 26 | hub = pop.hub.Hub() 27 | hub.pop.sub.add( 28 | 'tests.mods', 29 | stop_on_failures=True 30 | ) 31 | with pytest.raises(pop.exc.PopError, 32 | match='returned virtual error'): 33 | hub.mods.bad.func()['verror'] 34 | 35 | 36 | def _test_calling_load_error_raises_pop_error(): 37 | ''' 38 | In this test, pop will continue loading although, when trying to 39 | access a functions which should be accessible on the module, a 40 | PopError is raised. 41 | ''' 42 | hub = pop.hub.Hub() 43 | hub.pop.sub.add( 44 | 'tests.mods', 45 | stop_on_failures=True 46 | ) 47 | with pytest.raises(pop.exc.PopError, 48 | match='Failed to load python module'): 49 | hub.mods.bad_import.func() 50 | 51 | 52 | def test_load_error_traceback_stop_on_failures(): 53 | ''' 54 | In this test case pop will simply stop processing when the error is found 55 | ''' 56 | hub = pop.hub.Hub() 57 | hub.pop.sub.add( 58 | pypath='tests.mods.bad_import', 59 | subname='mods', 60 | stop_on_failures=True) 61 | with pytest.raises(pop.exc.PopError, 62 | match='Failed to load python module'): 63 | hub.mods.bad_import.func() 64 | 65 | 66 | def test_verror_does_not_overload_loaded_mod(): 67 | ''' 68 | This tests will load 2 mods under the vname virtualname, however, one of them 69 | will explicitly not load. This makes sure load errors to not shadow good mod loads 70 | ''' 71 | hub = pop.hub.Hub() 72 | hub.pop.sub.add( 73 | pypath='tests.mods.same_vname', 74 | subname='mods', 75 | ) 76 | assert hub.mods.vname.func() == 'wha? Yep!' 77 | 78 | 79 | def _test_load_error_by_virtualname(): 80 | ''' 81 | This test will make sure that even that the module did not load, it can still be 82 | found under it's defined __virtualname__ 83 | ''' 84 | hub = pop.hub.Hub() 85 | hub.pop.sub.add( 86 | pypath='tests.mods', 87 | subname='mods', 88 | ) 89 | with pytest.raises(pop.exc.PopError, 90 | match='returned virtual error'): 91 | hub.mods.virtual_bad.func() 92 | 93 | with pytest.raises(pop.exc.PopLookupError, 94 | match='Module "vbad" not found'): 95 | hub.mods.vbad.func() 96 | -------------------------------------------------------------------------------- /tests/unit/test_proc.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Test the proc subsystem 3 | ''' 4 | # Import python libs 5 | import tempfile 6 | # Import pop libs 7 | import pop.hub 8 | 9 | 10 | async def _test_create(hub): 11 | name = 'Tests' 12 | await hub.proc.init.pool(3, name, hub.mods.proc.callback, tempfile.mkdtemp()) 13 | #await asyncio.sleep(1) # Give the processes some time to spin up 14 | ret = await hub.proc.run.add_sub(name, 'tests.mods') 15 | # Make sure we round robin all the procs a few times 16 | for ind in range(20): 17 | ret = await hub.proc.run.func(name, 'mods.test.ping') 18 | assert ret == {} 19 | ret_ret = await hub.proc.run.func(name, 'mods.proc.ret') 20 | assert hub.set_me == 'Returned' 21 | assert ret_ret == 'inline' 22 | 23 | # Test iterator systems 24 | n = [] 25 | s = [] 26 | e = [] 27 | async for ind in hub.proc.run.gen(name, 'mods.proc.gen', 23, 77): 28 | n.append(ind) 29 | for ind in range(23, 77): 30 | e.append(ind) 31 | assert n == e 32 | async for ind in hub.proc.run.gen(name, 'mods.proc.simple_gen', 23, 77): 33 | s.append(ind) 34 | assert s == e 35 | # Test pub 36 | assert await hub.proc.run.pub(name, 'mods.proc.init_lasts') 37 | 38 | # Test track and ind func calls 39 | ind, coro = await hub.proc.run.track_func(name, 'mods.proc.echo_last') 40 | last_1, next_1 = await coro 41 | for _ in range(5): 42 | last_2, next_2 = await hub.proc.run.ind_func(name, ind, 'mods.proc.echo_last') 43 | assert next_1 == last_2 44 | next_1 = next_2 45 | # Test gen_track and ind_gen 46 | last_1, next_1 = await hub.proc.run.ind_func(name, ind, 'mods.proc.echo_last') 47 | for _ in range(5): 48 | async for last_2, next_2 in hub.proc.run.ind_gen(name, ind, 'mods.proc.gen_last'): 49 | assert next_1 == last_2 50 | next_1 = next_2 51 | 52 | # Test add_proc 53 | pre = len(hub.proc.Workers[name]) 54 | await hub.proc.run.add_proc(name) 55 | post = len(hub.proc.Workers[name]) 56 | assert pre < post 57 | for _ in range(20): 58 | ret = await hub.proc.run.func(name, 'mods.test.ping') 59 | 60 | 61 | def test_create(): 62 | hub = pop.hub.Hub() 63 | hub.opts = {} 64 | hub.pop.sub.add('pop.mods.proc') 65 | hub.pop.sub.add('tests.mods') 66 | hub.pop.loop.start(_test_create(hub)) 67 | --------------------------------------------------------------------------------