├── .coveragerc ├── .failonoutput.py ├── .github └── workflows │ └── ci.yml ├── .gitignore ├── .pydoctor.cfg ├── .travis ├── build_docs.sh └── install.sh ├── LICENSE ├── README.rst ├── docs ├── Makefile ├── api │ └── index.rst ├── conf.py ├── index.rst ├── listings │ ├── echoflow.py │ ├── echonetstrings.py │ ├── portforward.py │ ├── reversetube.py │ └── rpn.py ├── make.bat ├── spelling_wordlist.txt └── tube.rst ├── setup.cfg ├── setup.py ├── sketches ├── amptube.py ├── fanchat.py └── notes.rst ├── tox.ini └── tubes ├── __init__.py ├── _components.py ├── _siphon.py ├── fan.py ├── framing.py ├── itube.py ├── kit.py ├── listening.py ├── memory.py ├── protocol.py ├── routing.py ├── test ├── __init__.py ├── test_chatter.py ├── test_fan.py ├── test_framing.py ├── test_kit.py ├── test_listening.py ├── test_memory.py ├── test_protocol.py ├── test_routing.py ├── test_tube.py ├── test_undefer.py └── util.py ├── tube.py └── undefer.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | parallel = True 4 | include = 5 | tubes/* 6 | */site-packages/tubes/* 7 | 8 | [paths] 9 | source = 10 | tubes 11 | .tox/*/lib/python*/site-packages/tubes 12 | .tox/*/Lib/site-packages/tubes 13 | .tox/pypy*/site-packages/tubes 14 | 15 | [report] 16 | precision = 2 17 | ignore_errors = True 18 | -------------------------------------------------------------------------------- /.failonoutput.py: -------------------------------------------------------------------------------- 1 | #/usr/bin/env python3 2 | 3 | """ 4 | Report output in real time and then fail if there was any. 5 | 6 | workaround for https://github.com/twisted/twistedchecker/issues/89 7 | """ 8 | 9 | import os 10 | import sys 11 | f = os.popen(sys.argv[1]) 12 | data = f.read(1024) 13 | ever = False 14 | while data: 15 | sys.stdout.write(data) 16 | data = f.read(1024) 17 | ever = True 18 | if ever: 19 | sys.exit(1) 20 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: ci 2 | 3 | on: 4 | push: 5 | branches: 6 | - trunk 7 | - gh-pages 8 | 9 | pull_request: 10 | branches: 11 | - trunk 12 | - gh-pages 13 | 14 | jobs: 15 | build: 16 | name: ${{ matrix.TOX_ENV }} 17 | runs-on: ubuntu-latest 18 | strategy: 19 | fail-fast: false 20 | matrix: 21 | python: ["3.8", "3.11", "3.12"] 22 | include: 23 | - TOX_ENV: "lint" 24 | python: "3.11" 25 | - TOX_ENV: "py38" 26 | python: "3.8" 27 | - TOX_ENV: "py312" 28 | python: "3.12" 29 | - TOX_ENV: "py311" 30 | python: "3.11" 31 | - TOX_ENV: docs 32 | python: "3.11" 33 | - TOX_ENV: apidocs 34 | python: "3.11" 35 | - TOX_ENV: docs-spellcheck 36 | python: "3.11" 37 | - TOX_ENV: docs-linkcheck 38 | python: "3.11" 39 | allow_failures: 40 | - TOX_ENV: "docs-linkcheck" 41 | 42 | steps: 43 | - uses: actions/checkout@v3 44 | - name: Set up Python 45 | uses: actions/setup-python@v3 46 | with: 47 | python-version: ${{ matrix.python }} 48 | - name: Install 49 | run: source ./.travis/install.sh 50 | 51 | - name: Tox Run 52 | run: | 53 | TOX_ENV="${{ matrix.TOX_ENV }}"; 54 | echo "Starting: ${TOX_ENV} ${PUSH_DOCS}" 55 | if [[ -n "${TOX_ENV}" ]]; then 56 | tox -e "$TOX_ENV"; 57 | fi 58 | if [[ "$PUSH_DOCS" == "true" ]]; then 59 | ./.travis/build_docs.sh; 60 | fi; 61 | 62 | - name: after_success 63 | run: | 64 | if [[ "${TOX_ENV:0:2}" == 'py' ]]; then tox -e coveralls-push; fi 65 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /docs/_build 2 | .tox 3 | /dist 4 | /build 5 | twisted.system 6 | zope.interface.system 7 | apidocs/ 8 | *.egg-info 9 | _trial_temp* 10 | -------------------------------------------------------------------------------- /.pydoctor.cfg: -------------------------------------------------------------------------------- 1 | [tool:pydoctor] 2 | quiet=1 3 | warnings-as-errors=true 4 | project-name=Tubes 5 | project-url=https://github.com/twisted/tubes/ 6 | docformat=epytext 7 | theme=readthedocs 8 | intersphinx= 9 | https://docs.python.org/3/objects.inv 10 | https://cryptography.io/en/latest/objects.inv 11 | https://pyopenssl.readthedocs.io/en/stable/objects.inv 12 | https://hyperlink.readthedocs.io/en/stable/objects.inv 13 | https://twisted.org/constantly/docs/objects.inv 14 | https://twisted.org/incremental/docs/objects.inv 15 | https://python-hyper.org/projects/hyper-h2/en/stable/objects.inv 16 | https://priority.readthedocs.io/en/stable/objects.inv 17 | https://zopeinterface.readthedocs.io/en/latest/objects.inv 18 | https://automat.readthedocs.io/en/latest/objects.inv 19 | https://docs.twisted.org/en/stable/objects.inv 20 | project-base-dir=tubes 21 | html-output=docs/_build/api 22 | html-viewsource-base=https://github.com/twisted/tubes/tree/trunk 23 | -------------------------------------------------------------------------------- /.travis/build_docs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [[ ${TRAVIS_PULL_REQUEST} == "false" ]] && [[ ${TRAVIS_BRANCH} == "master" ]]; then 4 | 5 | echo "uploading docs" 6 | 7 | REV=`git rev-parse HEAD` 8 | 9 | # Build the docs 10 | tox -e apidocs 11 | 12 | # Make the directory 13 | git clone --branch gh-pages https://github.com/twisted/tubes.git /tmp/tmp-docs 14 | 15 | # Copy the docs 16 | rsync -rt --del --exclude=".git" apidocs/* /tmp/tmp-docs/docs/ 17 | 18 | cd /tmp/tmp-docs 19 | 20 | git add -A 21 | 22 | # set the username and email. The secure line in travis.yml that sets 23 | # these environment variables is created by: 24 | 25 | # travis encrypt 'GIT_NAME="HawkOwl (Automatic)" GIT_EMAIL=hawkowl@atleastfornow.net GH_TOKEN=' 26 | 27 | export GIT_COMMITTER_NAME="${GIT_NAME}"; 28 | export GIT_COMMITTER_EMAIL="${GIT_EMAIL}"; 29 | export GIT_AUTHOR_NAME="${GIT_NAME}"; 30 | export GIT_AUTHOR_EMAIL="${GIT_EMAIL}"; 31 | 32 | git commit -m "Built from ${REV}"; 33 | 34 | # Push it up 35 | git push -q "https://${GH_TOKEN}@github.com/twisted/tubes.git" gh-pages 36 | else 37 | echo "skipping docs upload" 38 | fi; 39 | -------------------------------------------------------------------------------- /.travis/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Getting around a buggy PyPy in Travis 4 | # Script from pyca/cryptography 5 | 6 | set -e 7 | set -x 8 | 9 | if [[ "${TOX_ENV}" == "pypy"* ]]; then 10 | sudo add-apt-repository -y ppa:pypy/ppa 11 | sudo apt-get -y update 12 | sudo apt-get install -y pypy pypy-dev 13 | 14 | # This is required because we need to get rid of the Travis installed PyPy 15 | # or it'll take precedence over the PPA installed one. 16 | sudo rm -rf /usr/local/pypy/bin 17 | fi 18 | 19 | if [[ "${TOX_ENV}" == "docs-spellcheck" ]]; then 20 | if [[ "$DARWIN" = true ]]; then 21 | brew update 22 | brew install enchant 23 | else 24 | sudo apt-get -y update 25 | sudo apt-get install libenchant-dev 26 | fi 27 | fi 28 | 29 | pip install tox coveralls 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2001-2014 2 | Allen Short 3 | Andy Gayton 4 | Andrew Bennetts 5 | Antoine Pitrou 6 | Apple Computer, Inc. 7 | Ashwini Oruganti 8 | Benjamin Bruheim 9 | Bob Ippolito 10 | Canonical Limited 11 | Christopher Armstrong 12 | David Reid 13 | Donovan Preston 14 | Eric Mangold 15 | Eyal Lotem 16 | Google Inc. 17 | Hybrid Logic Ltd. 18 | Hynek Schlawack 19 | Itamar Turner-Trauring 20 | James Knight 21 | Jason A. Mobarak 22 | Jean-Paul Calderone 23 | Jessica McKellar 24 | Jonathan Jacobs 25 | Jonathan Lange 26 | Jonathan D. Simms 27 | Jürgen Hermann 28 | Julian Berman 29 | Kevin Horn 30 | Kevin Turner 31 | Laurens Van Houtven 32 | Mary Gardiner 33 | Matthew Lefkowitz 34 | Massachusetts Institute of Technology 35 | Moshe Zadka 36 | Paul Swartz 37 | Pavel Pergamenshchik 38 | Ralph Meijer 39 | Richard Wall 40 | Sean Riley 41 | Software Freedom Conservancy 42 | Travis B. Hartwell 43 | Thijs Triemstra 44 | Thomas Herve 45 | Timothy Allen 46 | Tom Prince 47 | 48 | Permission is hereby granted, free of charge, to any person obtaining 49 | a copy of this software and associated documentation files (the 50 | "Software"), to deal in the Software without restriction, including 51 | without limitation the rights to use, copy, modify, merge, publish, 52 | distribute, sublicense, and/or sell copies of the Software, and to 53 | permit persons to whom the Software is furnished to do so, subject to 54 | the following conditions: 55 | 56 | The above copyright notice and this permission notice shall be 57 | included in all copies or substantial portions of the Software. 58 | 59 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 60 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 61 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 62 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 63 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 64 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 65 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 66 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Tubes 2 | ===== 3 | 4 | .. image:: https://img.shields.io/pypi/v/Tubes.svg 5 | :target: https://pypi.python.org/pypi/Tubes/ 6 | :alt: Latest Version 7 | 8 | .. image:: https://readthedocs.org/projects/tubes/badge/?version=latest 9 | :target: https://tubes.readthedocs.org/ 10 | :alt: Latest Docs 11 | 12 | .. image:: https://travis-ci.org/twisted/tubes.svg?branch=master 13 | :target: https://travis-ci.org/twisted/tubes 14 | 15 | .. image:: https://img.shields.io/coveralls/twisted/tubes/master.svg 16 | :target: https://coveralls.io/r/twisted/tubes?branch=master 17 | 18 | "Tubes" is a data-processing and flow-control engine for event-driven programs. 19 | 20 | Presently based primarily on Twisted, its core data structures are fairly 21 | framework-agnostic and could be repurposed to work with any event-driven 22 | container. 23 | 24 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from https://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 36 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 38 | @echo " text to make text files" 39 | @echo " man to make manual pages" 40 | @echo " texinfo to make Texinfo files" 41 | @echo " info to make Texinfo files and run them through makeinfo" 42 | @echo " gettext to make PO message catalogs" 43 | @echo " changes to make an overview of all changed/added/deprecated items" 44 | @echo " xml to make Docutils-native XML files" 45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 46 | @echo " linkcheck to check all external links for integrity" 47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 48 | 49 | clean: 50 | rm -rf $(BUILDDIR)/* 51 | 52 | html: 53 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 54 | @echo 55 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 56 | 57 | dirhtml: 58 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 59 | @echo 60 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 61 | 62 | singlehtml: 63 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 64 | @echo 65 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 66 | 67 | pickle: 68 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 69 | @echo 70 | @echo "Build finished; now you can process the pickle files." 71 | 72 | json: 73 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 74 | @echo 75 | @echo "Build finished; now you can process the JSON files." 76 | 77 | htmlhelp: 78 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 79 | @echo 80 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 81 | ".hhp project file in $(BUILDDIR)/htmlhelp." 82 | 83 | qthelp: 84 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 85 | @echo 86 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 87 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 88 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Tubes.qhcp" 89 | @echo "To view the help file:" 90 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Tubes.qhc" 91 | 92 | devhelp: 93 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 94 | @echo 95 | @echo "Build finished." 96 | @echo "To view the help file:" 97 | @echo "# mkdir -p $$HOME/.local/share/devhelp/Tubes" 98 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Tubes" 99 | @echo "# devhelp" 100 | 101 | epub: 102 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 103 | @echo 104 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 105 | 106 | latex: 107 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 108 | @echo 109 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 110 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 111 | "(use \`make latexpdf' here to do that automatically)." 112 | 113 | latexpdf: 114 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 115 | @echo "Running LaTeX files through pdflatex..." 116 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 117 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 118 | 119 | latexpdfja: 120 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 121 | @echo "Running LaTeX files through platex and dvipdfmx..." 122 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 123 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 124 | 125 | text: 126 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 127 | @echo 128 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 129 | 130 | man: 131 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 132 | @echo 133 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 134 | 135 | texinfo: 136 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 137 | @echo 138 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 139 | @echo "Run \`make' in that directory to run these through makeinfo" \ 140 | "(use \`make info' here to do that automatically)." 141 | 142 | info: 143 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 144 | @echo "Running Texinfo files through makeinfo..." 145 | make -C $(BUILDDIR)/texinfo info 146 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 147 | 148 | gettext: 149 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 150 | @echo 151 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 152 | 153 | changes: 154 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 155 | @echo 156 | @echo "The overview file is in $(BUILDDIR)/changes." 157 | 158 | linkcheck: 159 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 160 | @echo 161 | @echo "Link check complete; look for any errors in the above output " \ 162 | "or in $(BUILDDIR)/linkcheck/output.txt." 163 | 164 | doctest: 165 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 166 | @echo "Testing of doctests in the sources finished, look at the " \ 167 | "results in $(BUILDDIR)/doctest/output.txt." 168 | 169 | xml: 170 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 171 | @echo 172 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 173 | 174 | pseudoxml: 175 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 176 | @echo 177 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 178 | -------------------------------------------------------------------------------- /docs/api/index.rst: -------------------------------------------------------------------------------- 1 | API Reference 2 | ============= 3 | 4 | This file will be overwritten by the pydoctor build triggered at the end 5 | of the Sphinx build. 6 | 7 | It's a hack to be able to reference the API index page from inside Sphinx 8 | and have it as part of the TOC. 9 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # For the full list of built-in configuration values, see the documentation: 4 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 5 | 6 | # -- Project information ----------------------------------------------------- 7 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information 8 | 9 | import os 10 | from pprint import pprint 11 | import subprocess 12 | 13 | project = "Tubes" 14 | copyright = "2023, Glyph" 15 | author = "Glyph" 16 | 17 | # -- General configuration --------------------------------------------------- 18 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration 19 | 20 | extensions = [ 21 | "sphinx.ext.intersphinx", 22 | "pydoctor.sphinx_ext.build_apidocs", 23 | ] 24 | 25 | import pathlib 26 | 27 | _project_root = pathlib.Path(__file__).parent.parent 28 | 29 | # -- Extension configuration ---------------------------------------------- 30 | _git_reference = subprocess.run( 31 | ["git", "rev-parse", "--abbrev-ref", "HEAD"], 32 | text=True, 33 | encoding="utf8", 34 | capture_output=True, 35 | check=True, 36 | ).stdout 37 | 38 | 39 | print(f"== Environment dump for {_git_reference} ===") 40 | pprint(dict(os.environ)) 41 | print("======") 42 | 43 | 44 | # Try to find URL fragment for the GitHub source page based on current 45 | # branch or tag. 46 | 47 | if _git_reference == "HEAD": 48 | # It looks like the branch has no name. 49 | # Fallback to commit ID. 50 | _git_reference = subprocess.getoutput("git rev-parse HEAD") 51 | 52 | if os.environ.get("READTHEDOCS", "") == "True": 53 | rtd_version = os.environ.get("READTHEDOCS_VERSION", "") 54 | if "." in rtd_version: 55 | # It looks like we have a tag build. 56 | _git_reference = rtd_version 57 | 58 | _source_root = _project_root 59 | 60 | pydoctor_args = [ 61 | # pydoctor should not fail the sphinx build, we have another tox environment for that. 62 | "--intersphinx=https://docs.twisted.org/en/twisted-22.1.0/api/objects.inv", 63 | "--intersphinx=https://docs.python.org/3/objects.inv", 64 | "--intersphinx=https://zopeinterface.readthedocs.io/en/latest/objects.inv", 65 | # TODO: not sure why I have to specify these all twice. 66 | 67 | f"--config={_project_root}/.pydoctor.cfg", 68 | f"--html-viewsource-base=https://github.com/twisted/tubes/tree/{_git_reference}", 69 | f"--project-base-dir={_source_root}", 70 | "--html-output={outdir}/api", 71 | "--privacy=HIDDEN:tubes.test.*", 72 | "--privacy=HIDDEN:tubes.test", 73 | "--privacy=HIDDEN:**.__post_init__", 74 | str(_source_root / "tubes"), 75 | ] 76 | pydoctor_url_path = "/en/{rtd_version}/api/" 77 | intersphinx_mapping = { 78 | "py3": ("https://docs.python.org/3", None), 79 | "zopeinterface": ("https://zopeinterface.readthedocs.io/en/latest", None), 80 | "twisted": ("https://docs.twisted.org/en/twisted-22.1.0/api", None), 81 | } 82 | 83 | templates_path = ["_templates"] 84 | exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] 85 | 86 | 87 | # -- Options for HTML output ------------------------------------------------- 88 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output 89 | 90 | html_theme = "alabaster" 91 | htmlhelp_basename = "Tubesdoc" 92 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. Tubes documentation master file, created by 2 | sphinx-quickstart on Fri Dec 26 15:38:44 2014. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to Tubes's documentation! 7 | ================================= 8 | 9 | Contents: 10 | 11 | .. toctree:: 12 | :maxdepth: 2 13 | 14 | tube.rst 15 | api/index 16 | 17 | 18 | 19 | Indices and tables 20 | ================== 21 | 22 | * :ref:`genindex` 23 | * :ref:`modindex` 24 | * :ref:`search` 25 | 26 | -------------------------------------------------------------------------------- /docs/listings/echoflow.py: -------------------------------------------------------------------------------- 1 | from tubes.protocol import flowFountFromEndpoint 2 | from tubes.listening import Listener 3 | 4 | from twisted.internet.endpoints import serverFromString 5 | from twisted.internet.defer import Deferred, inlineCallbacks 6 | 7 | def echo(flow): 8 | flow.fount.flowTo(flow.drain) 9 | 10 | @inlineCallbacks 11 | def main(reactor, listenOn="stdio:"): 12 | listener = Listener(echo) 13 | endpoint = serverFromString(reactor, listenOn) 14 | flowFount = yield flowFountFromEndpoint(endpoint) 15 | flowFount.flowTo(listener) 16 | yield Deferred() 17 | 18 | if __name__ == '__main__': 19 | from twisted.internet.task import react 20 | from sys import argv 21 | react(main, argv[1:]) 22 | -------------------------------------------------------------------------------- /docs/listings/echonetstrings.py: -------------------------------------------------------------------------------- 1 | from tubes.tube import Tube 2 | from tubes.framing import stringsToNetstrings 3 | from tubes.protocol import flowFountFromEndpoint 4 | from tubes.listening import Listener 5 | 6 | from twisted.internet.endpoints import TCP4ServerEndpoint 7 | from twisted.internet.defer import inlineCallbacks, Deferred() 8 | 9 | def echoTubeFactory(flow): 10 | return (flow.fount.flowTo(Tube(stringsToNetstrings())) 11 | .flowTo(flow.drain)) 12 | 13 | @inlineCallbacks 14 | def main(reactor): 15 | endpoint = TCP4ServerEndpoint(reactor, 4321) 16 | flowFount = yield flowFountFromEndpoint(endpoint) 17 | flowFount.flowTo(Listener(echoTubeFactory)) 18 | yield Deferred() 19 | 20 | if __name__ == '__main__': 21 | from twisted.internet.task import react 22 | react(main, []) 23 | -------------------------------------------------------------------------------- /docs/listings/portforward.py: -------------------------------------------------------------------------------- 1 | from tubes.protocol import flowFountFromEndpoint, flowFromEndpoint 2 | from tubes.listening import Listener 3 | 4 | from twisted.internet.endpoints import serverFromString, clientFromString 5 | from twisted.internet.defer import Deferred, inlineCallbacks 6 | 7 | @inlineCallbacks 8 | def main(reactor, listen="tcp:4321", connect="tcp:localhost:6543"): 9 | clientEndpoint = clientFromString(reactor, connect) 10 | serverEndpoint = serverFromString(reactor, listen) 11 | 12 | def incoming(listening): 13 | def outgoing(connecting): 14 | listening.fount.flowTo(connecting.drain) 15 | connecting.fount.flowTo(listening.drain) 16 | flowFromEndpoint(clientEndpoint).addCallback(outgoing) 17 | flowFount = yield flowFountFromEndpoint(serverEndpoint) 18 | flowFount.flowTo(Listener(incoming)) 19 | yield Deferred() 20 | 21 | if __name__ == '__main__': 22 | from twisted.internet.task import react 23 | from sys import argv 24 | react(main, argv[1:]) 25 | -------------------------------------------------------------------------------- /docs/listings/reversetube.py: -------------------------------------------------------------------------------- 1 | from twisted.internet.endpoints import serverFromString 2 | from twisted.internet.defer import Deferred, inlineCallbacks 3 | 4 | from tubes.protocol import flowFountFromEndpoint 5 | from tubes.listening import Listener 6 | from tubes.tube import tube, series 7 | 8 | @tube 9 | class Reverser(object): 10 | def received(self, item): 11 | yield b"".join(reversed(item)) 12 | 13 | def reverseFlow(flow): 14 | from tubes.framing import bytesToLines, linesToBytes 15 | lineReverser = series(bytesToLines(), Reverser(), linesToBytes()) 16 | flow.fount.flowTo(lineReverser).flowTo(flow.drain) 17 | 18 | @inlineCallbacks 19 | def main(reactor, listenOn="stdio:"): 20 | endpoint = serverFromString(reactor, listenOn) 21 | flowFount = yield flowFountFromEndpoint(endpoint) 22 | flowFount.flowTo(Listener(reverseFlow)) 23 | yield Deferred() 24 | 25 | if __name__ == '__main__': 26 | from twisted.internet.task import react 27 | from sys import argv 28 | react(main, argv[1:]) 29 | -------------------------------------------------------------------------------- /docs/listings/rpn.py: -------------------------------------------------------------------------------- 1 | from tubes.itube import IFrame, ISegment 2 | from tubes.tube import tube, receiver 3 | from tubes.listening import Listener 4 | 5 | from twisted.internet.endpoints import serverFromString 6 | from twisted.internet.defer import Deferred, inlineCallbacks 7 | from tubes.protocol import flowFountFromEndpoint 8 | 9 | class Calculator(object): 10 | def __init__(self): 11 | self.stack = [] 12 | 13 | def push(self, number): 14 | self.stack.append(number) 15 | 16 | def do(self, operator): 17 | if len(self.stack) < 2: 18 | return "UNDERFLOW" 19 | left = self.stack.pop() 20 | right = self.stack.pop() 21 | result = operator(left, right) 22 | self.push(result) 23 | return result 24 | 25 | err = object() 26 | 27 | @receiver(inputType=IFrame) 28 | def linesToNumbersOrOperators(line): 29 | from operator import add, mul 30 | try: 31 | yield int(line) 32 | except ValueError: 33 | if line == b'+': 34 | yield add 35 | elif line == b'*': 36 | yield mul 37 | else: 38 | yield err 39 | 40 | @tube 41 | class CalculatingTube(object): 42 | def __init__(self, calculator): 43 | self.calculator = calculator 44 | 45 | def received(self, value): 46 | if isinstance(value, int): 47 | self.calculator.push(value) 48 | elif value is err: 49 | yield "SYNTAX" 50 | else: 51 | yield self.calculator.do(value) 52 | 53 | @receiver() 54 | def numbersToLines(value): 55 | yield str(value).encode("ascii") 56 | 57 | @tube 58 | class Prompter(object): 59 | outputType = ISegment 60 | def started(self): 61 | yield b"> " 62 | def received(self, item): 63 | yield b"> " 64 | def stopped(self, failure): 65 | yield b"BYE" 66 | 67 | def promptingCalculatorSeries(): 68 | from tubes.fan import Thru 69 | from tubes.tube import series 70 | from tubes.framing import bytesToLines, linesToBytes 71 | 72 | full = series(bytesToLines(), 73 | Thru([series(linesToNumbersOrOperators, 74 | CalculatingTube(Calculator()), 75 | numbersToLines, 76 | linesToBytes()), 77 | series(Prompter())])) 78 | return full 79 | 80 | def calculatorSeries(): 81 | from tubes.tube import series 82 | from tubes.framing import bytesToLines, linesToBytes 83 | 84 | return series( 85 | bytesToLines(), 86 | linesToNumbersOrOperators, 87 | CalculatingTube(Calculator()), 88 | numbersToLines, 89 | linesToBytes() 90 | ) 91 | 92 | def mathFlow(flow): 93 | processor = promptingCalculatorSeries() 94 | nextDrain = flow.fount.flowTo(processor) 95 | nextDrain.flowTo(flow.drain) 96 | 97 | @inlineCallbacks 98 | def main(reactor, port="stdio:"): 99 | endpoint = serverFromString(reactor, port) 100 | flowFount = yield flowFountFromEndpoint(endpoint) 101 | flowFount.flowTo(Listener(mathFlow)) 102 | yield Deferred() 103 | 104 | if __name__ == '__main__': 105 | from twisted.internet.task import react 106 | from sys import argv 107 | react(main, argv[1:]) 108 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. xml to make Docutils-native XML files 37 | echo. pseudoxml to make pseudoxml-XML files for display purposes 38 | echo. linkcheck to check all external links for integrity 39 | echo. doctest to run all doctests embedded in the documentation if enabled 40 | goto end 41 | ) 42 | 43 | if "%1" == "clean" ( 44 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 45 | del /q /s %BUILDDIR%\* 46 | goto end 47 | ) 48 | 49 | 50 | %SPHINXBUILD% 2> nul 51 | if errorlevel 9009 ( 52 | echo. 53 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 54 | echo.installed, then set the SPHINXBUILD environment variable to point 55 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 56 | echo.may add the Sphinx directory to PATH. 57 | echo. 58 | echo.If you don't have Sphinx installed, grab it from 59 | echo.https://sphinx-doc.org/ 60 | exit /b 1 61 | ) 62 | 63 | if "%1" == "html" ( 64 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 65 | if errorlevel 1 exit /b 1 66 | echo. 67 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 68 | goto end 69 | ) 70 | 71 | if "%1" == "dirhtml" ( 72 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 73 | if errorlevel 1 exit /b 1 74 | echo. 75 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 76 | goto end 77 | ) 78 | 79 | if "%1" == "singlehtml" ( 80 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 81 | if errorlevel 1 exit /b 1 82 | echo. 83 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 84 | goto end 85 | ) 86 | 87 | if "%1" == "pickle" ( 88 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 89 | if errorlevel 1 exit /b 1 90 | echo. 91 | echo.Build finished; now you can process the pickle files. 92 | goto end 93 | ) 94 | 95 | if "%1" == "json" ( 96 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 97 | if errorlevel 1 exit /b 1 98 | echo. 99 | echo.Build finished; now you can process the JSON files. 100 | goto end 101 | ) 102 | 103 | if "%1" == "htmlhelp" ( 104 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 105 | if errorlevel 1 exit /b 1 106 | echo. 107 | echo.Build finished; now you can run HTML Help Workshop with the ^ 108 | .hhp project file in %BUILDDIR%/htmlhelp. 109 | goto end 110 | ) 111 | 112 | if "%1" == "qthelp" ( 113 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 114 | if errorlevel 1 exit /b 1 115 | echo. 116 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 117 | .qhcp project file in %BUILDDIR%/qthelp, like this: 118 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\Tubes.qhcp 119 | echo.To view the help file: 120 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\Tubes.ghc 121 | goto end 122 | ) 123 | 124 | if "%1" == "devhelp" ( 125 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished. 129 | goto end 130 | ) 131 | 132 | if "%1" == "epub" ( 133 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 134 | if errorlevel 1 exit /b 1 135 | echo. 136 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 137 | goto end 138 | ) 139 | 140 | if "%1" == "latex" ( 141 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 142 | if errorlevel 1 exit /b 1 143 | echo. 144 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 145 | goto end 146 | ) 147 | 148 | if "%1" == "latexpdf" ( 149 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 150 | cd %BUILDDIR%/latex 151 | make all-pdf 152 | cd %BUILDDIR%/.. 153 | echo. 154 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 155 | goto end 156 | ) 157 | 158 | if "%1" == "latexpdfja" ( 159 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 160 | cd %BUILDDIR%/latex 161 | make all-pdf-ja 162 | cd %BUILDDIR%/.. 163 | echo. 164 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 165 | goto end 166 | ) 167 | 168 | if "%1" == "text" ( 169 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 170 | if errorlevel 1 exit /b 1 171 | echo. 172 | echo.Build finished. The text files are in %BUILDDIR%/text. 173 | goto end 174 | ) 175 | 176 | if "%1" == "man" ( 177 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 178 | if errorlevel 1 exit /b 1 179 | echo. 180 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 181 | goto end 182 | ) 183 | 184 | if "%1" == "texinfo" ( 185 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 186 | if errorlevel 1 exit /b 1 187 | echo. 188 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 189 | goto end 190 | ) 191 | 192 | if "%1" == "gettext" ( 193 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 194 | if errorlevel 1 exit /b 1 195 | echo. 196 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 197 | goto end 198 | ) 199 | 200 | if "%1" == "changes" ( 201 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 202 | if errorlevel 1 exit /b 1 203 | echo. 204 | echo.The overview file is in %BUILDDIR%/changes. 205 | goto end 206 | ) 207 | 208 | if "%1" == "linkcheck" ( 209 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 210 | if errorlevel 1 exit /b 1 211 | echo. 212 | echo.Link check complete; look for any errors in the above output ^ 213 | or in %BUILDDIR%/linkcheck/output.txt. 214 | goto end 215 | ) 216 | 217 | if "%1" == "doctest" ( 218 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 219 | if errorlevel 1 exit /b 1 220 | echo. 221 | echo.Testing of doctests in the sources finished, look at the ^ 222 | results in %BUILDDIR%/doctest/output.txt. 223 | goto end 224 | ) 225 | 226 | if "%1" == "xml" ( 227 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 228 | if errorlevel 1 exit /b 1 229 | echo. 230 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 231 | goto end 232 | ) 233 | 234 | if "%1" == "pseudoxml" ( 235 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 236 | if errorlevel 1 exit /b 1 237 | echo. 238 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 239 | goto end 240 | ) 241 | 242 | :end 243 | -------------------------------------------------------------------------------- /docs/spelling_wordlist.txt: -------------------------------------------------------------------------------- 1 | iterable 2 | composable 3 | indices 4 | -------------------------------------------------------------------------------- /docs/tube.rst: -------------------------------------------------------------------------------- 1 | 2 | An Introduction to Tubes 3 | ======================== 4 | 5 | What Are Tubes? 6 | --------------- 7 | 8 | The :py:mod:`tubes ` package provides composable flow-control and data processing. 9 | 10 | Flow-control is control over the source, destination, and rate of data being processed. 11 | Tubes implements this in a type-agnostic way, meaning that a set of rules for controlling the flow of data can control that flow regardless of the type of that data, from raw streams of bytes to application-specific messages and back again. 12 | 13 | Composable data processing refers to processing that can occur in independent units. 14 | For example, the conversion of a continuous stream of bytes into a discrete sequence of messages can be implemented independently from the presentation of or reactions to those messages. 15 | This allows for similar messages to be relayed in different formats and by different protocols, but be processed by the same code. 16 | 17 | In this document, you will learn how to compose founts (places where data comes from), drains (places where data goes to), and tubes (things that modify data by converting inputs to outputs). 18 | You'll also learn how to create your own tubes to perform your own conversions of input to output. 19 | By the end, you should be able to put a series of tubes onto the Internet as a server or a client. 20 | 21 | 22 | Getting Connected: an Echo Server 23 | --------------------------------- 24 | 25 | Let's start with an example. 26 | The simplest way to process any data is to avoid processing it entirely, to pass input straight on to output. 27 | On a network, that means an echo server as described in :rfc:`862`. 28 | Here's a function which uses interfaces defined by ``tubes`` to send its input straight on to its output: 29 | 30 | .. literalinclude:: listings/echoflow.py 31 | :pyobject: echo 32 | 33 | In the above example, ``echo`` requires a :py:class:`flow ` as an argument. 34 | A ``Flow`` represents the connection that we just received: a stream of inbound data, which we call a fount, and a stream of outbound data, which we call a drain. 35 | As such, it has 2 attributes: ``.fount``, which is a :py:class:`fount `, or a source of data, and ``.drain``, which is a :py:class:`drain ` , or a place where data eventually goes. 36 | This object is called a "flow", because it establishes a flow of data from one place to and from another. 37 | 38 | Let's look at the full example that turns ``echo`` into a real server. 39 | 40 | :download:`echoflow.py ` 41 | 42 | .. literalinclude:: listings/echoflow.py 43 | 44 | To *use* ``echo`` as a server, first we have to tell Tubes that it's a :py:func:`drain ` that wants :py:func:`flow `\ s. 45 | We do this by wrapping it in a :py:func:`Listener `\ . 46 | 47 | Next, we need to actually listen on a port: we do this with Twisted's `"endpoints" API `_ ; specifically, we use ``serverFromString`` on the string ``"stdio:"`` by default, which treats the console as an incoming connection so we can type directly into it, and see the results as output. 48 | 49 | Next, we need to convert this endpoint into a :py:func:`fount ` with an ``outputType`` of :py:func:`Flow `. 50 | To do this, we use the aptly named :py:func:`flowFountFromEndpoint `. 51 | 52 | Finally, we connect the listening socket with our application via ``flowFount.flowTo(listener)``\ . 53 | 54 | This fully-functioning example (just run it with "``python echoflow.py``") implements an echo server. 55 | By default, you can test it out by typing into it. 56 | 57 | .. code-block:: console 58 | 59 | $ python echoflow.py 60 | are you an echo server? 61 | are you an echo server? 62 | ^C 63 | 64 | If you want to see this run on a network, you can give it an endpoint description. 65 | For example, to run on TCP port 4321: 66 | 67 | .. code-block:: console 68 | 69 | $ python echoflow.py tcp:4321 70 | 71 | and then in another command-line window: 72 | 73 | .. code-block:: console 74 | 75 | $ telnet 127.0.0.1 4321 76 | Trying 127.0.0.1... 77 | Connected to localhost. 78 | Escape character is '^]'. 79 | are you an echo server? 80 | are you an echo server? 81 | ^] 82 | telnet> close 83 | Connection closed. 84 | 85 | You can test it out with ``telnet localhost 4321``. 86 | 87 | .. note:: 88 | 89 | If you are on Windows, ``telnet`` is not installed by default. 90 | If you see an error message like: 91 | 92 | .. code-block:: console 93 | 94 | 'telnet' is not recognized as an internal or external command, 95 | operable program or batch file. 96 | 97 | then you can install ``telnet`` by running the command 98 | 99 | .. code-block:: console 100 | 101 | C:\> dism /online /Enable-Feature /FeatureName:TelnetClient 102 | 103 | in an Administrator command-prompt first. 104 | 105 | However, this example still performs no processing of the data that it is receiving. 106 | 107 | A Brief Aside About Types 108 | ^^^^^^^^^^^^^^^^^^^^^^^^^ 109 | 110 | Each fount, and each drain, have a type associated with them: in the fount's case, the type of data it produces, and in the drain's case, the type of data that it accepts. 111 | You can inspect these using the :py:func:`outputType ` and :py:func:`inputType ` attributes of founts and drains respectively. 112 | Even in our tiny example, we already have two types of founts: a fount of ``bytes`` — one for each connection — and a fount of :py:func:`flow `\ s — the listening port). 113 | We have a drain for ``bytes``, also on each connection, and a drain for :py:func:`flow `\ s: the :py:func:`listener ` wrapped around ``echo``\ . 114 | 115 | Attempting to hook up a fount and a drain of mismatched types should result in an immediate ``TypeError``, which is a helpful debugging tool. 116 | (However, it's the responsibility of the specific fount and drain implementation, and those which have an ``inputType`` or ``outputType`` of ``None`` will not be checked, so you can't rely on this *always* happening.) 117 | Always make sure you've matched up the expected types of the output of your founts and the input of the drains they're connected to. 118 | 119 | Processing A Little Data: Reversing A String 120 | -------------------------------------------- 121 | 122 | Let's perform some very simple processing on our input data: we will reverse each line that we receive. 123 | 124 | This immediately raises the question: how do we tell when we have received a whole line? 125 | The previous echo example didn't care because it would just emit whatever bytes were sent to it, regardless of whether it had received a whole message or not. 126 | (It just so happens that your terminal only sends the bytes when you hit the "enter" key.) 127 | We can't just split up the incoming data with ``bytes.split`` because we might receive one line, part of a line, or multiple lines in one network message. 128 | Luckily Tubes implements this for us, with the handy :py:func:`tubes.framing` module (so called because it puts "frames" around chunks of bytes, and you can distinguish one chunk from the next). 129 | 130 | .. note:: 131 | 132 | There are many types of framing mechanisms, and the one we're demonstrating here, line-oriented message separation, while it is extremely common, is one of the worst ones. 133 | For example, a line-delimited message obviously cannot include a newline, and if you try to transmit one that does, you may get a garbled data stream. 134 | The main advantage of a line-separated protocol is that it works well for interactive examples, since a human being can type lines into a terminal. 135 | For example, it works well for documentation :-). 136 | However, if you're designing your own network protocol, please consider using a length-prefixed framing mechanism, such as :py:func:`netstrings `. 137 | 138 | Much like in the echo example, we need a function which accepts a ``flow`` which sets up the flow of data from a fount to a drain on an individual connection. 139 | 140 | .. literalinclude:: listings/reversetube.py 141 | :pyobject: reverseFlow 142 | 143 | In this function, we create a new type of object, a *series of Tubes*, created by the :py:func:`series ` function. 144 | You can read the construction of the ``lineReverser`` series as a flow of data from left to right. 145 | The output from each tube in the series is passed as the input to the tube to its right. 146 | 147 | We are expecting a stream of bytes as our input, because that's the only thing that ever comes in from a network. 148 | Therefore the first element in the series, :py:func:`bytesToLines `, as its name implies, converts the stream of bytes into a sequence of lines. 149 | The next element in the series, ``Reverser``, reverses its inputs, which, being the output of :py:func:`bytesToLines `, are lines. 150 | 151 | ``Reverser`` is implemented like so: 152 | 153 | .. literalinclude:: listings/reversetube.py 154 | :pyobject: Reverser 155 | 156 | 157 | Managing State with a Tube: A Networked Calculator 158 | -------------------------------------------------- 159 | 160 | To demonstrate both receiving and processing data, let's write a `reverse Polish notation `_ calculator for addition and multiplication. 161 | 162 | Interacting with it should look like this: 163 | 164 | .. code-block:: console 165 | :emphasize-lines: 4,7 166 | 167 | 3 168 | 4 169 | + 170 | 7 171 | 2 172 | * 173 | 14 174 | 175 | In order to implement this program, you will construct a *series* of objects which process the data; specifically, you will create a :py:func:`series ` of :py:func:`Tube `\s. 176 | Each :py:func:`Tube ` in the :py:func:`tubes.tube.series` will be responsible for processing part of the data. 177 | 178 | Lets get started with just the core component that will actually perform calculations. 179 | 180 | .. literalinclude:: listings/rpn.py 181 | :pyobject: Calculator 182 | 183 | ``Calculator`` gives you an API for pushing numbers onto a stack, and for performing an operation on the top two items in the stack, the result of which is then pushed to the top of the stack. 184 | 185 | Now let's look at the full flow which will pass inputs to a ``Calculator`` and relay its output: 186 | 187 | .. literalinclude:: listings/rpn.py 188 | :pyobject: calculatorSeries 189 | 190 | The first tube in this series, provided by the :py:func:`tubes.framing` module, transforms a stream of bytes into lines. 191 | Then, ``linesToNumbersOrOperators`` - which you'll write in a moment - should transform lines into a combination of numbers and operators (functions that perform the work of the ``"+"`` and ``"*"`` commands), then from numbers and operators into more numbers - sums and products - from those integers into lines, and finally from those lines into newline-terminated segments of data that are sent back out. 192 | A ``CalculatingTube`` should pass those numbers and operators to a ``Calculator``, and produce numbers as output. 193 | ``numbersToLines`` should convert the output numbers into byte strings, and ``linesToBytes`` performs the inverse of ``bytesToLines`` by appending newlines to those outputs. 194 | 195 | Let's look at ``linesToNumbersOrOperators``. 196 | 197 | .. literalinclude:: listings/rpn.py 198 | :pyobject: linesToNumbersOrOperators 199 | 200 | :py:func:`tubes.itube.ITube.received` takes an input and produces an iterable of outputs. 201 | A tube's input is the output of the tube preceding it in the series. 202 | In this case, ``linesToNumbersOrOperators`` receives the output of :py:func:`tubes.framing.bytesToLines`, which outputs sequences of bytes (without a trailing line separator). 203 | Given the specification for the RPN calculator's input above, those lines may contain ASCII integers (like ``b"123"``) or ASCII characters representing arithmetic operations (``b"+"`` or ``b"*"``). 204 | ``linesToNumbersOrOperators`` output falls into two categories: each line containing decimal numbers results in an integer output, and each operator character is represented by a python function object that can perform that operation. 205 | 206 | Now that you've parsed those inputs into meaningful values, you can send them on to the ``Calculator`` for processing. 207 | 208 | .. literalinclude:: listings/rpn.py 209 | :pyobject: CalculatingTube 210 | 211 | ``CalculatingTube`` takes a ``Calculator`` to its constructor, and provides a `received` method which takes, as input, the outputs produced by `LinesToNumbersOrOperators`. 212 | It needs to distinguish between the two types it might be handling --- integers, or operators --- and it does so with `isinstance`. 213 | When it is handling an integer, it pushes that value onto its calculator's stack, and, importantly, does not produce any output. 214 | When it is handling an operator, it applies that operator with its calculator's `do` method, and outputs the result (which will be an integer). 215 | 216 | Unlike ``linesToNumbersOrOperators``, ``CalculatingTube`` is *stateful*. 217 | It does not produce an output for every input. 218 | It only produces output when it encounters an operator. 219 | 220 | Finally we need to move this output along so that the user can see it. 221 | 222 | To do this, we use the very simple ``numbersToLines`` which takes integer inputs and transforms them into ASCII bytes. 223 | 224 | .. literalinclude:: listings/rpn.py 225 | :pyobject: numbersToLines 226 | 227 | Like ``linesToNumbersOrOperators``, ``numbersToLines`` is stateless, and produces one output for every input. 228 | 229 | Before sending the output back to the user, you need to add a newline to each number so it is legible to the user. 230 | Otherwise the distinct numbers "3", "4", and "5" would show up as "345". 231 | 232 | For this, we use the aforementioned ``bytesToLines`` tube, which appends newlines to its inputs. 233 | 234 | Tubes Versus Protocols 235 | ====================== 236 | 237 | If you've used Twisted before, you may notice that half of the line-splitting above is exactly what :py:func:`LineReceiver ` does, and that there are lots of related classes that can do similar things for other message types. 238 | The other half is handled by `producers and consumers `_. 239 | ``tubes`` is a *newer* interface than those things, and you will find it somewhat improved. 240 | If you're writing new code, you should generally prefer to use ``tubes``. 241 | 242 | There are three ways in which ``tubes`` is better than using producers, consumers, and the various ``XXXReceiver`` classes directly. 243 | 244 | #. ``tubes`` is *general purpose*. 245 | Whereas each ``FooReceiver`` class receives ``Foo`` objects in its own way, ``tubes`` provides consistent, re-usable abstractions for sending and receiving. 246 | #. ``tubes`` *does not require subclassing*. 247 | The fact that different responsibilities live in different objects makes it easier to test and instrument them. 248 | #. ``tubes`` *handles flow-control automatically*. 249 | The manual flow-control notifications provided by ``IProducer`` and ``IConsumer`` are still used internally in ``tubes`` to hook up to ``twisted.internet``, but the interfaces defined in ``tubes`` itself are considerably more flexible, as they allow you to hook together chains of arbitrary length, as opposed to just getting buffer notifications for a single connection to a single object. 250 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = Tubes 3 | version = 0.2.1 4 | description = Flow control and backpressure for event-driven applications. 5 | license = MIT 6 | url = https://github.com/twisted/tubes/ 7 | 8 | [options] 9 | packages = find: 10 | install_requires = 11 | Twisted 12 | 13 | [bdist_wheel] 14 | universal = 1 15 | 16 | [flake8] 17 | ignore = E302,E303,E402 18 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | setup() 3 | -------------------------------------------------------------------------------- /sketches/amptube.py: -------------------------------------------------------------------------------- 1 | from zope.interface import implementer 2 | 3 | from twisted.internet.endpoints import serverFromString, clientFromString 4 | 5 | from twisted.internet.defer import Deferred, inlineCallbacks 6 | 7 | from twisted.internet.protocol import Factory 8 | from twisted.protocols.amp import AmpBox, Command, IBoxSender, Integer, AMP, CommandLocator, BoxDispatcher 9 | 10 | from tubes.protocol import flowFountFromEndpoint 11 | from tubes.listening import Listener 12 | from tubes.itube import ISegment 13 | from tubes.tube import tube, series 14 | from tubes.framing import bytesToIntPrefixed 15 | 16 | @tube 17 | class StringsToBoxes: 18 | 19 | inputType = None # I... Packet? IString? IDatagram? 20 | outputType = None # AmpBox -> TODO, implement classes. 21 | 22 | state = 'new' 23 | 24 | def received(self, item): 25 | return getattr(self, 'received_' + self.state)(item) 26 | 27 | 28 | def received_new(self, item): 29 | self._currentBox = AmpBox() 30 | return self.received_key(item) 31 | 32 | 33 | def received_key(self, item): 34 | if item: 35 | self._currentKey = item 36 | self.state = 'value' 37 | else: 38 | self.state = 'new' 39 | yield self._currentBox 40 | 41 | 42 | def received_value(self, item): 43 | self._currentBox[self._currentKey] = item 44 | self.state = 'key' 45 | 46 | 47 | 48 | @tube 49 | class BoxesToData: 50 | """ 51 | Shortcut: I want to go from boxes directly to data. 52 | """ 53 | inputType = None # AmpBox 54 | outputType = ISegment 55 | 56 | def received(self, item): 57 | yield item.serialize() 58 | 59 | 60 | @implementer(IBoxSender) 61 | class BufferingBoxSender(object): 62 | def __init__(self): 63 | self.boxesToSend = [] 64 | 65 | def sendBox(self, box): 66 | self.boxesToSend.append(box) 67 | 68 | def unhandledError(failure): 69 | from twisted.python import log 70 | log.err(failure) 71 | 72 | 73 | @tube 74 | class BoxConsumer: 75 | 76 | inputType = None # AmpBox 77 | outputType = None # AmpBox 78 | 79 | def __init__(self, boxReceiver): 80 | self.boxReceiver = boxReceiver 81 | self.bbs = BufferingBoxSender() 82 | 83 | 84 | def started(self): 85 | self.boxReceiver.startReceivingBoxes(self.bbs) 86 | 87 | 88 | def unhandledError(self, failure): 89 | failure.printTraceback() 90 | 91 | 92 | def received(self, box): 93 | self.boxReceiver.ampBoxReceived(box) 94 | boxes = self.bbs.boxesToSend 95 | self.bbs.boxesToSend = [] 96 | return boxes 97 | 98 | 99 | 100 | class Add(Command): 101 | arguments = [(b'a', Integer()), 102 | (b'b', Integer())] 103 | response = [(b'result', Integer())] 104 | 105 | 106 | class Math(CommandLocator): 107 | @Add.responder 108 | def add(self, a, b): 109 | return dict(result=a + b) 110 | 111 | 112 | def mathFlow(flow): 113 | byteParser = bytesToIntPrefixed(16) 114 | messageParser = StringsToBoxes() 115 | applicationCode = Math() 116 | dispatcher = BoxDispatcher(applicationCode) 117 | messageConsumer = BoxConsumer(dispatcher) 118 | messageSerializer = BoxesToData() 119 | combined = series( 120 | byteParser, messageParser, messageConsumer, messageSerializer, flow.drain 121 | ) 122 | flow.fount.flowTo(combined) 123 | 124 | 125 | 126 | @inlineCallbacks 127 | def main(reactor, type="server"): 128 | if type == "server": 129 | serverEndpoint = serverFromString(reactor, "tcp:1234") 130 | flowFount = yield flowFountFromEndpoint(serverEndpoint) 131 | flowFount.flowTo(Listener(mathFlow)) 132 | else: 133 | clientEndpoint = clientFromString(reactor, "tcp:localhost:1234") 134 | amp = yield clientEndpoint.connect(Factory.forProtocol(AMP)) 135 | for x in range(20): 136 | print((yield amp.callRemote(Add, a=1, b=2))) 137 | yield Deferred() 138 | 139 | 140 | from twisted.internet.task import react 141 | from sys import argv 142 | react(main, argv[1:]) 143 | -------------------------------------------------------------------------------- /sketches/fanchat.py: -------------------------------------------------------------------------------- 1 | 2 | from collections import defaultdict 3 | from json import loads, dumps 4 | 5 | from zope.interface.common.mapping import IMapping 6 | 7 | from twisted.internet.endpoints import serverFromString 8 | from twisted.internet.defer import Deferred, inlineCallbacks 9 | 10 | from tubes.routing import Router, Routed, to 11 | from tubes.itube import IFrame 12 | from tubes.tube import series, tube, receiver 13 | from tubes.framing import bytesToLines, linesToBytes 14 | from tubes.fan import Out, In 15 | from tubes.listening import Listener 16 | from tubes.protocol import flowFountFromEndpoint 17 | 18 | 19 | 20 | 21 | @tube 22 | class Participant(object): 23 | outputType = Routed(IMapping) 24 | 25 | def __init__(self, hub, requestsFount, responsesDrain): 26 | self._hub = hub 27 | self._participation = {} 28 | self._in = In() 29 | self._router = Router() 30 | self._participating = {} 31 | 32 | # self._in is both commands from our own client and also messages from 33 | # other clients. 34 | requestsFount.flowTo(self._in.newDrain()) 35 | self._in.fount.flowTo(series(self, self._router.drain)) 36 | 37 | self.client = self._router.newRoute() 38 | self.client.flowTo(responsesDrain) 39 | 40 | def received(self, item): 41 | kwargs = item.copy() 42 | return getattr(self, "do_" + kwargs.pop("type"))(**kwargs) 43 | 44 | def do_name(self, name): 45 | self.name = name 46 | yield to(self.client, dict(named=name)) 47 | 48 | def do_joined(self, sender, channel): 49 | """ 50 | Someone joined a channel I'm participating in. 51 | """ 52 | yield to(self.client, dict(type="joined")) 53 | 54 | def do_join(self, channel): 55 | fountFromChannel, drainToChannel = ( 56 | self._hub.channelNamed(channel).participate(self) 57 | ) 58 | fountFromChannel.flowTo(self._in.newDrain()) 59 | fountToChannel = self._router.newRoute() 60 | fountToChannel.flowTo(drainToChannel) 61 | 62 | self._participating[channel] = fountToChannel 63 | yield to(self._participating[channel], 64 | dict(type="joined")) 65 | 66 | def do_speak(self, channel, message, id): 67 | yield to(self._participating[channel], 68 | dict(type="spoke", message=message, id=id)) 69 | 70 | def do_shout(self, message, id): 71 | for channel in self._participating.values(): 72 | yield to(channel, dict(type="spoke", message=message, id=id)) 73 | yield to(self.client, dict(type="shouted", id=id)) 74 | 75 | def do_tell(self, receiver, message): 76 | # TODO: implement _establishRapportWith; should be more or less like 77 | # joining a channel. 78 | rapport = self._establishRapportWith(receiver) 79 | yield to(rapport, dict(type="told", message=message)) 80 | # TODO: when does a rapport end? timeout as soon as the write buffer 81 | # is empty? 82 | 83 | def do_told(self, sender, message): 84 | yield to(self.client, message) 85 | 86 | def do_spoke(self, channel, sender, message, id): 87 | yield to(self.client, 88 | dict(type="spoke", channel=channel, 89 | sender=sender.name, message=message, 90 | id=id)) 91 | 92 | 93 | 94 | @receiver(IFrame, IMapping) 95 | def linesToCommands(line): 96 | yield loads(line) 97 | 98 | 99 | 100 | @receiver(IMapping, IFrame) 101 | def commandsToLines(message): 102 | yield dumps(message) 103 | 104 | 105 | 106 | class Channel(object): 107 | def __init__(self, name): 108 | self._name = name 109 | self._out = Out() 110 | self._in = In() 111 | self._in.fount.flowTo(self._out.drain) 112 | 113 | def participate(self, participant): 114 | @receiver(IMapping, IMapping) 115 | def addSender(item): 116 | yield dict(item, sender=participant, channel=self._name) 117 | 118 | return (self._out.newFount(), 119 | series(addSender, self._in.newDrain())) 120 | 121 | 122 | 123 | @tube 124 | class OnStop(object): 125 | def __init__(self, callback): 126 | self.callback = callback 127 | def received(self, item): 128 | yield item 129 | def stopped(self, reason): 130 | self.callback() 131 | return () 132 | 133 | 134 | 135 | class Hub(object): 136 | def __init__(self): 137 | self.participants = [] 138 | self.channels = {} 139 | 140 | def newParticipantFlow(self, flow): 141 | commandFount = flow.fount.flowTo( 142 | series(OnStop(lambda: self.participants.remove(participant)), 143 | bytesToLines(), linesToCommands) 144 | ) 145 | commandDrain = series(commandsToLines, linesToBytes(), flow.drain) 146 | participant = Participant(self, commandFount, commandDrain) 147 | self.participants.append(participant) 148 | 149 | def channelNamed(self, name): 150 | if name not in self.channels: 151 | self.channels[name] = Channel(name) 152 | return self.channels[name] 153 | 154 | 155 | 156 | @inlineCallbacks 157 | def main(reactor, port="stdio:"): 158 | endpoint = serverFromString(reactor, port) 159 | flowFount = yield flowFountFromEndpoint(endpoint) 160 | flowFount.flowTo(Listener(Hub().newParticipantFlow)) 161 | yield Deferred() 162 | 163 | 164 | 165 | from twisted.internet.task import react 166 | from sys import argv 167 | react(main, argv[1:]) 168 | -------------------------------------------------------------------------------- /sketches/notes.rst: -------------------------------------------------------------------------------- 1 | In the interst of making this branch more accessible to additional contributors, here are some thoughts that we have about what's going on right now. 2 | 3 | We should be at 100% test coverage. 4 | 5 | Framing needs a ton of tests. 6 | It hasn't changed a whole lot so documenting and testing this module might be a good way to get started. 7 | 8 | ``tubes.protocol`` is pretty well tested and roughly complete but could really use some docstrings, and improve the ones it has. 9 | See for example the docstring for flowFountFromEndpoint. 10 | 11 | The objects in ``tubes.protocol``, especially those that show up in log messages, could really use nicer reprs that indicate what they're doing. 12 | For example ``_ProtocolPlumbing`` and ``_FlowFactory`` should both include information about the flow function they're working on behalf of. 13 | 14 | Similarly, ``tubes.fan`` is a pretty rough sketch, although it's a bit less self-evident what is going on there since it's not fully implemented. 15 | (*Hopefully* it's straightforward, but let's not count on hope.) 16 | 17 | There are a bunch of un-covered `__repr__`s, probably. 18 | 19 | `tubes.tube.Diverter` could use some better docstrings, as could its helpers `_DrainingFount` and `_DrainingTube`. 20 | 21 | There are some asserts littered around the code. 22 | They all need to be deleted. 23 | Some of them should be replaced with real exceptions, because they're a result of bad inputs, and some of them should be replaced with unit tests that more convincingly prove to us that the internal state can never get into that bad place. 24 | 25 | The adapter registry in ``_siphon.py`` is probably silly. 26 | It used to contain a lot more entries, but as the code has evolved it has boiled down into 20 or 30 lines of code and docstrings that might be more easily expressed as a single providedBy check. 27 | Unless more entries show up, we want to delete it and fix ``series`` to just do ``ITube.providedBy`` and inline the implementation of ``tube2drain``. 28 | 29 | things that we might want to change 30 | =================================== 31 | 32 | Currently the contract around flowStopped / stopFlow and then more calls to flowTo / flowingFrom is vague. We might want to adjust this contract so that a fount that has been stopped or a drain that has received a flowStopped is simply "dead" and may not be re-used in any capacity. For things like real sockets, this is a fact of life; however, it might just as well be communicated by an instant stopFlow() or flowStopped() upon hook-up. 33 | 34 | STATK MAECHINES 35 | --------------- 36 | 37 | With flowTo in FLOWING (current state, sort of, it's not really implemented all 38 | the way): 39 | 40 | 41 | Fount 42 | ~~~~~ 43 | 44 | :: 45 | 46 | INITIAL -flowTo(None)-> INITIAL, 47 | -flowTo()-> FLOWING, 48 | -actuallyPause()-> PAUSED_INITIAL, 49 | -stopFlow()-> STOPPED; 50 | 51 | PAUSED_INITIAL -actuallyUnpause()-> INITIAL, 52 | -actuallyPause()-> PAUSED_INITIAL, 53 | -stopFlow()-> STOPPED; 54 | 55 | FLOWING -flowTo(other)-> FLOWING, 56 | -flowTo(None)-> INITIAL, 57 | -actuallyPause()-> PAUSED, 58 | -stopFlow()-> STOPPED; 59 | 60 | PAUSED -flowTo(other)-> FLOWING, 61 | -flowTo(None)-> INITIAL, 62 | 63 | ^ note that these are problematic, because you have to re-set the 64 | pause state, which means you have to discard previous pause tokens, 65 | which we don't currently do 66 | 67 | -actuallyResume()-> FLOWING, 68 | -actuallyPause()-> PAUSED, 69 | -stopFlow()-> STOPPED; 70 | 71 | STOPPED. 72 | 73 | 74 | Drain 75 | ~~~~~ 76 | 77 | :: 78 | 79 | INITIAL -flowingFrom()-> FLOWING, 80 | -flowingFrom(None)-> INITIAL; 81 | 82 | FLOWING -receive()-> FLOWING, 83 | -flowingFrom(None)-> INITIAL, 84 | -flowingFrom(other)-> FLOWING, 85 | -flowStopped()-> STOPPED; 86 | 87 | STOPPED. 88 | 89 | 90 | Without flowTo in FLOWING (desired state): 91 | 92 | 93 | Fount 94 | ~~~~~ 95 | 96 | :: 97 | 98 | INITIAL -flowTo()-> FLOWING, 99 | -actuallyPause()-> PAUSED_INITIAL, 100 | -stopFlow()-> STOPPED; 101 | 102 | PAUSED_INITIAL -actuallyUnpause()-> INITIAL, 103 | -actuallyPause()-> PAUSED_INITIAL, 104 | -stopFlow()-> STOPPED; 105 | 106 | FLOWING -actuallyPause()-> PAUSED, 107 | -stopFlow()-> STOPPED; 108 | 109 | PAUSED -actuallyResume()-> FLOWING, 110 | -actuallyPause()-> PAUSED, 111 | -stopFlow()-> STOPPED; 112 | 113 | STOPPED. 114 | 115 | 116 | Drain 117 | ~~~~~ 118 | 119 | :: 120 | 121 | INITIAL -flowingFrom()-> FLOWING; 122 | 123 | FLOWING -receive()-> FLOWING, 124 | -flowStopped()-> STOPPED; 125 | 126 | STOPPED. 127 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py38, py310, py311, py312, pypy, docs, lint, apidocs, docs-spellcheck 3 | 4 | [testenv] 5 | passenv = * 6 | deps = 7 | coverage 8 | commands = 9 | coverage run -p {envbindir}/trial --rterrors {posargs:tubes} 10 | 11 | [testenv:docs] 12 | deps = 13 | doc8 14 | pygments 15 | pydoctor 16 | sphinx!=1.6.1 17 | sphinx_rtd_theme 18 | commands = 19 | sphinx-build -vvv -W -b html -d {envtmpdir}/doctrees docs docs/_build/html 20 | sphinx-build -W -b latex -d {envtmpdir}/doctrees docs docs/_build/latex 21 | doc8 --ignore D000 --ignore D001 --allow-long-titles docs/ 22 | 23 | [testenv:docs-spellcheck] 24 | deps = 25 | {[testenv:docs]deps} 26 | pyenchant 27 | sphinxcontrib-spelling 28 | commands = 29 | sphinx-build -W -b spelling docs docs/_build/html 30 | 31 | [testenv:docs-linkcheck] 32 | deps = 33 | {[testenv:docs]deps} 34 | commands = 35 | sphinx-build -W -b linkcheck docs docs/_build/html 36 | 37 | [testenv:lint] 38 | deps = 39 | twistedchecker==0.7.2 40 | commands = 41 | # pep257 --ignore=D400,D401,D200,D203,D204,D205 ./tubes 42 | python .failonoutput.py "twistedchecker --msg-template=\{path\}:\{line\}:\{column\}:\ [\{msg_id\}\(\{symbol\}\),\ \{obj\}]\ \{msg\} ./tubes" 43 | 44 | [testenv:coveralls-push] 45 | deps = 46 | coveralls 47 | coverage 48 | commands = 49 | coverage combine 50 | coverage report 51 | coveralls 52 | 53 | [flake8] 54 | exclude = docs,.tox,*.egg,*.pyc,.git,__pycache 55 | max-line-length = 105 56 | 57 | [doc8] 58 | extensions = rst 59 | -------------------------------------------------------------------------------- /tubes/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- test-case-name: tubes.test -*- 2 | # Copyright (c) Twisted Matrix Laboratories. 3 | # See LICENSE for details. 4 | 5 | """ 6 | L{tubes} offers an abstraction of data flow and backpressure for event-driven 7 | applications. 8 | 9 | @see: L{tubes.tube} 10 | """ 11 | -------------------------------------------------------------------------------- /tubes/_components.py: -------------------------------------------------------------------------------- 1 | # -*- test-case-name: tubes.test.test_tube -*- 2 | # Copyright (c) Twisted Matrix Laboratories. 3 | # See LICENSE for details. 4 | 5 | """ 6 | Various component utilities. 7 | """ 8 | 9 | from zope.interface.adapter import AdapterRegistry 10 | from twisted.python.components import _addHook, _removeHook 11 | from contextlib import contextmanager 12 | 13 | if 0: 14 | from zope.interface.interfaces import IInterface 15 | IInterface 16 | 17 | 18 | 19 | @contextmanager 20 | def _registryActive(registry): 21 | """ 22 | A context manager that activates and deactivates a zope adapter registry 23 | for the duration of the call. 24 | 25 | For example, if you wanted to have a function that could adapt C{IFoo} to 26 | C{IBar}, but doesn't expose that adapter outside of itself:: 27 | 28 | def convertToBar(maybeFoo): 29 | with _registryActive(_registryAdapting((IFoo, IBar, fooToBar))): 30 | return IBar(maybeFoo) 31 | 32 | @note: This isn't thread safe, so other threads will be affected as well. 33 | 34 | @param registry: The registry to activate. 35 | @type registry: L{AdapterRegistry} 36 | 37 | @rtype: 38 | """ 39 | hook = _addHook(registry) 40 | yield 41 | _removeHook(hook) 42 | 43 | 44 | 45 | def _registryAdapting(*fromToAdapterTuples): 46 | """ 47 | Construct a Zope Interface adapter registry. 48 | 49 | For example, if you want to construct an adapter registry that can convert 50 | C{IFoo} to C{IBar} with C{fooToBar}. 51 | 52 | @param fromToAdapterTuples: A sequence of tuples of C{(fromInterface, 53 | toInterface, adapterCallable)}, where C{fromInterface} and 54 | C{toInterface} are L{IInterface}s, and C{adapterCallable} is a callable 55 | that takes one argument which provides C{fromInterface} and returns an 56 | object providing C{toInterface}. 57 | @type fromToAdapterTuples: C{tuple} of 3-C{tuple}s of C{(Interface, 58 | Interface, callable)} 59 | 60 | @return: an adapter registry adapting the given tuples. 61 | @rtype: L{AdapterRegistry} 62 | """ 63 | result = AdapterRegistry() 64 | for _from, to, adapter in fromToAdapterTuples: 65 | result.register([_from], to, '', adapter) 66 | return result 67 | -------------------------------------------------------------------------------- /tubes/_siphon.py: -------------------------------------------------------------------------------- 1 | # -*- test-case-name: tubes.test.test_tube -*- 2 | # Copyright (c) Twisted Matrix Laboratories. 3 | # See LICENSE for details. 4 | 5 | """ 6 | Adapters for converting L{ITube} to L{IDrain} and L{IFount}. 7 | """ 8 | 9 | from collections import deque 10 | 11 | from zope.interface import implementer 12 | 13 | from .itube import IDrain, IFount, ITube 14 | from .kit import Pauser, beginFlowingFrom, beginFlowingTo, NoPause, OncePause 15 | from ._components import _registryAdapting 16 | 17 | from twisted.python.failure import Failure 18 | 19 | from twisted.python import log 20 | 21 | whatever = object() 22 | 23 | 24 | 25 | def suspended(): 26 | """ 27 | A token value meaning that L{SiphonPendingValues} was suspended. 28 | """ 29 | 30 | 31 | 32 | def finished(): 33 | """ 34 | A token value meaning that L{SiphonPendingValues} has no more values in its 35 | queue. 36 | """ 37 | 38 | 39 | 40 | def skip(): 41 | """ 42 | A token value yielded by a tube meaning that its L{_Siphon} should not 43 | deliver this value on, so that tubes may engage in flow control. 44 | """ 45 | 46 | 47 | 48 | class SiphonPendingValues(object): 49 | """ 50 | A queue of pending values which can be suspended and resumed, for 51 | representing values pending delivery for a L{_Siphon}. 52 | 53 | @ivar _deque: a deque containing iterators containing queued values. 54 | 55 | @ivar _suspended: Is this L{SiphonPendingValues} currently suspended? 56 | """ 57 | 58 | def __init__(self): 59 | self._deque = deque() 60 | self._suspended = False 61 | 62 | 63 | def suspend(self): 64 | """ 65 | L{SiphonPendingValues.popPendingValue} should return L{suspended}. 66 | """ 67 | self._suspended = True 68 | 69 | 70 | def resume(self): 71 | """ 72 | L{SiphonPendingValues.popPendingValue} 73 | """ 74 | self._suspended = False 75 | 76 | 77 | def prepend(self, iterator): 78 | """ 79 | Add the given iterator to the beginning of the queue. 80 | 81 | @param iterator: an iterator of values to deliver via popPendingValue. 82 | """ 83 | self._deque.appendleft(iterator) 84 | 85 | 86 | def append(self, iterator): 87 | """ 88 | Add the given iterator to the end of the queue. 89 | 90 | @param iterator: an iterator of values to deliver via popPendingValue. 91 | """ 92 | self._deque.append(iterator) 93 | 94 | 95 | def clear(self): 96 | """ 97 | Clear the entire queue. 98 | """ 99 | self._deque.clear() 100 | 101 | 102 | def popPendingValue(self, evenIfSuspended=False): 103 | """ 104 | Get the next value in the leftmost iterator in the deque. 105 | 106 | @param evenIfSuspended: return the next pending value regardless of 107 | whether this L{SiphonPendingValues} is suspended or not. 108 | @type evenIfSuspended: L{bool} 109 | 110 | @return: The next value yielded by the first iterator in the queue, 111 | L{suspended} if this L{SiphonPendingValues} is suspended and 112 | C{evenIfSuspended} was not passed, or L{finished} if the queue is 113 | empty. 114 | """ 115 | if self._suspended and not evenIfSuspended: 116 | return suspended 117 | while self._deque: 118 | result = next(self._deque[0], whatever) 119 | if self._suspended and not evenIfSuspended: 120 | self.prepend(iter([result])) 121 | return suspended 122 | if result is whatever: 123 | self._deque.popleft() 124 | else: 125 | return result 126 | return finished 127 | 128 | 129 | 130 | class _SiphonPiece(object): 131 | """ 132 | Shared functionality between L{_SiphonFount} and L{_SiphonDrain} 133 | """ 134 | def __init__(self, siphon): 135 | self._siphon = siphon 136 | 137 | 138 | @property 139 | def _tube(self): 140 | """ 141 | Expose the siphon's C{_tube} directly since many things will want to 142 | manipulate it. 143 | 144 | @return: L{ITube} 145 | """ 146 | return self._siphon._tube 147 | 148 | 149 | 150 | @implementer(IFount) 151 | class _SiphonFount(_SiphonPiece): 152 | """ 153 | Implementation of L{IFount} for L{_Siphon}. 154 | 155 | @ivar fount: the implementation of the L{IDrain.fount} attribute. The 156 | L{IFount} which is flowing to this L{_Siphon}'s L{IDrain} 157 | implementation. 158 | 159 | @ivar drain: the implementation of the L{IFount.drain} attribute. The 160 | L{IDrain} to which this L{_Siphon}'s L{IFount} implementation is 161 | flowing. 162 | """ 163 | drain = None 164 | 165 | def __init__(self, siphon): 166 | super(_SiphonFount, self).__init__(siphon) 167 | 168 | def _actuallyPause(): 169 | fount = self._siphon._tdrain.fount 170 | self._siphon._pending.suspend() 171 | if fount is not None: 172 | pbpc = fount.pauseFlow() 173 | else: 174 | pbpc = NoPause() 175 | self._siphon._pauseBecausePauseCalled = pbpc 176 | 177 | def _actuallyResume(): 178 | fp = self._siphon._pauseBecausePauseCalled 179 | self._siphon._pauseBecausePauseCalled = None 180 | 181 | self._siphon._pending.resume() 182 | self._siphon._unbufferIterator() 183 | 184 | fp.unpause() 185 | 186 | self._pauser = Pauser(_actuallyPause, _actuallyResume) 187 | 188 | 189 | def __repr__(self): 190 | """ 191 | Nice string representation. 192 | """ 193 | return "".format(repr(self._siphon._tube)) 194 | 195 | 196 | @property 197 | def outputType(self): 198 | """ 199 | Relay the C{outputType} declared by the tube. 200 | 201 | @return: see L{IFount.outputType} 202 | """ 203 | return self._tube.outputType 204 | 205 | 206 | def flowTo(self, drain): 207 | """ 208 | Flow data from this L{_Siphon} to the given drain. 209 | 210 | @param drain: see L{IFount.flowTo} 211 | 212 | @return: an L{IFount} that emits items of the output-type of this 213 | siphon's tube. 214 | """ 215 | result = beginFlowingTo(self, drain) 216 | self._siphon._pauseBecauseNoDrain.maybeUnpause() 217 | self._siphon._unbufferIterator() 218 | return result 219 | 220 | 221 | def pauseFlow(self): 222 | """ 223 | Pause the flow from the fount, or remember to do that when the fount is 224 | attached, if it isn't yet. 225 | 226 | @return: L{IPause} 227 | """ 228 | return self._pauser.pause() 229 | 230 | 231 | def stopFlow(self): 232 | """ 233 | Stop the flow from the fount to this L{_Siphon}, and stop delivering 234 | buffered items. 235 | """ 236 | self._siphon._noMore(input=True, output=True) 237 | fount = self._siphon._tdrain.fount 238 | if fount is None: 239 | return 240 | fount.stopFlow() 241 | 242 | 243 | 244 | @implementer(IDrain) 245 | class _SiphonDrain(_SiphonPiece): 246 | """ 247 | Implementation of L{IDrain} for L{_Siphon}. 248 | """ 249 | fount = None 250 | 251 | def __repr__(self): 252 | """ 253 | Nice string representation. 254 | """ 255 | return ''.format(self._siphon._tube) 256 | 257 | 258 | @property 259 | def inputType(self): 260 | """ 261 | Relay the tube's declared inputType. 262 | 263 | @return: see L{IDrain.inputType} 264 | """ 265 | return self._tube.inputType 266 | 267 | 268 | def flowingFrom(self, fount): 269 | """ 270 | This siphon will now have 'receive' called on it by the given fount. 271 | 272 | @param fount: see L{IDrain.flowingFrom} 273 | 274 | @return: see L{IDrain.flowingFrom} 275 | """ 276 | beginFlowingFrom(self, fount) 277 | if self._siphon._pauseBecausePauseCalled: 278 | pbpc = self._siphon._pauseBecausePauseCalled 279 | self._siphon._pauseBecausePauseCalled = None 280 | if fount is None: 281 | pauseFlow = NoPause 282 | else: 283 | pauseFlow = fount.pauseFlow 284 | self._siphon._pauseBecausePauseCalled = pauseFlow() 285 | pbpc.unpause() 286 | if fount is not None: 287 | if not self._siphon._canStillProcessInput: 288 | fount.stopFlow() 289 | # Is this the right place, or does this need to come after 290 | # _pauseBecausePauseCalled's check? 291 | if not self._siphon._everStarted: 292 | self._siphon._everStarted = True 293 | self._siphon._deliverFrom(self._tube.started) 294 | nextFount = self._siphon._tfount 295 | nextDrain = nextFount.drain 296 | if nextDrain is None: 297 | return nextFount 298 | return nextFount.flowTo(nextDrain) 299 | 300 | 301 | def receive(self, item): 302 | """ 303 | An item was received. Pass it on to the tube for processing. 304 | 305 | @param item: an item to deliver to the tube. 306 | """ 307 | def tubeReceivedItem(): 308 | return self._tube.received(item) 309 | self._siphon._deliverFrom(tubeReceivedItem) 310 | 311 | 312 | def flowStopped(self, reason): 313 | """ 314 | This siphon's fount has communicated the end of the flow to this 315 | siphon. This siphon should finish yielding its current buffer, then 316 | yield the result of it's C{_tube}'s C{stopped} method, then communicate 317 | the end of flow to its downstream drain. 318 | 319 | @param reason: the reason why our fount stopped the flow. 320 | """ 321 | self._siphon._noMore(input=True, output=False) 322 | self._siphon._flowStoppingReason = reason 323 | def tubeStopped(): 324 | return self._tube.stopped(reason) 325 | self._siphon._deliverFrom(tubeStopped) 326 | 327 | 328 | 329 | class _Siphon(object): 330 | """ 331 | A L{_Siphon} is an L{IDrain} and possibly also an L{IFount}, and provides 332 | lots of conveniences to make it easy to implement something that does fancy 333 | flow control with just a few methods. 334 | 335 | @ivar _tube: the L{ITube} which will receive values from this siphon and 336 | call C{deliver} to deliver output to it. (When set, this will 337 | automatically set the C{siphon} attribute of said L{ITube} as well, as 338 | well as un-setting the C{siphon} attribute of the old tube.) 339 | 340 | @ivar _currentlyPaused: is this L{_Siphon} currently paused? Boolean: 341 | C{True} if paused, C{False} if not. 342 | 343 | @ivar _pauseBecausePauseCalled: an L{IPause} from the upstream fount, 344 | present because pauseFlow has been called. 345 | 346 | @ivar _flowStoppingReason: If this is not C{None}, then call C{flowStopped} 347 | on the downstream L{IDrain} at the next opportunity, where "the next 348 | opportunity" is when all buffered input (values yielded from 349 | C{started}, C{received}, and C{stopped}) has been written to the 350 | downstream drain and we are unpaused. 351 | 352 | @ivar _everStarted: Has this L{_Siphon} ever called C{started} on its 353 | L{ITube}? 354 | @type _everStarted: L{bool} 355 | """ 356 | 357 | def __init__(self, tube): 358 | """ 359 | Initialize this L{_Siphon} with the given L{ITube} to control its 360 | behavior. 361 | """ 362 | self._canStillProcessInput = True 363 | self._pauseBecausePauseCalled = None 364 | self._tube = None 365 | self._everStarted = False 366 | self._unbuffering = False 367 | self._flowStoppingReason = None 368 | 369 | self._tfount = _SiphonFount(self) 370 | self._pauseBecauseNoDrain = OncePause(self._tfount._pauser) 371 | self._tdrain = _SiphonDrain(self) 372 | self._tube = tube 373 | self._pending = SiphonPendingValues() 374 | 375 | 376 | def _noMore(self, input, output): 377 | """ 378 | I am now unable to produce further input, or output, or both. 379 | 380 | @param input: L{True} if I can no longer produce input. 381 | 382 | @param output: L{True} if I can no longer produce output. 383 | """ 384 | if input: 385 | self._canStillProcessInput = False 386 | if output: 387 | self._pending.clear() 388 | 389 | 390 | def __repr__(self): 391 | """ 392 | Nice string representation. 393 | """ 394 | return '<_Siphon for {0}>'.format(repr(self._tube)) 395 | 396 | 397 | def _deliverFrom(self, deliverySource): 398 | """ 399 | Deliver some items from a callable that will produce an iterator. 400 | 401 | @param deliverySource: a 0-argument callable that will return an 402 | iterable. 403 | """ 404 | try: 405 | iterableOrNot = deliverySource() 406 | except: 407 | f = Failure() 408 | log.err(f, "Exception raised when delivering from {0!r}" 409 | .format(deliverySource)) 410 | self._tdrain.fount.stopFlow() 411 | downstream = self._tfount.drain 412 | if downstream is not None: 413 | downstream.flowStopped(f) 414 | return 415 | if iterableOrNot is None: 416 | return 417 | self._pending.append(iter(iterableOrNot)) 418 | if self._tfount.drain is None: 419 | self._pauseBecauseNoDrain.pauseOnce() 420 | self._unbufferIterator() 421 | 422 | 423 | def _unbufferIterator(self): 424 | """ 425 | Un-buffer some items buffered in C{self._pending} and actually deliver 426 | them, as long as we're not paused. 427 | """ 428 | if self._unbuffering: 429 | return 430 | 431 | self._unbuffering = True 432 | 433 | while True: 434 | value = self._pending.popPendingValue() 435 | if value is suspended: 436 | break 437 | elif value is skip: 438 | continue 439 | elif value is finished: 440 | if self._flowStoppingReason: 441 | self._endOfLine(self._flowStoppingReason) 442 | break 443 | else: 444 | self._tfount.drain.receive(value) 445 | 446 | self._unbuffering = False 447 | 448 | 449 | def _endOfLine(self, flowStoppingReason): 450 | """ 451 | We've reached the end of the line. Immediately stop delivering all 452 | buffers and notify our downstream drain why the flow has stopped. 453 | 454 | @param flowStoppingReason: the reason that the flow was stopped. 455 | """ 456 | self._noMore(input=True, output=True) 457 | self._flowStoppingReason = None 458 | self._pending.clear() 459 | downstream = self._tfount.drain 460 | if downstream is not None: 461 | self._tfount.drain.flowStopped(flowStoppingReason) 462 | 463 | 464 | def ejectPending(self): 465 | """ 466 | Eject the entire pending buffer into a list for reassembly by a 467 | diverter. 468 | 469 | @return: a L{list} of all buffered output values. 470 | """ 471 | result = [] 472 | while True: 473 | value = self._pending.popPendingValue(evenIfSuspended=True) 474 | if value is finished: 475 | return result 476 | result.append(value) 477 | 478 | 479 | 480 | def _tube2drain(tube): 481 | """ 482 | An adapter that can convert an L{ITube} to an L{IDrain} by wrapping it in a 483 | L{_Siphon}. 484 | 485 | @param tube: L{ITube} 486 | 487 | @return: L{IDrain} 488 | """ 489 | return _Siphon(tube)._tdrain 490 | 491 | 492 | 493 | _tubeRegistry = _registryAdapting( 494 | (ITube, IDrain, _tube2drain), 495 | ) 496 | -------------------------------------------------------------------------------- /tubes/fan.py: -------------------------------------------------------------------------------- 1 | # -*- test-case-name: tubes.test.test_fan -*- 2 | # Copyright (c) Twisted Matrix Laboratories. 3 | # See LICENSE for details. 4 | 5 | """ 6 | Tools for turning L{founts } and L{drains } into multiple 7 | founts and drains. 8 | """ 9 | 10 | from itertools import count 11 | 12 | from zope.interface import implementer 13 | 14 | from twisted.python.components import proxyForInterface 15 | 16 | from .kit import Pauser, beginFlowingTo, beginFlowingFrom, OncePause 17 | from .itube import IDrain, IFount 18 | 19 | 20 | @implementer(IDrain) 21 | class _InDrain(object): 22 | """ 23 | The one of the drains associated with an fan.L{In}. 24 | """ 25 | 26 | inputType = None 27 | 28 | fount = None 29 | 30 | def __init__(self, fanIn): 31 | """ 32 | Create an L{_InDrain} with an L{In}. 33 | """ 34 | self._in = fanIn 35 | self._presentPause = None 36 | 37 | 38 | def flowingFrom(self, fount): 39 | """ 40 | The attached L{In} is now receiving inputs from the given fount. 41 | 42 | @param fount: Any fount. 43 | 44 | @return: C{None}; this is a terminal drain and not a data-processing 45 | drain. 46 | """ 47 | if fount is self.fount: 48 | return 49 | beginFlowingFrom(self, fount) 50 | if self._presentPause is not None: 51 | p, self._presentPause = self._presentPause, None 52 | p.unpause() 53 | if self._in.fount._isPaused: 54 | self._presentPause = fount.pauseFlow() 55 | return None 56 | 57 | 58 | def receive(self, item): 59 | """ 60 | Pass along any received item to the drain that the L{In}'s fount is 61 | flowing to. 62 | 63 | @param item: any object 64 | 65 | @return: passed through from the active drain. 66 | """ 67 | return self._in.fount.drain.receive(item) 68 | 69 | 70 | def flowStopped(self, reason): 71 | """ 72 | Remove this drain from its attached L{In}. 73 | 74 | @param reason: the reason the flow stopped. 75 | """ 76 | self._in._drains.remove(self) 77 | 78 | 79 | 80 | @implementer(IFount) 81 | class _InFount(object): 82 | """ 83 | An L{_InFount} is the single fount associated with an L{In}. 84 | """ 85 | 86 | outputType = None 87 | 88 | drain = None 89 | 90 | def __init__(self, fanIn): 91 | """ 92 | Create an L{_InFount} with an L{In}. 93 | """ 94 | self._in = fanIn 95 | self._isPaused = False 96 | def doPause(): 97 | self._isPaused = True 98 | for drain in self._in._drains: 99 | drain._presentPause = drain.fount.pauseFlow() 100 | def doResume(): 101 | self._isPaused = False 102 | for drain in self._in._drains: 103 | drain._presentPause, currentPause = None, drain._presentPause 104 | currentPause.unpause() 105 | self._pauser = Pauser(doPause, doResume) 106 | self._pauseBecauseNoDrain = OncePause(self._pauser) 107 | self._pauseBecauseNoDrain.pauseOnce() 108 | 109 | 110 | def flowTo(self, drain): 111 | """ 112 | Start flowing to the given C{drain}. 113 | 114 | @param drain: the drain to deliver all inputs from all founts attached 115 | to the underlying L{In}. 116 | 117 | @return: the fount downstream of C{drain}. 118 | """ 119 | result = beginFlowingTo(self, drain) 120 | # TODO: if drain is not None 121 | if self.drain is None: 122 | self._pauseBecauseNoDrain.pauseOnce() 123 | else: 124 | self._pauseBecauseNoDrain.maybeUnpause() 125 | return result 126 | 127 | 128 | def pauseFlow(self): 129 | """ 130 | Pause the flow of all founts flowing into L{_InDrain}s for this L{In}. 131 | 132 | @return: A pause which pauses all upstream founts. 133 | """ 134 | return self._pauser.pause() 135 | 136 | 137 | def stopFlow(self): 138 | """ 139 | Stop the flow of all founts flowing into L{_InDrain}s for this L{In}. 140 | """ 141 | while self._in._drains: 142 | it = self._in._drains[0] 143 | it.fount.stopFlow() 144 | if it in self._in._drains: 145 | self._in._drains.remove(it) 146 | 147 | 148 | 149 | class In(object): 150 | r""" 151 | A fan.L{In} presents a single L{fount } that delivers the inputs 152 | from multiple L{drains }:: 153 | 154 | your fount ---> In.newDrain()--\ 155 | \ 156 | your fount ---> In.newDrain()----> In ---> In.fount ---> your drain 157 | / 158 | your fount ---> In.newDrain()--/ 159 | 160 | @ivar fount: The fount which produces all new attributes. 161 | @type fount: L{IFount} 162 | """ 163 | def __init__(self): 164 | self._drains = [] 165 | self.fount = _InFount(self) 166 | 167 | 168 | def newDrain(self): 169 | """ 170 | Create a new L{drains } which will send its 171 | inputs out via C{self.fount}. 172 | 173 | @return: a drain. 174 | """ 175 | it = _InDrain(self) 176 | self._drains.append(it) 177 | return it 178 | 179 | 180 | 181 | @implementer(IFount) 182 | class _OutFount(object): 183 | """ 184 | The concrete fount type returned by L{Out.newFount}. 185 | """ 186 | drain = None 187 | 188 | outputType = None 189 | 190 | def __init__(self, upstreamPauser, stopper): 191 | """ 192 | @param upstreamPauser: A L{Pauser} which will pause the upstream fount 193 | flowing into our L{Out}. 194 | 195 | @param stopper: A 0-argument callback to execute on 196 | L{IFount.stopFlow} 197 | """ 198 | self._receivedWhilePaused = [] 199 | self._myPause = None 200 | self._stopper = stopper 201 | 202 | def actuallyPause(): 203 | self._myPause = upstreamPauser.pause() 204 | 205 | def actuallyUnpause(): 206 | aPause = self._myPause 207 | self._myPause = None 208 | if self._receivedWhilePaused: 209 | self.drain.receive(self._receivedWhilePaused.pop(0)) 210 | aPause.unpause() 211 | 212 | self._pauser = Pauser(actuallyPause, actuallyUnpause) 213 | 214 | 215 | def flowTo(self, drain): 216 | """ 217 | Flow to the given drain. Don't do anything special; just set up the 218 | drain attribute and return the appropriate value. 219 | 220 | @param drain: A drain to fan out values to. 221 | 222 | @return: the result of C{drain.flowingFrom} 223 | """ 224 | return beginFlowingTo(self, drain) 225 | 226 | 227 | def pauseFlow(self): 228 | """ 229 | Pause the flow. 230 | 231 | @return: a pause 232 | @rtype: L{IPause} 233 | """ 234 | return self._pauser.pause() 235 | 236 | 237 | def stopFlow(self): 238 | """ 239 | Invoke the callback supplied to C{__init__} for stopping. 240 | """ 241 | self._stopper(self) 242 | 243 | 244 | def _deliverOne(self, item): 245 | """ 246 | Deliver one item to this fount's drain. 247 | 248 | This is only invoked when the upstream is unpaused. 249 | 250 | @param item: An item that the upstream would like to pass on. 251 | """ 252 | if self.drain is None: 253 | return 254 | if self._myPause is not None: 255 | self._receivedWhilePaused.append(item) 256 | return 257 | self.drain.receive(item) 258 | 259 | 260 | 261 | @implementer(IDrain) 262 | class _OutDrain(object): 263 | """ 264 | An L{_OutDrain} is the single L{IDrain} associated with an L{Out}. 265 | """ 266 | 267 | fount = None 268 | 269 | def __init__(self, founts): 270 | """ 271 | Construct an L{_OutDrain} with a collection of founts, an input type 272 | and an output type. 273 | 274 | @param founts: the founts whose drains we should flow to. 275 | @type founts: L{list} of L{IFount} 276 | """ 277 | self._pause = None 278 | self._paused = False 279 | 280 | self._founts = founts 281 | 282 | def _actuallyPause(): 283 | self._paused = True 284 | if self.fount is not None: 285 | self._pause = self.fount.pauseFlow() 286 | 287 | def _actuallyResume(): 288 | p = self._pause 289 | self._pause = None 290 | self._paused = False 291 | if p is not None: 292 | p.unpause() 293 | 294 | self._pauser = Pauser(_actuallyPause, _actuallyResume) 295 | 296 | 297 | @property 298 | def inputType(self): 299 | """ 300 | Implement the C{inputType} property by relaying it to the input type of 301 | the drains. 302 | """ 303 | # TODO: prevent drains from different inputTypes from being added 304 | for fount in self._founts: 305 | if fount.drain is not None: 306 | return fount.drain.inputType 307 | 308 | 309 | def flowingFrom(self, fount): 310 | """ 311 | The L{Out} associated with this L{_OutDrain} is now receiving inputs 312 | from the given fount. 313 | 314 | @param fount: the new source of input for all drains attached to this 315 | L{Out}. 316 | 317 | @return: L{None}, as this is a terminal drain. 318 | """ 319 | if self._paused: 320 | p = self._pause 321 | if fount is not None: 322 | self._pause = fount.pauseFlow() 323 | else: 324 | self._pause = None 325 | if p is not None: 326 | p.unpause() 327 | beginFlowingFrom(self, fount) 328 | 329 | 330 | def receive(self, item): 331 | """ 332 | Deliver an item to each L{IDrain} attached to the L{Out} via 333 | C{Out().newFount().flowTo(...)}. 334 | 335 | @param item: any object 336 | """ 337 | for fount in self._founts[:]: 338 | fount._deliverOne(item) 339 | 340 | 341 | def flowStopped(self, reason): 342 | """ 343 | Deliver an item to each L{IDrain} attached to the L{Out} via 344 | C{Out().newFount().flowTo(...)}. 345 | 346 | @param reason: the reason that the flow stopped. 347 | """ 348 | for fount in self._founts[:]: 349 | if fount.drain is not None: 350 | fount.drain.flowStopped(reason) 351 | 352 | 353 | 354 | class Out(object): 355 | r""" 356 | A fan.L{Out} presents a single L{drain } that delivers the inputs 357 | to multiple L{founts }:: 358 | 359 | /--> Out.newFount() --> your drain 360 | / 361 | your fount --> Out.drain --> Out <----> Out.newFount() --> your drain 362 | \ 363 | \--> Out.newFount() --> your drain 364 | 365 | @ivar drain: The fount which produces all new attributes. 366 | @type drain: L{IDrain} 367 | """ 368 | 369 | def __init__(self): 370 | """ 371 | Create an L{Out}. 372 | """ 373 | self._founts = [] 374 | self.drain = _OutDrain(self._founts) 375 | 376 | 377 | def newFount(self): 378 | """ 379 | Create a new L{IFount} whose drain will receive inputs from this 380 | L{Out}. 381 | 382 | @return: a fount associated with this fan-L{Out}. 383 | @rtype: L{IFount}. 384 | """ 385 | f = _OutFount(self.drain._pauser, self._founts.remove) 386 | self._founts.append(f) 387 | return f 388 | 389 | 390 | 391 | class Thru(proxyForInterface(IDrain, "_outDrain")): 392 | r""" 393 | A fan.L{Thru} takes an input and fans it I{thru} multiple 394 | drains-which-produce-founts, such as L{tubes }:: 395 | 396 | Your Fount 397 | (producing "foo") 398 | | 399 | v 400 | Thru 401 | | 402 | _/|\_ 403 | _/ | \_ 404 | / | \ 405 | foo2bar foo2baz foo2qux 406 | \_ | _/ 407 | \_ | _/ 408 | \|/ 409 | | 410 | v 411 | Thru 412 | | 413 | v 414 | Your Drain 415 | (receiving a combination 416 | of foo, bar, baz) 417 | 418 | The way you would construct such a flow in code would be:: 419 | 420 | yourFount.flowTo(Thru([series(foo2bar()), 421 | series(foo2baz()), 422 | series(foo2qux())])).flowTo(yourDrain) 423 | """ 424 | 425 | def __init__(self, drains): 426 | """ 427 | Create a L{Thru} with an iterable of L{IDrain}. 428 | 429 | All of the drains in C{drains} should be drains that produce a new 430 | L{IFount} from L{flowingFrom }, which means they 431 | should be a L{series } of L{tubes 432 | }, or drains that behave like that, such as L{Thru} 433 | itself. 434 | 435 | @param drains: an iterable of L{IDrain} 436 | """ 437 | self._in = In() 438 | self._out = Out() 439 | 440 | self._drains = list(drains) 441 | self._founts = list(None for drain in self._drains) 442 | self._outFounts = list(self._out.newFount() for drain in self._drains) 443 | self._inDrains = list(self._in.newDrain() for drain in self._drains) 444 | self._outDrain = self._out.drain 445 | 446 | 447 | def flowingFrom(self, fount): 448 | """ 449 | Accept input from C{fount} and produce output filtered by all of the 450 | C{drain}s given to this L{Thru}'s constructor. 451 | 452 | @param fount: a fount whose outputs should flow through our series of 453 | transformations. 454 | 455 | @return: an output fount which aggregates all the values produced by 456 | the drains given to this L{Thru}'s constructor. 457 | """ 458 | super(Thru, self).flowingFrom(fount) 459 | for idx, appDrain, outFount, inDrain in zip( 460 | count(), self._drains, self._outFounts, self._inDrains): 461 | appFount = outFount.flowTo(appDrain) 462 | if appFount is None: 463 | appFount = self._founts[idx] 464 | else: 465 | self._founts[idx] = appFount 466 | appFount.flowTo(inDrain) 467 | nextFount = self._in.fount 468 | 469 | # Literally copy/pasted from _SiphonDrain.flowingFrom. Hmm. 470 | nextDrain = nextFount.drain 471 | if nextDrain is None: 472 | return nextFount 473 | return nextFount.flowTo(nextDrain) 474 | -------------------------------------------------------------------------------- /tubes/framing.py: -------------------------------------------------------------------------------- 1 | # -*- test-case-name: tubes.test.test_framing -*- 2 | # Copyright (c) Twisted Matrix Laboratories. 3 | # See LICENSE for details. 4 | 5 | """ 6 | Tubes that can convert streams of data into discrete chunks and back again. 7 | """ 8 | 9 | from zope.interface import implementer 10 | 11 | from .itube import IDivertable, IFrame, ISegment 12 | from .tube import tube, series, Diverter 13 | from twisted.protocols.basic import ( 14 | LineOnlyReceiver, NetstringReceiver, Int8StringReceiver, 15 | Int16StringReceiver, Int32StringReceiver 16 | ) 17 | 18 | if 0: 19 | # Workaround for inability of pydoctor to resolve references. 20 | from twisted.internet.interfaces import ITransport 21 | ITransport 22 | from twisted.protocols import basic 23 | basic 24 | 25 | 26 | 27 | class _Transporter(object): 28 | """ 29 | Just enough of a mock of L{ITransport} to work 30 | with the protocols in L{basic}, as a wrapper around a 31 | callable taking some data. 32 | 33 | @ivar _dataWritten: 1-argument callable taking L{bytes}, a chunk of data 34 | from a stream. 35 | """ 36 | 37 | def __init__(self, dataWritten): 38 | self._dataWritten = dataWritten 39 | 40 | 41 | def write(self, data): 42 | """ 43 | Call the C{_dataWritten} callback. 44 | 45 | @param data: The data to write. 46 | @type data: L{bytes} 47 | """ 48 | self._dataWritten(data) 49 | 50 | 51 | def writeSequence(self, dati): 52 | """ 53 | Call the C{_dataWritten} callback for each element. 54 | 55 | @param dati: The sequence of data to write. 56 | @type dati: L{list} of L{bytes} 57 | """ 58 | for data in dati: 59 | self._dataWritten(data) 60 | 61 | 62 | 63 | @tube 64 | class _FramesToSegments(object): 65 | """ 66 | A tube which could convert "L{frames