├── .coveragerc ├── .gitignore ├── .travis.yml ├── CHANGELOG.rst ├── LICENSE ├── MANIFEST.in ├── README.rst ├── TODO.md ├── bench └── heap.py ├── docs ├── Makefile ├── _static │ ├── logo1.png │ └── logo2.png ├── api.rst ├── conf.py ├── index.rst └── requirements.txt ├── examples ├── tcp-basic.py ├── tcp-echo.py └── ticktock.py ├── requirements.txt ├── setup.py ├── tests ├── integration │ └── test_http.py └── unit │ ├── test_core.py │ ├── test_http.py │ ├── test_io.py │ ├── test_message.py │ ├── test_poll.py │ ├── test_process.py │ ├── test_signal.py │ ├── test_tcp.py │ ├── test_thread.py │ └── test_udp.py ├── tox.ini ├── vanilla ├── __init__.py ├── core.py ├── exception.py ├── http.py ├── io.py ├── message.py ├── meta.py ├── poll.py ├── process.py ├── signal.py ├── tcp.py ├── thread.py └── udp.py └── xx ├── notes.md ├── pipe.py ├── pipe2.py ├── react.py ├── router.py └── stream.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | concurrency = greenlet 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # python 2 | *.pyc 3 | *.pyo 4 | _trial_temp 5 | __pycache__ 6 | 7 | # osx 8 | *.DS_Store* 9 | 10 | # vim 11 | .*.swp 12 | 13 | # sphinx build folder 14 | _build 15 | 16 | *.py[co] 17 | 18 | # Packages 19 | *.egg 20 | *.egg-info 21 | dist 22 | build 23 | eggs 24 | parts 25 | bin 26 | include 27 | lib 28 | local 29 | share 30 | var 31 | sdist 32 | develop-eggs 33 | .installed.cfg 34 | 35 | # Installer logs 36 | pip-log.txt 37 | 38 | # Unit test / coverage reports 39 | .coverage 40 | .tox 41 | 42 | # Translations 43 | *.mo 44 | 45 | # Ropemacs 46 | .ropeproject 47 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | python: 2.7 4 | 5 | os: 6 | - linux 7 | 8 | env: 9 | - TOX_ENV=flake8 10 | - TOX_ENV=py27 11 | - TOX_ENV=pypy 12 | - TOX_ENV=coverage 13 | 14 | install: 15 | - pip install tox 16 | 17 | script: 18 | - tox -e $TOX_ENV 19 | -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | Change log 2 | ========== 3 | 4 | 0.1.16 5 | ------ 6 | 7 | Bug Fixes 8 | ~~~~~~~~~ 9 | 10 | * ensure process reaper is restarted after an initial batch of children have 11 | run 12 | 13 | 0.1.15 14 | ------ 15 | 16 | Features 17 | ~~~~~~~~ 18 | 19 | * synthesize appropiate exit codes if process.execv is not executable 20 | 21 | 0.1.14 22 | ------ 23 | 24 | Features 25 | ~~~~~~~~ 26 | 27 | * add a close method for HTTP Clients. add a .json convenience for HTTP 28 | Responses 29 | 30 | 0.1.13 31 | ------ 32 | 33 | Features 34 | ~~~~~~~~ 35 | 36 | * Threading support returns! 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 7 | the Software, and to permit persons to whom the Software is furnished to do so, 8 | subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 15 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 16 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 17 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 18 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 19 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include requirements.txt 2 | include README.rst 3 | include LICENSE 4 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | |Vanilla| Welcome to Vanilla! 2 | ============================= 3 | 4 | *If Go and ZeroMQ had a baby, and that baby grew up and started dating PyPy, 5 | and they had a baby, it might look like Vanilla.* 6 | 7 | Overview 8 | -------- 9 | 10 | Vanilla allows you to build concurrent software in Python. Vanilla programs are 11 | structured around independent coroutines (greenlets) which communicate with 12 | each other via Pipes. Pipes are similar to channels in Go programming. 13 | 14 | There's no callback crazyness and no monkey patching. Vanilla strives to be as 15 | explict and straightforward as possible. 16 | 17 | Documentation 18 | ------------- 19 | 20 | `Read the Docs`_ 21 | 22 | 23 | Here's how it looks: 24 | -------------------- 25 | 26 | You spawn coroutines: 27 | 28 | .. code:: python 29 | 30 | h = vanilla.Hub() 31 | 32 | def beat(message): 33 | while True: 34 | print(message) 35 | h.sleep(1000) 36 | 37 | h.spawn(beat, 'Tick') 38 | h.spawn_later(500, beat, 'Tock') 39 | # Tick / Tock / Tick / Tock 40 | 41 | Coroutines communicate via Pipes: 42 | 43 | .. code:: python 44 | 45 | h = vanilla.Hub() 46 | sender, recver = h.pipe() 47 | h.spawn(sender.send, 'Hello World') 48 | recver.recv() 49 | # 'Hello World' 50 | 51 | Pipe-fu; inspired by reactive functional patterns, Pipes can be chained: 52 | 53 | .. code:: python 54 | 55 | h = vanilla.Hub() 56 | p = h.pipe().map(lambda x: x*2) 57 | h.spawn(p.send, 4) 58 | p.recv() 59 | # 8 60 | 61 | In Vanilla, everything is a Pipe. Here's how TCP looks: 62 | 63 | .. code:: python 64 | 65 | h = vanilla.Hub() 66 | 67 | server = h.tcp.listen(port=9000) 68 | # server is a Recver which dispenses new TCP connections 69 | 70 | conn = server.recv() 71 | # conn is a Pipe you can recv and send on 72 | 73 | message = conn.recv() 74 | conn.send("Echo: " + message) 75 | 76 | Installation 77 | ------------ 78 | 79 | Vanilla works with Python 2.6 - 2.9 and PyPy. 80 | 81 | :: 82 | 83 | pip install vanilla 84 | 85 | Status 86 | ------ 87 | 88 | |Build Status|\ |Coverage Status| 89 | 90 | .. _Read the Docs: http://vanillapy.readthedocs.org/ 91 | .. |Vanilla| image:: https://raw.githubusercontent.com/cablehead/vanilla/master/docs/_static/logo2.png 92 | .. |Build Status| image:: https://img.shields.io/travis/cablehead/vanilla.svg?style=flat-square 93 | :target: https://travis-ci.org/cablehead/vanilla 94 | .. |Coverage Status| image:: https://img.shields.io/codecov/c/github/cablehead/vanilla.svg?style=flat-square 95 | :target: https://codecov.io/github/cablehead/vanilla?branch=master 96 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | - Handle disconnecting things which are piped to routers (selectors) 4 | -------------------------------------------------------------------------------- /bench/heap.py: -------------------------------------------------------------------------------- 1 | import random 2 | import heapq 3 | import time 4 | 5 | heaps = [] 6 | 7 | 8 | def benchmark(name, n, f): 9 | start = time.time() 10 | for i in xrange(n): 11 | f() 12 | print '%-20s %8.2f' % (name, n / (time.time() - start)) 13 | 14 | 15 | def push(): 16 | h = [] 17 | for i in xrange(1000): 18 | heapq.heappush(h, (random.random(), i)) 19 | heaps.append(h) 20 | 21 | 22 | def pop(): 23 | h = heaps.pop() 24 | while h: 25 | heapq.heappop(h) 26 | 27 | 28 | benchmark("push", 100, push) 29 | benchmark("pop", 100, pop) 30 | 31 | benchmark("push", 100, push) 32 | benchmark("pop", 100, pop) 33 | 34 | benchmark("push", 100, push) 35 | benchmark("pop", 100, pop) 36 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 36 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 38 | @echo " text to make text files" 39 | @echo " man to make manual pages" 40 | @echo " texinfo to make Texinfo files" 41 | @echo " info to make Texinfo files and run them through makeinfo" 42 | @echo " gettext to make PO message catalogs" 43 | @echo " changes to make an overview of all changed/added/deprecated items" 44 | @echo " xml to make Docutils-native XML files" 45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 46 | @echo " linkcheck to check all external links for integrity" 47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 48 | 49 | clean: 50 | rm -rf $(BUILDDIR)/* 51 | 52 | html: 53 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 54 | @echo 55 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 56 | 57 | dirhtml: 58 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 59 | @echo 60 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 61 | 62 | singlehtml: 63 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 64 | @echo 65 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 66 | 67 | pickle: 68 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 69 | @echo 70 | @echo "Build finished; now you can process the pickle files." 71 | 72 | json: 73 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 74 | @echo 75 | @echo "Build finished; now you can process the JSON files." 76 | 77 | htmlhelp: 78 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 79 | @echo 80 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 81 | ".hhp project file in $(BUILDDIR)/htmlhelp." 82 | 83 | qthelp: 84 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 85 | @echo 86 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 87 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 88 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Vanilla.qhcp" 89 | @echo "To view the help file:" 90 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Vanilla.qhc" 91 | 92 | devhelp: 93 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 94 | @echo 95 | @echo "Build finished." 96 | @echo "To view the help file:" 97 | @echo "# mkdir -p $$HOME/.local/share/devhelp/Vanilla" 98 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Vanilla" 99 | @echo "# devhelp" 100 | 101 | epub: 102 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 103 | @echo 104 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 105 | 106 | latex: 107 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 108 | @echo 109 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 110 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 111 | "(use \`make latexpdf' here to do that automatically)." 112 | 113 | latexpdf: 114 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 115 | @echo "Running LaTeX files through pdflatex..." 116 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 117 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 118 | 119 | latexpdfja: 120 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 121 | @echo "Running LaTeX files through platex and dvipdfmx..." 122 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 123 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 124 | 125 | text: 126 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 127 | @echo 128 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 129 | 130 | man: 131 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 132 | @echo 133 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 134 | 135 | texinfo: 136 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 137 | @echo 138 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 139 | @echo "Run \`make' in that directory to run these through makeinfo" \ 140 | "(use \`make info' here to do that automatically)." 141 | 142 | info: 143 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 144 | @echo "Running Texinfo files through makeinfo..." 145 | make -C $(BUILDDIR)/texinfo info 146 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 147 | 148 | gettext: 149 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 150 | @echo 151 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 152 | 153 | changes: 154 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 155 | @echo 156 | @echo "The overview file is in $(BUILDDIR)/changes." 157 | 158 | linkcheck: 159 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 160 | @echo 161 | @echo "Link check complete; look for any errors in the above output " \ 162 | "or in $(BUILDDIR)/linkcheck/output.txt." 163 | 164 | doctest: 165 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 166 | @echo "Testing of doctests in the sources finished, look at the " \ 167 | "results in $(BUILDDIR)/doctest/output.txt." 168 | 169 | xml: 170 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 171 | @echo 172 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 173 | 174 | pseudoxml: 175 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 176 | @echo 177 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 178 | -------------------------------------------------------------------------------- /docs/_static/logo1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cablehead/vanilla/c9f5b86f45720a30e8840fb68b1429b919c4ca66/docs/_static/logo1.png -------------------------------------------------------------------------------- /docs/_static/logo2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cablehead/vanilla/c9f5b86f45720a30e8840fb68b1429b919c4ca66/docs/_static/logo2.png -------------------------------------------------------------------------------- /docs/api.rst: -------------------------------------------------------------------------------- 1 | ================= 2 | API Documentation 3 | ================= 4 | 5 | .. testsetup:: 6 | 7 | import vanilla 8 | h = vanilla.Hub() 9 | 10 | Hub 11 | === 12 | 13 | .. autoclass:: vanilla.core.Hub 14 | 15 | Concurrency 16 | ----------- 17 | 18 | .. automethod:: vanilla.core.Hub.spawn 19 | 20 | .. automethod:: vanilla.core.Hub.spawn_later 21 | 22 | .. automethod:: vanilla.core.Hub.sleep 23 | 24 | Message Passing 25 | --------------- 26 | 27 | .. automethod:: vanilla.core.Hub.pipe 28 | 29 | .. automethod:: vanilla.core.Hub.select 30 | 31 | .. automethod:: vanilla.core.Hub.dealer 32 | 33 | .. automethod:: vanilla.core.Hub.router 34 | 35 | .. automethod:: vanilla.core.Hub.queue 36 | 37 | .. automethod:: vanilla.core.Hub.channel 38 | 39 | .. automethod:: vanilla.core.Hub.state 40 | 41 | Pipe Conveniences 42 | ----------------- 43 | 44 | .. automethod:: vanilla.core.Hub.producer 45 | 46 | .. automethod:: vanilla.core.Hub.pulse 47 | 48 | Thread 49 | ------ 50 | 51 | .. py:method:: Hub.thread.spawn(f, *a) 52 | 53 | - Spawns callable *f* in a new thread. A new Hub is initialized for the 54 | thread and passed to *f* along with arguments *a* 55 | - A *parent* attribute is available on the new thread's Hub which is a 56 | `Pipe`_ to communicate with it's parent thread 57 | - spawn returns a `Pipe`_ to communicate with the new child thread 58 | 59 | Example usage:: 60 | 61 | def ticker(h, n): 62 | import time 63 | 64 | while true: 65 | h.parent.send(time.time()) 66 | time.sleep(n) 67 | 68 | h = vanilla.Hub() 69 | 70 | child = h.thread.spawn(ticker, 1) 71 | 72 | while true: 73 | child.recv() 74 | 75 | .. py:method:: Hub.thread.call(f, *a) 76 | 77 | - Spawns a one-off thread to run callable *f* with arguments *a* 78 | - Returns a `Recver`_ which can be recv'd on to get *f*'s result 79 | 80 | Example usage:: 81 | 82 | def add(a, b): 83 | return a + b 84 | 85 | h = vanilla.Hub() 86 | h.thread.call(add, 2, 3).recv() # 5 87 | 88 | .. py:method:: Hub.thread.pool(size) 89 | 90 | - Returns a reusable pool of *size* threads 91 | 92 | Pool 93 | ~~~~ 94 | 95 | .. py:method:: Thread.Pool.call(f, *a) 96 | 97 | - Runs callable *f* with arguments *a* on one of the pool's threads 98 | - Returns a `Recver`_ which can be recv'd on to get *f*'s result 99 | 100 | Example usage:: 101 | 102 | h = vanilla.Hub() 103 | 104 | def sleeper(x): 105 | time.sleep(x) 106 | return x 107 | 108 | p = h.thread.pool(2) 109 | gather = h.router() 110 | 111 | p.call(sleeper, 0.2).pipe(gather) 112 | p.call(sleeper, 0.1).pipe(gather) 113 | p.call(sleeper, 0.05).pipe(gather) 114 | 115 | gather.recv() # 0.1 116 | gather.recv() # 0.05 117 | gather.recv() # 0.2 118 | 119 | .. py:method:: Thread.Pool.wrap(ob) 120 | 121 | - Wraps *ob* with a proxy which will delegate method calls on *ob* to run on 122 | the pool's threads 123 | - Each method call on the proxy returns a `Recver`_ which can be recv'd on 124 | the get the calls result 125 | 126 | Example usage:: 127 | 128 | h = vanilla.Hub() 129 | 130 | p = h.thread.pool(2) 131 | 132 | db = pymongo.MongoClient()['database'] 133 | db = p.wrap(db) 134 | 135 | response = db.posts.find_one({"author": "Mike"}) 136 | response.recv() 137 | 138 | Process 139 | ------- 140 | 141 | .. py:method:: Hub.process.execv(args, env=None, stderrtoout=False) 142 | 143 | - Forks a child process using args. 144 | - *env* is an optional dictionary of environment variables which will 145 | replace the parent's environment for the child process. If not supplied 146 | the child will have access to the parent's environment. 147 | - if *stderrtoout* is *True* the child's stderr will be redirected to its 148 | stdout. 149 | - A `Child`_ object is return to interact with the child process. 150 | 151 | Example usage:: 152 | 153 | h = vanilla.Hub() 154 | 155 | child = h.process.execv( 156 | ['/usr/bin/env', 'grep', '--line-buffered', 'foo']) 157 | 158 | child.stdin.send('foo1\n') 159 | child.stdout.recv_partition('\n') # foo1 160 | child.stdin.send('bar1\n') 161 | child.stdout.recv_partition('\n') # would hang forever 162 | child.stdin.send('foo2\n') 163 | child.stdout.recv_partition('\n') # foo2 164 | 165 | child.terminate() 166 | child.done.recv() 167 | 168 | Child 169 | ~~~~~ 170 | 171 | .. py:attribute:: Child.stdin 172 | 173 | A `Sender`_ which allows you to send data to the child's stdin. 174 | 175 | .. py:attribute:: Child.stdout 176 | 177 | A `Stream`_ recver which allows you to receive data from the child's 178 | stdout. 179 | 180 | .. py:attribute:: Child.stderr 181 | 182 | A `Stream`_ recver which allows you to receive data from the child's 183 | stderr. Only available if *stderrtoout* is *False*. 184 | 185 | .. py:attribute:: Child.done 186 | 187 | A `State`_ that will be set once the child terminates. 188 | 189 | .. py:method:: Child.terminate() 190 | 191 | Sends the child a SIGTERM. 192 | 193 | .. py:method:: Child.signal(signum) 194 | 195 | Sends the child *signum*. 196 | 197 | TCP 198 | --- 199 | 200 | .. py:method:: Hub.tcp.listen(port=0, host='127.0.0.1') 201 | 202 | Listens for TCP connections on *host* and *port*. If *port* is 0, it will 203 | listen on a randomly available port. Returns a `Recver`_ which dispenses 204 | TCP connections:: 205 | 206 | h = vanilla.Hub() 207 | 208 | server = h.tcp.listen() 209 | 210 | @server.consume 211 | def echo(conn): 212 | for date in conn.recver: 213 | conn.send('Echo: ' + data) 214 | 215 | The Recver returned has an additional attribute *port* which is the port 216 | that was bound to. 217 | 218 | .. py:method:: Hub.tcp.connect(port, host='127.0.0.1') 219 | 220 | Creates a TCP connection to *host* and *port* and returns a `Pair`_ of a 221 | `Sender`_ and `Stream`_ receiver. 222 | 223 | HTTP 224 | ---- 225 | 226 | HTTPServer 227 | ~~~~~~~~~~ 228 | 229 | .. py:method:: Hub.http.listen(port=0, host='127.0.0.1') 230 | 231 | - Listens for HTTP connections on *host* and *port*. If *port* is 0, it will 232 | listen on a randomly available port. 233 | 234 | - Returns a `Recver`_ which dispenses HTTP connections. 235 | 236 | - These HTTP connections are a `Recver`_ which dispense 237 | `HTTPRequest`_. Note that if this is a Keep-Alive connection, it can 238 | dispense more than one `HTTPRequest`_. 239 | 240 | An example server:: 241 | 242 | import vanilla 243 | 244 | h = vanilla.Hub() 245 | 246 | def handle_connection(conn): 247 | for request in conn: 248 | request.reply(vanilla.http.Status(200), {}, "Hello") 249 | 250 | server = h.http.listen(8080) 251 | 252 | for conn in server: 253 | h.spawn(handle_connection, conn) 254 | 255 | HTTPRequest 256 | ~~~~~~~~~~~ 257 | 258 | A HTTP Request is a namedtuple with the following ordered items / attributes: 259 | 260 | .. py:attribute:: Request.method 261 | 262 | The HTTP request method e.g. 'GET', 'POST', 'PUT', 'DELETE', ... 263 | 264 | .. py:attribute:: Request.path 265 | 266 | The path requested 267 | 268 | .. py:attribute:: Request.version 269 | 270 | The HTTP version of the request 271 | 272 | .. py:attribute:: Request.headers 273 | 274 | A dictionary like interface to HTTP request headers. Keys are case 275 | insensitive. 276 | 277 | .. py:attribute:: Request.body 278 | 279 | A `Recver`_ which yields the request's body. If the Transfer-Encoding is 280 | chunked the entire body could be yielded over a period of time with 281 | successive receives. 282 | 283 | A HTTP Request also has three methods: 284 | 285 | .. py:method:: Request.consume() 286 | 287 | Blocks until the entire request body has been received and returns it as a 288 | single string. 289 | 290 | .. py:method:: Request.json() 291 | 292 | Convenience to consume the entire request body, and json decode it. 293 | 294 | .. py:attribute:: Request.form 295 | 296 | A convenience to access form url encoded data as a dictionary. The form 297 | data is available as a key, value mappings. If a key is in the form more 298 | than once, only it's last value will be available. 299 | 300 | .. py:attribute:: Request.form_multi 301 | 302 | A convenience to access form url encoded data as a dictionary. The form 303 | data is available as a key, list of values mappings. 304 | 305 | .. py:method:: Request.reply(status, headers, body) 306 | 307 | Initiates a reply to this HTTP request. *status* is a tuple of (HTTP Code, 308 | message), for example (200, 'OK'). *headers* is a dictionary like interface 309 | to the HTTP headers to respond with. *body* can either be a string, in which 310 | case this response will be completed immediately. Otherwise, *body* can be a 311 | `Recver`_ which can have a series of strings sent, before being closed to 312 | indicated the response has completed. There's no need to set Content-Length 313 | or Transfer-Encoding in the response headers, this will be inferred 314 | depending on whether *body* is a string or a `Recver`_. 315 | 316 | .. py:method:: Request.upgrade() 317 | 318 | If this is a request to establish a `Websocket`_, the server can call this 319 | method to upgrade this connection. This method returns a `Websocket`_, and 320 | this connection can no longer be used as a HTTP connection. 321 | 322 | .. py:method:: Hub.http.connect(port, host='127.0.0.1') 323 | 324 | Establishes a `HTTPClient`_ connection to *host* and *port* and requests a 325 | HTTP client connection. Note that if supported, this connection will be a 326 | Keep-Alive and multiple requests can be made over the same connection. 327 | 328 | An example server with chunked transfer:: 329 | 330 | import vanilla 331 | 332 | h = vanilla.Hub() 333 | 334 | 335 | serve = h.http.listen() 336 | 337 | client = h.http.connect('http://localhost:%s' % serve.port) 338 | response = client.get('/') 339 | 340 | 341 | conn = serve.recv() # recvs http connection 342 | request = conn.recv() # recvs http request + headers 343 | 344 | sender, recver = h.pipe() 345 | request.reply(vanilla.http.Status(200), {}, recver) 346 | 347 | response = response.recv() # recvs the response + headers, but not the body 348 | 349 | sender.send('oh') 350 | print response.body.recv() # 'oh' 351 | 352 | sender.send('hai') 353 | print response.body.recv() # 'hai' 354 | 355 | sender.close() 356 | print response.body.recv() # raises Closed 357 | 358 | HTTPClient 359 | ~~~~~~~~~~ 360 | 361 | .. autoclass:: vanilla.http.HTTPClient() 362 | :members: request, get, post, put, delete, websocket 363 | :undoc-members: 364 | 365 | 366 | Message Passing Primitives 367 | ========================== 368 | 369 | Pair 370 | ---- 371 | 372 | .. autoclass:: vanilla.message.Pair 373 | :members: send, recv, pipe, map, consume, close 374 | 375 | Sender 376 | ------ 377 | 378 | .. autoclass:: vanilla.message.Sender 379 | :members: send 380 | 381 | Recver 382 | ------ 383 | 384 | .. autoclass:: vanilla.message.Recver 385 | :members: 386 | 387 | Pipe 388 | ---- 389 | 390 | .. autoclass:: vanilla.message.Pipe 391 | 392 | Dealer 393 | ------ 394 | 395 | .. autoclass:: vanilla.message.Dealer 396 | 397 | Router 398 | ------ 399 | 400 | .. autoclass:: vanilla.message.Router 401 | 402 | Queue 403 | ----- 404 | 405 | .. automethod:: vanilla.message.Queue 406 | 407 | Stream 408 | ------ 409 | 410 | .. autoclass:: vanilla.message.Stream 411 | 412 | .. autoclass:: vanilla.message::Stream.Recver 413 | :members: 414 | 415 | State 416 | ----- 417 | 418 | .. automethod:: vanilla.message.State 419 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Vanilla documentation build configuration file, created by 4 | # sphinx-quickstart on Thu Oct 2 11:25:45 2014. 5 | # 6 | # This file is execfile()d with the current directory set to its 7 | # containing dir. 8 | # 9 | # Note that not all possible configuration values are present in this 10 | # autogenerated file. 11 | # 12 | # All configuration values have a default; values that are commented out 13 | # serve to show the default. 14 | 15 | import sys 16 | import os 17 | 18 | # If extensions (or modules to document with autodoc) are in another directory, 19 | # add these directories to sys.path here. If the directory is relative to the 20 | # documentation root, use os.path.abspath to make it absolute, like shown here. 21 | #sys.path.insert(0, os.path.abspath('.')) 22 | 23 | # -- General configuration ------------------------------------------------ 24 | 25 | # If your documentation needs a minimal Sphinx version, state it here. 26 | #needs_sphinx = '1.0' 27 | 28 | # Add any Sphinx extension module names here, as strings. They can be 29 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 30 | # ones. 31 | extensions = [ 32 | 'sphinx.ext.autodoc', 33 | 'sphinx.ext.doctest', 34 | # 'sphinx.ext.viewcode', 35 | ] 36 | 37 | # Add any paths that contain templates here, relative to this directory. 38 | templates_path = ['_templates'] 39 | 40 | # The suffix of source filenames. 41 | source_suffix = '.rst' 42 | 43 | # The encoding of source files. 44 | #source_encoding = 'utf-8-sig' 45 | 46 | # The master toctree document. 47 | master_doc = 'index' 48 | 49 | # General information about the project. 50 | project = u'Vanilla' 51 | copyright = u'2015, Andy Gayton' 52 | 53 | # The version info for the project you're documenting, acts as replacement for 54 | # |version| and |release|, also used in various other places throughout the 55 | # built documents. 56 | # 57 | # The short X.Y version. 58 | version = '0.0.1' 59 | # The full version, including alpha/beta/rc tags. 60 | release = '0.0.1' 61 | 62 | # The language for content autogenerated by Sphinx. Refer to documentation 63 | # for a list of supported languages. 64 | #language = None 65 | 66 | # There are two options for replacing |today|: either, you set today to some 67 | # non-false value, then it is used: 68 | #today = '' 69 | # Else, today_fmt is used as the format for a strftime call. 70 | #today_fmt = '%B %d, %Y' 71 | 72 | # List of patterns, relative to source directory, that match files and 73 | # directories to ignore when looking for source files. 74 | exclude_patterns = ['_build'] 75 | 76 | # The reST default role (used for this markup: `text`) to use for all 77 | # documents. 78 | #default_role = None 79 | 80 | # If true, '()' will be appended to :func: etc. cross-reference text. 81 | #add_function_parentheses = True 82 | 83 | # If true, the current module name will be prepended to all description 84 | # unit titles (such as .. function::). 85 | #add_module_names = True 86 | 87 | # If true, sectionauthor and moduleauthor directives will be shown in the 88 | # output. They are ignored by default. 89 | #show_authors = False 90 | 91 | # The name of the Pygments (syntax highlighting) style to use. 92 | pygments_style = 'sphinx' 93 | 94 | # A list of ignored prefixes for module index sorting. 95 | #modindex_common_prefix = [] 96 | 97 | # If true, keep warnings as "system message" paragraphs in the built documents. 98 | #keep_warnings = False 99 | 100 | 101 | # -- Options for HTML output ---------------------------------------------- 102 | 103 | 104 | import alabaster 105 | 106 | html_theme_path = [alabaster.get_path()] 107 | extensions.append('alabaster') 108 | html_theme = 'alabaster' 109 | html_sidebars = { 110 | '**': [ 111 | 'about.html', 'navigation.html', 112 | # 'searchbox.html', 'donate.html', 113 | ] 114 | } 115 | 116 | html_theme_options = { 117 | 'github_user': 'cablehead', 118 | 'github_repo': 'vanilla', 119 | 'logo': 'logo1.png', 120 | 'logo_name': True, 121 | } 122 | 123 | # The name for this set of Sphinx documents. If None, it defaults to 124 | # " v documentation". 125 | #html_title = None 126 | 127 | # A shorter title for the navigation bar. Default is the same as html_title. 128 | #html_short_title = None 129 | 130 | # The name of an image file (relative to this directory) to place at the top 131 | # of the sidebar. 132 | #html_logo = None 133 | 134 | # The name of an image file (within the static path) to use as favicon of the 135 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 136 | # pixels large. 137 | #html_favicon = None 138 | 139 | # Add any paths that contain custom static files (such as style sheets) here, 140 | # relative to this directory. They are copied after the builtin static files, 141 | # so a file named "default.css" will overwrite the builtin "default.css". 142 | html_static_path = ['_static'] 143 | 144 | # Add any extra paths that contain custom files (such as robots.txt or 145 | # .htaccess) here, relative to this directory. These files are copied 146 | # directly to the root of the documentation. 147 | #html_extra_path = [] 148 | 149 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 150 | # using the given strftime format. 151 | #html_last_updated_fmt = '%b %d, %Y' 152 | 153 | # If true, SmartyPants will be used to convert quotes and dashes to 154 | # typographically correct entities. 155 | #html_use_smartypants = True 156 | 157 | # Custom sidebar templates, maps document names to template names. 158 | #html_sidebars = {} 159 | 160 | # Additional templates that should be rendered to pages, maps page names to 161 | # template names. 162 | #html_additional_pages = {} 163 | 164 | # If false, no module index is generated. 165 | #html_domain_indices = True 166 | 167 | # If false, no index is generated. 168 | #html_use_index = True 169 | 170 | # If true, the index is split into individual pages for each letter. 171 | #html_split_index = False 172 | 173 | # If true, links to the reST sources are added to the pages. 174 | #html_show_sourcelink = True 175 | 176 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 177 | #html_show_sphinx = True 178 | 179 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 180 | #html_show_copyright = True 181 | 182 | # If true, an OpenSearch description file will be output, and all pages will 183 | # contain a tag referring to it. The value of this option must be the 184 | # base URL from which the finished HTML is served. 185 | #html_use_opensearch = '' 186 | 187 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 188 | #html_file_suffix = None 189 | 190 | # Output file base name for HTML help builder. 191 | htmlhelp_basename = 'Vanilladoc' 192 | 193 | 194 | # -- Options for LaTeX output --------------------------------------------- 195 | 196 | latex_elements = { 197 | # The paper size ('letterpaper' or 'a4paper'). 198 | #'papersize': 'letterpaper', 199 | 200 | # The font size ('10pt', '11pt' or '12pt'). 201 | #'pointsize': '10pt', 202 | 203 | # Additional stuff for the LaTeX preamble. 204 | #'preamble': '', 205 | } 206 | 207 | # Grouping the document tree into LaTeX files. List of tuples 208 | # (source start file, target name, title, 209 | # author, documentclass [howto, manual, or own class]). 210 | latex_documents = [ 211 | ('index', 'Vanilla.tex', u'Vanilla Documentation', 212 | u'Andy Gayton', 'manual'), 213 | ] 214 | 215 | # The name of an image file (relative to this directory) to place at the top of 216 | # the title page. 217 | #latex_logo = None 218 | 219 | # For "manual" documents, if this is true, then toplevel headings are parts, 220 | # not chapters. 221 | #latex_use_parts = False 222 | 223 | # If true, show page references after internal links. 224 | #latex_show_pagerefs = False 225 | 226 | # If true, show URL addresses after external links. 227 | #latex_show_urls = False 228 | 229 | # Documents to append as an appendix to all manuals. 230 | #latex_appendices = [] 231 | 232 | # If false, no module index is generated. 233 | #latex_domain_indices = True 234 | 235 | 236 | # -- Options for manual page output --------------------------------------- 237 | 238 | # One entry per manual page. List of tuples 239 | # (source start file, name, description, authors, manual section). 240 | man_pages = [ 241 | ('index', 'vanilla', u'Vanilla Documentation', 242 | [u'Andy Gayton'], 1) 243 | ] 244 | 245 | # If true, show URL addresses after external links. 246 | #man_show_urls = False 247 | 248 | 249 | # -- Options for Texinfo output ------------------------------------------- 250 | 251 | # Grouping the document tree into Texinfo files. List of tuples 252 | # (source start file, target name, title, author, 253 | # dir menu entry, description, category) 254 | texinfo_documents = [ 255 | ('index', 'Vanilla', u'Vanilla Documentation', 256 | u'Andy Gayton', 'Vanilla', 'One line description of project.', 257 | 'Miscellaneous'), 258 | ] 259 | 260 | # Documents to append as an appendix to all manuals. 261 | #texinfo_appendices = [] 262 | 263 | # If false, no module index is generated. 264 | #texinfo_domain_indices = True 265 | 266 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 267 | #texinfo_show_urls = 'footnote' 268 | 269 | # If true, do not generate a @detailmenu in the "Top" node's menu. 270 | #texinfo_no_detailmenu = False 271 | 272 | autodoc_member_order = 'bysource' 273 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Welcome to Vanilla! 2 | =================== 3 | 4 | .. include:: ../README.rst 5 | :start-line: 5 6 | 7 | Contents 8 | -------- 9 | 10 | .. toctree:: 11 | :maxdepth: 3 12 | 13 | api 14 | 15 | Acknowledgements 16 | ---------------- 17 | 18 | The Twisted Project and the entire Twisted community for the strong 19 | grounding in `Evented 20 | Async `__. 21 | Libor Michalek and Slide Inc for showing me using `Python 22 | coroutines `__ at phenomenal scale 23 | isn't completely insane. Mike Johnston and Fred Cheng for 24 | `Simplenote `__, 25 | `Simperium `__, the chance to experiment with 26 | how apps will work with realtime APIs and for making it possible for me 27 | to live in San Francisco. `Littleinc `__ for 28 | giving me a chance to send a lot of messages and to open source `this 29 | project `__ which I started 30 | working on while there. Justin Rosenthal for believing in me, 30% of the 31 | time. Andres Buritica who did a lot of the early HTTP and UDP fragment 32 | (not pushed just yet!) work and Alison Kosinski, the coolest wife in 33 | the world. 34 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | alabaster 2 | -------------------------------------------------------------------------------- /examples/tcp-basic.py: -------------------------------------------------------------------------------- 1 | import vanilla 2 | 3 | h = vanilla.Hub() 4 | 5 | server = h.tcp.listen() 6 | 7 | print("Listening on port: {0}".format(server.port)) 8 | 9 | for conn in server: 10 | conn.send('Hi\n') 11 | conn.close() 12 | -------------------------------------------------------------------------------- /examples/tcp-echo.py: -------------------------------------------------------------------------------- 1 | import vanilla 2 | 3 | h = vanilla.Hub() 4 | 5 | server = h.tcp.listen() 6 | 7 | print("Listening on port: {0}".format(server.port)) 8 | 9 | 10 | @server.consume 11 | def echo(conn): 12 | for data in conn.recver: 13 | conn.send(data) 14 | 15 | h.stop_on_term() 16 | -------------------------------------------------------------------------------- /examples/ticktock.py: -------------------------------------------------------------------------------- 1 | import vanilla 2 | 3 | h = vanilla.Hub() 4 | 5 | r = h.router() 6 | 7 | 8 | def beat(message): 9 | while True: 10 | r.send(message) 11 | h.sleep(1000) 12 | 13 | h.spawn(beat, 'Tick') 14 | h.spawn_later(500, beat, 'Tock') 15 | 16 | for message in r.recver: 17 | print(message) 18 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | greenlet 2 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import re 3 | 4 | from setuptools.command.test import test as TestCommand 5 | from setuptools import setup 6 | from setuptools import find_packages 7 | 8 | 9 | metadata = dict( 10 | re.findall("__([a-z]+)__ = '([^']+)'", open('vanilla/meta.py').read())) 11 | 12 | requirements = [ 13 | x.strip() for x 14 | in open('requirements.txt').readlines() if not x.startswith('#')] 15 | 16 | 17 | description = "Straightforward concurrency for Python " \ 18 | "http://vanillapy.readthedocs.org/" 19 | 20 | 21 | class PyTest(TestCommand): 22 | def finalize_options(self): 23 | TestCommand.finalize_options(self) 24 | self.test_args = [] 25 | self.test_suite = True 26 | 27 | def run_tests(self): 28 | import pytest 29 | errno = pytest.main(self.test_args) 30 | sys.exit(errno) 31 | 32 | 33 | setup( 34 | name='vanilla', 35 | version=metadata['version'], 36 | author='Andy Gayton', 37 | author_email='andy@thecablelounge.com', 38 | install_requires=requirements, 39 | packages=find_packages(), 40 | url='https://github.com/cablehead/vanilla', 41 | license='MIT', 42 | description=description, 43 | long_description=open('README.rst').read(), 44 | tests_require=['pytest'], 45 | cmdclass={'test': PyTest}, 46 | ) 47 | -------------------------------------------------------------------------------- /tests/integration/test_http.py: -------------------------------------------------------------------------------- 1 | import json 2 | import time 3 | 4 | import pytest 5 | 6 | import vanilla 7 | 8 | 9 | @pytest.mark.parametrize('scheme', ['http', 'https']) 10 | class TestHTTP(object): 11 | def test_get_basic(self, scheme): 12 | h = vanilla.Hub() 13 | conn = h.http.connect('%s://httpbin.org' % scheme) 14 | response = conn.get('/get', params={'foo': 'bar'}).recv() 15 | assert response.status.code == 200 16 | assert json.loads(response.consume())['args'] == {'foo': 'bar'} 17 | 18 | @pytest.mark.skipif(True, reason='TODO') 19 | def test_get_keepalive(self, scheme): 20 | h = vanilla.Hub() 21 | 22 | conn = h.http.connect('%s://httpbin.org' % scheme) 23 | 24 | get1 = conn.get('/get', params={'foo': 'bar'}) 25 | drip = conn.get( 26 | '/drip', params={'numbytes': 3, 'duration': 3, 'delay': 1}) 27 | get2 = conn.get('/get', params={'foo': 'bar2'}) 28 | 29 | response = get1.recv() 30 | assert response.status.code == 200 31 | assert json.loads(response.consume())['args'] == {'foo': 'bar'} 32 | 33 | # assert the first payload from drip takes roughly a second 34 | start = time.time() 35 | response = drip.recv() 36 | took, start = time.time() - start, time.time() 37 | assert scheme == 'https' or 1.5 > took > .9 38 | assert response.status.code == 200 39 | 40 | # response should be chunked 41 | assert response.headers['transfer-encoding'] == 'chunked' 42 | 43 | # the first chunk should come immediately 44 | assert response.body.recv() == '*' 45 | took, start = time.time() - start, time.time() 46 | assert scheme == 'https' or took < 0.005 47 | 48 | # check remaining chunks come every second 49 | for item in response.body: 50 | took, start = time.time() - start, time.time() 51 | assert item == '*' 52 | assert scheme == 'https' or 1.4 > took > .8 53 | 54 | response = get2.recv() 55 | assert response.status.code == 200 56 | assert json.loads(response.consume())['args'] == {'foo': 'bar2'} 57 | 58 | def test_post(self, scheme): 59 | h = vanilla.Hub() 60 | 61 | conn = h.http.connect('%s://httpbin.org' % scheme) 62 | 63 | response = conn.post('/post', data='toby').recv() 64 | assert response.status.code == 200 65 | body = response.consume() 66 | assert json.loads(body)['data'] == 'toby' 67 | 68 | def test_post_form_encoded(self, scheme): 69 | h = vanilla.Hub() 70 | data = {'k1': 'v1', 'k2': 'v2'} 71 | conn = h.http.connect('%s://httpbin.org' % scheme) 72 | response = conn.post('/post', data=data).recv() 73 | assert response.status.code == 200 74 | body = response.consume() 75 | assert json.loads(body)['form'] == data 76 | 77 | 78 | def test_WebSocketClient(): 79 | h = vanilla.Hub() 80 | 81 | conn = h.http.connect('ws://echo.websocket.org') 82 | ws = conn.websocket('/') 83 | 84 | message = 'x' * 125 85 | ws.send(message) 86 | assert ws.recv() == message 87 | 88 | message = 'x' * 126 89 | ws.send(message) 90 | assert ws.recv() == message 91 | 92 | message = 'x' * 65535 93 | ws.send(message) 94 | assert ws.recv() == message 95 | 96 | message = 'x' * 65536 97 | ws.send(message) 98 | assert ws.recv() == message 99 | -------------------------------------------------------------------------------- /tests/unit/test_core.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | import vanilla 4 | import vanilla.core 5 | 6 | 7 | def test_lazy(): 8 | class C(object): 9 | @vanilla.core.lazy 10 | def now(self): 11 | return time.time() 12 | 13 | c = C() 14 | want = c.now 15 | time.sleep(0.01) 16 | assert c.now == want 17 | 18 | 19 | def test_Scheduler(): 20 | s = vanilla.core.Scheduler() 21 | s.add(4, 'f2') 22 | s.add(9, 'f4') 23 | s.add(3, 'f1') 24 | item3 = s.add(7, 'f3') 25 | 26 | assert 0.003 - s.timeout() < 0.001 27 | assert len(s) == 4 28 | 29 | s.remove(item3) 30 | assert 0.003 - s.timeout() < 0.001 31 | assert len(s) == 3 32 | 33 | assert s.pop() == ('f1', ()) 34 | assert 0.004 - s.timeout() < 0.001 35 | assert len(s) == 2 36 | 37 | assert s.pop() == ('f2', ()) 38 | assert 0.009 - s.timeout() < 0.001 39 | assert len(s) == 1 40 | 41 | assert s.pop() == ('f4', ()) 42 | assert not s 43 | 44 | 45 | class TestHub(object): 46 | def test_spawn(self): 47 | h = vanilla.Hub() 48 | a = [] 49 | 50 | h.spawn_later(10, lambda: a.append(1)) 51 | h.spawn(lambda: a.append(2)) 52 | 53 | h.sleep(1) 54 | assert a == [2] 55 | 56 | h.sleep(10) 57 | assert a == [2, 1] 58 | 59 | def test_exception(self): 60 | h = vanilla.Hub() 61 | 62 | def raiser(): 63 | raise Exception() 64 | h.spawn(raiser) 65 | h.sleep(1) 66 | 67 | a = [] 68 | h.spawn(lambda: a.append(2)) 69 | h.sleep(1) 70 | assert a == [2] 71 | 72 | def test_stop(self): 73 | h = vanilla.Hub() 74 | 75 | @h.spawn 76 | def _(): 77 | h.sleep(20) 78 | 79 | h.stop() 80 | -------------------------------------------------------------------------------- /tests/unit/test_http.py: -------------------------------------------------------------------------------- 1 | import json 2 | import gc 3 | 4 | import pytest 5 | 6 | import vanilla 7 | 8 | import vanilla.http 9 | 10 | 11 | # TODO: remove 12 | import logging 13 | logging.basicConfig() 14 | 15 | 16 | class TestHTTP(object): 17 | def test_get_body(self): 18 | h = vanilla.Hub() 19 | 20 | serve = h.http.listen() 21 | 22 | @h.spawn 23 | def _(): 24 | conn = serve.recv() 25 | for request in conn: 26 | request.reply(vanilla.http.Status(200), {}, request.path) 27 | 28 | uri = 'http://localhost:%s' % serve.port 29 | conn = h.http.connect(uri) 30 | 31 | response = conn.get('/') 32 | response = response.recv() 33 | assert response.status.code == 200 34 | assert response.consume() == '/' 35 | assert response.headers['Date'] 36 | 37 | response = conn.get('/toby').recv() 38 | assert response.status.code == 200 39 | assert response.consume() == '/toby' 40 | h.stop() 41 | assert not h.registered 42 | 43 | def test_get_chunked(self): 44 | h = vanilla.Hub() 45 | 46 | serve = h.http.listen() 47 | 48 | @h.spawn 49 | def _(): 50 | conn = serve.recv() 51 | for request in conn: 52 | sender, recver = h.pipe() 53 | request.reply(vanilla.http.Status(200), {}, recver) 54 | 55 | for i in xrange(3): 56 | h.sleep(10) 57 | sender.send(str(i)) 58 | 59 | if len(request.path) > 1: 60 | sender.send(request.path[1:]) 61 | 62 | sender.close() 63 | 64 | uri = 'http://localhost:%s' % serve.port 65 | conn = h.http.connect(uri) 66 | 67 | response = conn.get('/').recv() 68 | assert response.status.code == 200 69 | assert list(response.body) == ['0', '1', '2'] 70 | 71 | response = conn.get('/peace').recv() 72 | assert response.status.code == 200 73 | assert list(response.body) == ['0', '1', '2', 'peace'] 74 | h.stop() 75 | assert not h.registered 76 | 77 | def test_post(self): 78 | h = vanilla.Hub() 79 | 80 | serve = h.http.listen() 81 | 82 | @h.spawn 83 | def _(): 84 | conn = serve.recv() 85 | for request in conn: 86 | request.reply(vanilla.http.Status(200), {}, request.consume()) 87 | 88 | uri = 'http://localhost:%s' % serve.port 89 | conn = h.http.connect(uri) 90 | 91 | response = conn.post('/').recv() 92 | assert response.status.code == 200 93 | assert response.consume() == '' 94 | 95 | response = conn.post('/', data='toby').recv() 96 | assert response.status.code == 200 97 | assert response.consume() == 'toby' 98 | h.stop() 99 | 100 | @pytest.mark.skipif(True, reason='TODO') 101 | def test_post_chunked(self): 102 | print 103 | print 104 | h = vanilla.Hub() 105 | 106 | serve = h.http.listen() 107 | 108 | @h.spawn 109 | def _(): 110 | conn = serve.recv() 111 | for request in conn: 112 | sender, recver = h.pipe() 113 | request.reply(vanilla.http.Status(200), {}, recver) 114 | for data in request.body: 115 | sender.send(data) 116 | sender.close() 117 | 118 | uri = 'http://localhost:%s' % serve.port 119 | conn = h.http.connect(uri) 120 | 121 | sender, recver = h.pipe() 122 | 123 | @h.spawn 124 | def _(): 125 | for i in xrange(3): 126 | sender.send(str(i)) 127 | h.sleep(10) 128 | sender.close() 129 | 130 | response = conn.post('/', data=recver).recv() 131 | 132 | for data in response.body: 133 | print data 134 | 135 | # h.stop() 136 | 137 | def test_post_form_encoded(self): 138 | h = vanilla.Hub() 139 | 140 | serve = h.http.listen() 141 | 142 | uri = 'http://localhost:%s' % serve.port 143 | client = h.http.connect(uri) 144 | response = client.post('/', data={'k1': 'v1', 'k2': 'v2'}) 145 | 146 | conn = serve.recv() 147 | request = conn.recv() 148 | assert request.form == {'k1': 'v1', 'k2': 'v2'} 149 | assert request.form_multi == {'k1': ['v1'], 'k2': ['v2']} 150 | request.reply(vanilla.http.Status(200), {}, '') 151 | 152 | response = response.recv() 153 | assert response.status.code == 200 154 | 155 | h.stop() 156 | 157 | def test_put(self): 158 | h = vanilla.Hub() 159 | 160 | serve = h.http.listen() 161 | 162 | @h.spawn 163 | def _(): 164 | conn = serve.recv() 165 | for request in conn: 166 | request.reply( 167 | vanilla.http.Status(200), 168 | {}, 169 | request.method+request.consume()) 170 | 171 | uri = 'http://localhost:%s' % serve.port 172 | conn = h.http.connect(uri) 173 | 174 | response = conn.put('/').recv() 175 | assert response.status.code == 200 176 | assert response.consume() == 'PUT' 177 | 178 | response = conn.put('/', data='toby').recv() 179 | assert response.status.code == 200 180 | assert response.consume() == 'PUTtoby' 181 | h.stop() 182 | 183 | def test_delete(self): 184 | h = vanilla.Hub() 185 | 186 | serve = h.http.listen() 187 | 188 | @h.spawn 189 | def _(): 190 | conn = serve.recv() 191 | for request in conn: 192 | request.reply(vanilla.http.Status(200), {}, request.method) 193 | 194 | uri = 'http://localhost:%s' % serve.port 195 | conn = h.http.connect(uri) 196 | 197 | response = conn.delete('/').recv() 198 | assert response.status.code == 200 199 | assert response.consume() == 'DELETE' 200 | h.stop() 201 | 202 | def test_404(self): 203 | h = vanilla.Hub() 204 | 205 | serve = h.http.listen() 206 | 207 | @h.spawn 208 | def _(): 209 | conn = serve.recv() 210 | for request in conn: 211 | request.reply(vanilla.http.Status(404), {}, '') 212 | 213 | uri = 'http://localhost:%s' % serve.port 214 | response = h.http.connect(uri).get('/').recv() 215 | assert response.status.code == 404 216 | h.stop() 217 | 218 | def test_overlap(self): 219 | h = vanilla.Hub() 220 | 221 | serve = h.http.listen() 222 | 223 | @h.spawn 224 | def _(): 225 | conn = serve.recv() 226 | for request in conn: 227 | t = request.path[1:] 228 | h.sleep(int(t)) 229 | request.reply(vanilla.http.Status(200), {}, t) 230 | 231 | uri = 'http://localhost:%s' % serve.port 232 | conn = h.http.connect(uri) 233 | 234 | q = h.queue(10) 235 | 236 | def go(t): 237 | r = conn.get('/'+str(t)).recv() 238 | q.send(int(r.consume())) 239 | 240 | h.spawn(go, 50) 241 | h.spawn(go, 20) 242 | 243 | assert q.recv() == 50 244 | assert q.recv() == 20 245 | h.stop() 246 | 247 | def test_basic_auth(self): 248 | h = vanilla.Hub() 249 | 250 | serve = h.http.listen() 251 | 252 | @h.spawn 253 | def _(): 254 | conn = serve.recv() 255 | for request in conn: 256 | request.reply( 257 | vanilla.http.Status(200), 258 | {}, 259 | request.headers['Authorization']) 260 | 261 | uri = 'http://localhost:%s' % serve.port 262 | conn = h.http.connect(uri) 263 | 264 | response = conn.get('/', auth=('foo', 'bar')) 265 | response = response.recv() 266 | assert response.consume() == 'Basic Zm9vOmJhcg==' 267 | 268 | def test_connection_lost(self): 269 | h = vanilla.Hub() 270 | 271 | serve = h.http.listen() 272 | 273 | @h.spawn 274 | def _(): 275 | conn = serve.recv() 276 | for request in conn: 277 | conn.socket.close() 278 | 279 | uri = 'http://localhost:%s' % serve.port 280 | conn = h.http.connect(uri) 281 | 282 | response = conn.get('/') 283 | pytest.raises(vanilla.ConnectionLost, response.recv) 284 | h.stop() 285 | 286 | def test_json(self): 287 | h = vanilla.Hub() 288 | serve = h.http.listen() 289 | uri = 'http://localhost:%s' % serve.port 290 | 291 | response = h.http.get(uri) 292 | 293 | conn = serve.recv() 294 | request = conn.recv() 295 | request.reply(vanilla.http.Status(200), {}, json.dumps({'foo': 'bar'})) 296 | 297 | response = response.recv() 298 | assert response.json() == {'foo': 'bar'} 299 | 300 | 301 | class TestWebsocket(object): 302 | def test_websocket(self): 303 | h = vanilla.Hub() 304 | 305 | serve = h.http.listen() 306 | 307 | @h.spawn 308 | def _(): 309 | conn = serve.recv() 310 | request = conn.recv() 311 | ws = request.upgrade() 312 | for item in ws.recver: 313 | ws.send(item) 314 | 315 | uri = 'ws://localhost:%s' % serve.port 316 | ws = h.http.connect(uri).websocket('/') 317 | 318 | gc.collect() 319 | 320 | message = 'x' * 125 321 | ws.send(message) 322 | assert ws.recv() == message 323 | 324 | message = 'x' * 126 325 | ws.send(message) 326 | assert ws.recv() == message 327 | 328 | message = 'x' * 65535 329 | ws.send(message) 330 | assert ws.recv() == message 331 | 332 | message = 'x' * 65536 333 | ws.send(message) 334 | assert ws.recv() == message 335 | 336 | # test we can call select on the websocket 337 | message = 'x' * 125 338 | ws.send(message) 339 | assert h.select([ws.recver]) == (ws.recver, message) 340 | h.stop() 341 | 342 | def test_websocket_end(self): 343 | h = vanilla.Hub() 344 | 345 | serve = h.http.listen() 346 | 347 | @h.spawn 348 | def _(): 349 | conn = serve.recv() 350 | request = conn.recv() 351 | ws = request.upgrade() 352 | ws.recv() 353 | ws.close() 354 | 355 | uri = 'ws://localhost:%s' % serve.port 356 | ws = h.http.connect(uri).websocket('/') 357 | ws.send('1') 358 | pytest.raises(vanilla.Closed, ws.recv) 359 | h.stop() 360 | -------------------------------------------------------------------------------- /tests/unit/test_io.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | import vanilla 4 | 5 | 6 | # TODO: remove 7 | import logging 8 | logging.basicConfig() 9 | 10 | 11 | class TestIO(object): 12 | def test_pipe(self): 13 | h = vanilla.Hub() 14 | sender, recver = h.io.pipe() 15 | pytest.raises(vanilla.Timeout, recver.recv, timeout=0) 16 | sender.send('123') 17 | assert recver.recv() == '123' 18 | h.stop() 19 | 20 | def test_pipe_pipe(self): 21 | """check we can pipe io.pipes""" 22 | h = vanilla.Hub() 23 | p1 = h.io.pipe() 24 | p2 = h.pipe() 25 | p1.pipe(p2) 26 | p1.send('123') 27 | assert p2.recv() == '123' 28 | 29 | def test_write_eagain(self): 30 | h = vanilla.Hub() 31 | sender, recver = h.io.pipe() 32 | 33 | want = 'x' * 1024 * 1024 34 | 35 | @h.spawn 36 | def _(): 37 | sender.send(want) 38 | 39 | got = '' 40 | while len(got) < len(want): 41 | got += recver.recv() 42 | assert want == got 43 | 44 | def test_write_serialize(self): 45 | h = vanilla.Hub() 46 | sender, recver = h.io.pipe() 47 | 48 | want1 = 'a' * 1024 * 1024 49 | want2 = 'b' * 1024 * 1024 50 | 51 | @h.spawn 52 | def _(): 53 | sender.send(want1) 54 | 55 | @h.spawn 56 | def _(): 57 | sender.send(want2) 58 | 59 | got = '' 60 | while len(got) < (len(want1) + len(want2)): 61 | got += recver.recv() 62 | assert got == want1+want2 63 | 64 | def test_write_close(self): 65 | h = vanilla.Hub() 66 | sender, recver = h.io.pipe() 67 | 68 | sender.send('123') 69 | h.sleep(10) 70 | sender.send('456') 71 | 72 | sender.close() 73 | pytest.raises(vanilla.Closed, sender.send, '789') 74 | 75 | assert recver.recv() == '123' 76 | assert recver.recv() == '456' 77 | pytest.raises(vanilla.Halt, recver.recv) 78 | h.sleep(1) 79 | assert not h.registered 80 | 81 | def test_read_close(self): 82 | h = vanilla.Hub() 83 | sender, recver = h.io.pipe() 84 | recver.close() 85 | pytest.raises(vanilla.Closed, sender.send, '123') 86 | pytest.raises(vanilla.Closed, recver.recv) 87 | assert not h.registered 88 | 89 | def test_api(self): 90 | h = vanilla.Hub() 91 | p1 = h.io.pipe() 92 | p2 = h.io.pipe() 93 | 94 | p1.pipe(p2) 95 | p3 = p2.map(lambda x: int(x)*2) 96 | 97 | p1.send('3') 98 | assert p3.recv() == 6 99 | 100 | def test_stream(self): 101 | h = vanilla.Hub() 102 | p = h.io.pipe() 103 | h.spawn(p.send, '12foo\n') 104 | assert p.recv_n(2) == '12' 105 | assert p.recv_line(2) == 'foo' 106 | 107 | def test_read_final(self): 108 | h = vanilla.Hub() 109 | sender, recver = h.io.pipe() 110 | sender.send('123') 111 | sender.close() 112 | pytest.raises(vanilla.Closed, sender.send, '123') 113 | assert recver.recv() == '123' 114 | pytest.raises(vanilla.Closed, recver.recv) 115 | h.sleep(1) 116 | assert not h.registered 117 | -------------------------------------------------------------------------------- /tests/unit/test_message.py: -------------------------------------------------------------------------------- 1 | import gc 2 | 3 | import pytest 4 | 5 | import vanilla 6 | import vanilla.message 7 | 8 | 9 | # TODO: remove 10 | import logging 11 | logging.basicConfig() 12 | 13 | 14 | @pytest.mark.parametrize('primitive, a', [ 15 | ('pipe', ()), 16 | ('dealer', ()), 17 | ('router', ()), 18 | ('channel', ()), ]) 19 | class TestAbandoned(object): 20 | @staticmethod 21 | def assert_abandoned_sender(sender, check): 22 | pytest.raises(vanilla.Abandoned, sender.send, 10) 23 | check.send('done') 24 | 25 | def test_abandoned_sender_after_wait(self, primitive, a): 26 | h = vanilla.Hub() 27 | sender, recver = getattr(h, primitive)(*a) 28 | check = h.pipe() 29 | h.spawn(self.assert_abandoned_sender, sender, check) 30 | h.sleep(1) 31 | del recver 32 | gc.collect() 33 | assert check.recv() == 'done' 34 | 35 | def test_abandoned_sender_before_wait(self, primitive, a): 36 | h = vanilla.Hub() 37 | sender, recver = getattr(h, primitive)(*a) 38 | check = h.pipe() 39 | h.spawn(self.assert_abandoned_sender, sender, check) 40 | del recver 41 | gc.collect() 42 | assert check.recv() == 'done' 43 | 44 | @staticmethod 45 | def assert_abandoned_recver(recver, check): 46 | pytest.raises(vanilla.Abandoned, recver.recv) 47 | check.send('done') 48 | 49 | def test_abandoned_recver_after_wait(self, primitive, a): 50 | h = vanilla.Hub() 51 | sender, recver = getattr(h, primitive)(*a) 52 | check = h.pipe() 53 | h.spawn(self.assert_abandoned_recver, recver, check) 54 | h.sleep(1) 55 | del sender 56 | gc.collect() 57 | assert check.recv() == 'done' 58 | 59 | def test_abandoned_recver_before_wait(self, primitive, a): 60 | h = vanilla.Hub() 61 | sender, recver = getattr(h, primitive)(*a) 62 | check = h.pipe() 63 | h.spawn(self.assert_abandoned_recver, recver, check) 64 | del sender 65 | gc.collect() 66 | assert check.recv() == 'done' 67 | 68 | 69 | class TestPipe(object): 70 | def test_deadlock(self): 71 | h = vanilla.Hub() 72 | p = h.pipe() 73 | pytest.raises(vanilla.Stop, p.send, 1) 74 | 75 | def test_close_recver(self): 76 | h = vanilla.Hub() 77 | 78 | check = h.pipe() 79 | sender, recver = h.pipe() 80 | 81 | @h.spawn 82 | def _(): 83 | for i in xrange(10): 84 | try: 85 | sender.send(i) 86 | except vanilla.Closed: 87 | break 88 | check.send('done') 89 | 90 | assert recver.recv() == 0 91 | assert recver.recv() == 1 92 | recver.close() 93 | assert check.recv() == 'done' 94 | 95 | def test_close_sender(self): 96 | h = vanilla.Hub() 97 | 98 | check = h.pipe() 99 | sender, recver = h.pipe() 100 | 101 | @h.spawn 102 | def _(): 103 | for i in recver: 104 | check.send(i) 105 | check.send('done') 106 | 107 | sender.send(0) 108 | assert check.recv() == 0 109 | sender.send(1) 110 | assert check.recv() == 1 111 | sender.close() 112 | assert check.recv() == 'done' 113 | 114 | def test_onclose(self): 115 | h = vanilla.Hub() 116 | p1 = h.pipe() 117 | p2 = h.queue(1) 118 | p1.onclose(p2.send, 'Toby') 119 | p1.close() 120 | assert p2.recv() == 'Toby' 121 | 122 | def test_onclose_piped_to_router(self): 123 | h = vanilla.Hub() 124 | sender, recver = h.pipe() 125 | 126 | p2 = h.queue(1) 127 | recver.onclose(p2.send, 'Toby') 128 | 129 | r = h.router() 130 | recver.pipe(r) 131 | r.close() 132 | 133 | assert p2.recv() == 'Toby' 134 | 135 | def test_timeout(self): 136 | h = vanilla.Hub() 137 | 138 | sender, recver = h.pipe() 139 | check = h.pipe() 140 | 141 | pytest.raises(vanilla.Timeout, sender.send, 12, timeout=0) 142 | pytest.raises(vanilla.Timeout, recver.recv, timeout=0) 143 | pytest.raises(vanilla.Timeout, sender.send, 12, timeout=0) 144 | 145 | @h.spawn 146 | def _(): 147 | h.sleep(20) 148 | check.send(recver.recv()) 149 | 150 | pytest.raises(vanilla.Timeout, sender.send, 12, timeout=10) 151 | sender.send(12, timeout=20) 152 | assert check.recv() == 12 153 | 154 | @h.spawn 155 | def _(): 156 | h.sleep(20) 157 | sender.send(12) 158 | 159 | pytest.raises(vanilla.Timeout, recver.recv, timeout=10) 160 | assert recver.recv(timeout=20) == 12 161 | 162 | def test_throw_with_timeout(self): 163 | h = vanilla.Hub() 164 | sender, recver = h.pipe() 165 | h.spawn(sender.send, Exception()) 166 | pytest.raises(Exception, recver.recv, timeout=20) 167 | assert h.scheduled.count == 0 168 | 169 | def test_select(self): 170 | h = vanilla.Hub() 171 | 172 | s1, r1 = h.pipe() 173 | s2, r2 = h.pipe() 174 | check = h.pipe() 175 | 176 | @h.spawn 177 | def _(): 178 | check.send(r1.recv()) 179 | 180 | @h.spawn 181 | def _(): 182 | s2.send(10) 183 | check.send('done') 184 | 185 | ch, item = h.select([s1, r2]) 186 | assert ch == s1 187 | s1.send(20) 188 | 189 | ch, item = h.select([s1, r2]) 190 | assert ch == r2 191 | assert item == 10 192 | 193 | assert check.recv() == 20 194 | assert check.recv() == 'done' 195 | 196 | def test_select_timeout(self): 197 | h = vanilla.Hub() 198 | 199 | s1, r1 = h.pipe() 200 | s2, r2 = h.pipe() 201 | check = h.pipe() 202 | 203 | pytest.raises(vanilla.Timeout, h.select, [s1, r2], timeout=0) 204 | 205 | @h.spawn 206 | def _(): 207 | h.sleep(20) 208 | check.send(r1.recv()) 209 | 210 | pytest.raises(vanilla.Timeout, h.select, [s1, r2], timeout=10) 211 | 212 | ch, item = h.select([s1, r2], timeout=20) 213 | assert ch == s1 214 | s1.send(20) 215 | assert check.recv() == 20 216 | 217 | @h.spawn 218 | def _(): 219 | h.sleep(20) 220 | s2.send(10) 221 | check.send('done') 222 | 223 | pytest.raises(vanilla.Timeout, h.select, [s1, r2], timeout=10) 224 | 225 | ch, item = h.select([s1, r2], timeout=20) 226 | assert ch == r2 227 | assert item == 10 228 | assert check.recv() == 'done' 229 | 230 | def test_pipe(self): 231 | h = vanilla.Hub() 232 | 233 | p1 = h.pipe() 234 | p2 = h.pipe() 235 | 236 | p1.pipe(p2) 237 | 238 | h.spawn(p1.send, 1) 239 | assert p2.recv() == 1 240 | 241 | def test_pipe_to_function(self): 242 | h = vanilla.Hub() 243 | 244 | p1 = h.pipe() 245 | 246 | @p1.pipe 247 | def p2(recver, sender): 248 | for item in recver: 249 | sender.send(item*2) 250 | 251 | h.spawn(p1.send, 1) 252 | assert p2.recv() == 2 253 | h.spawn(p1.send, 2) 254 | assert p2.recv() == 4 255 | 256 | def test_map(self): 257 | h = vanilla.Hub() 258 | p1 = h.pipe() 259 | p2 = p1.map(lambda x: x * 2) 260 | 261 | h.spawn(p1.send, 1) 262 | assert p2.recv() == 2 263 | 264 | h.spawn(p1.send, 2) 265 | assert p2.recv() == 4 266 | 267 | def test_map_raises(self): 268 | h = vanilla.Hub() 269 | 270 | class E(Exception): 271 | pass 272 | 273 | def f(x): 274 | raise E() 275 | 276 | p1 = h.pipe() 277 | p2 = p1.map(f) 278 | 279 | h.spawn(p1.send, 1) 280 | pytest.raises(E, p2.recv) 281 | 282 | def test_chain(self): 283 | h = vanilla.Hub() 284 | 285 | p = h.pipe() 286 | # s1, m1, r1 287 | # p is: s2, r1 288 | 289 | p = h.pipe().map(lambda x: x + 1).pipe(p) 290 | # s2, m3, f(r3, s3), m2, r2 --> s1, m1, r1 291 | # s2, m3, f(r3, s3), m2, r1 292 | # p is: s2, r1 293 | 294 | p = h.pipe().map(lambda x: x * 2).pipe(p) 295 | # s4, m5, f(r5, s5), m4, r4 --> s2, m3, f(r3, s3), m2, r1 296 | # s4, m5, f(r5, s5), m4, f(r3, s3), m2, r1 297 | # p is: s4, r1 298 | 299 | h.spawn(p.send, 2) 300 | h.sleep(1) 301 | 302 | assert p.recv() == 5 303 | 304 | def test_pipefu(self): 305 | h = vanilla.Hub() 306 | p = h.pipe() \ 307 | .map(lambda x: x*2) \ 308 | .pipe(h.pipe()) \ 309 | .map(lambda x: x+'.') 310 | h.spawn(p.send, 'foo') 311 | assert p.recv() == 'foofoo.' 312 | 313 | def test_consume(self): 314 | h = vanilla.Hub() 315 | p = h.pipe() 316 | check = h.pipe() 317 | p.consume(lambda x: check.send(x)) 318 | p.send(1) 319 | assert check.recv() == 1 320 | 321 | def test_exception(self): 322 | h = vanilla.Hub() 323 | 324 | p = h.pipe() 325 | check = h.pipe() 326 | 327 | @h.spawn 328 | def _(): 329 | try: 330 | p.recv() 331 | except Exception, e: 332 | check.send(e.message) 333 | 334 | p.send(Exception('hai')) 335 | assert check.recv() == 'hai' 336 | 337 | # TODO: move to their own test suite 338 | def test_producer(self): 339 | h = vanilla.Hub() 340 | 341 | @h.producer 342 | def counter(sender): 343 | for i in xrange(10): 344 | sender.send(i) 345 | 346 | assert counter.recv() == 0 347 | h.sleep(10) 348 | assert counter.recv() == 1 349 | 350 | def test_trigger(self): 351 | h = vanilla.Hub() 352 | 353 | check = h.pipe() 354 | 355 | @h.trigger 356 | def go(): 357 | check.send(1) 358 | 359 | h.sleep(1) 360 | gc.collect() 361 | 362 | go.trigger() 363 | assert check.recv() == 1 364 | 365 | pipe = go.middle 366 | 367 | h.sleep(1) 368 | del go 369 | gc.collect() 370 | h.sleep(1) 371 | gc.collect() 372 | h.sleep(1) 373 | 374 | assert pipe.recver() is None 375 | assert pipe.sender() is None 376 | 377 | 378 | class TestQueue(object): 379 | def test_queue(self): 380 | h = vanilla.Hub() 381 | b = h.queue(2) 382 | 383 | b.send(1, timeout=0) 384 | b.send(2, timeout=0) 385 | pytest.raises(vanilla.Timeout, b.send, 3, timeout=0) 386 | assert b.recv() == 1 387 | 388 | b.send(3, timeout=0) 389 | assert b.recv() == 2 390 | assert b.recv() == 3 391 | 392 | gc.collect() 393 | h.sleep(1) 394 | 395 | def test_queue_close_sender(self): 396 | h = vanilla.Hub() 397 | sender, recver = h.queue(2) 398 | 399 | sender.send(1, timeout=0) 400 | sender.send(2, timeout=0) 401 | 402 | del sender 403 | gc.collect() 404 | 405 | assert recver.recv() == 1 406 | assert recver.recv() == 2 407 | 408 | gc.collect() 409 | h.sleep(1) 410 | 411 | 412 | class TestPulse(object): 413 | def test_pulse(self): 414 | h = vanilla.Hub() 415 | 416 | trigger = h.pulse(20) 417 | pytest.raises(vanilla.Timeout, trigger.recv, timeout=0) 418 | 419 | h.sleep(20) 420 | assert trigger.recv(timeout=0) 421 | pytest.raises(vanilla.Timeout, trigger.recv, timeout=0) 422 | 423 | h.sleep(20) 424 | assert trigger.recv(timeout=0) 425 | pytest.raises(vanilla.Timeout, trigger.recv, timeout=0) 426 | 427 | h.stop() 428 | 429 | 430 | class TestDealer(object): 431 | def test_send_then_recv(self): 432 | h = vanilla.Hub() 433 | d = h.dealer() 434 | h.spawn(d.send, 1) 435 | assert d.recv() == 1 436 | 437 | def test_recv_then_send(self): 438 | h = vanilla.Hub() 439 | 440 | d = h.dealer() 441 | q = h.queue(10) 442 | 443 | h.spawn(lambda: q.send(d.recv())) 444 | h.spawn(lambda: q.send(d.recv())) 445 | h.spawn(lambda: q.send(d.recv())) 446 | h.sleep(1) 447 | 448 | d.send(1) 449 | assert q.recv() == 1 450 | d.send(2) 451 | d.send(3) 452 | assert q.recv() == 2 453 | assert q.recv() == 3 454 | 455 | def test_send_timeout(self): 456 | h = vanilla.Hub() 457 | d = h.dealer() 458 | pytest.raises(vanilla.Timeout, d.send, 1, timeout=10) 459 | 460 | def test_recv_timeout(self): 461 | h = vanilla.Hub() 462 | d = h.dealer() 463 | pytest.raises(vanilla.Timeout, d.recv, timeout=10) 464 | # assert that waiters is cleaned up after timeout 465 | assert not d.recver.current 466 | 467 | def test_send_select(self): 468 | h = vanilla.Hub() 469 | d = h.dealer() 470 | 471 | @h.spawn 472 | def _(): 473 | ch, _ = h.select([d.sender]) 474 | assert ch == d.sender 475 | d.send(1) 476 | 477 | h.sleep(1) 478 | assert d.recv() == 1 479 | 480 | def test_recv_select(self): 481 | h = vanilla.Hub() 482 | 483 | d = h.dealer() 484 | q = h.queue(10) 485 | 486 | def selector(): 487 | ch, item = h.select([d.recver]) 488 | assert ch == d.recver 489 | q.send(item) 490 | 491 | h.spawn(selector) 492 | h.spawn(selector) 493 | h.spawn(selector) 494 | h.sleep(1) 495 | 496 | d.send(1) 497 | assert q.recv() == 1 498 | d.send(2) 499 | d.send(3) 500 | assert q.recv() == 2 501 | assert q.recv() == 3 502 | 503 | 504 | class TestRouter(object): 505 | def test_send_then_recv(self): 506 | h = vanilla.Hub() 507 | r = h.router() 508 | 509 | h.spawn(r.send, 3) 510 | h.spawn(r.send, 2) 511 | h.spawn(r.send, 1) 512 | h.sleep(1) 513 | 514 | assert r.recv() == 3 515 | assert r.recv() == 2 516 | assert r.recv() == 1 517 | 518 | def test_recv_then_send(self): 519 | h = vanilla.Hub() 520 | 521 | r = h.router() 522 | q = h.queue(10) 523 | 524 | @h.spawn 525 | def _(): 526 | q.send(r.recv()) 527 | 528 | h.sleep(1) 529 | r.send(1) 530 | assert q.recv() == 1 531 | 532 | def test_pipe(self): 533 | h = vanilla.Hub() 534 | 535 | r = h.router() 536 | 537 | p1 = h.pipe() 538 | p2 = h.pipe() 539 | 540 | p1.pipe(r) 541 | p2.pipe(r) 542 | 543 | h.spawn(p1.send, 1) 544 | h.spawn(p2.send, 2) 545 | h.spawn(p1.send, 1) 546 | h.spawn(p2.send, 2) 547 | 548 | assert r.recv() == 1 549 | assert r.recv() == 2 550 | assert r.recv() == 1 551 | assert r.recv() == 2 552 | 553 | def test_state(self): 554 | h = vanilla.Hub() 555 | 556 | r = h.router() 557 | s1 = h.state() 558 | s2 = h.state() 559 | s1.pipe(r) 560 | s2.pipe(r) 561 | 562 | s1.send('s1') 563 | s2.send('s2') 564 | assert r.recv() == 's1' 565 | assert r.recv() == 's2' 566 | assert r.recv() == 's1' 567 | 568 | s1.close() 569 | assert r.recv() == 's2' 570 | assert r.recv() == 's2' 571 | assert r.recv() == 's2' 572 | 573 | 574 | class TestChannel(object): 575 | def test_send_then_recv(self): 576 | h = vanilla.Hub() 577 | ch = h.channel() 578 | 579 | h.spawn(ch.send, 3) 580 | h.spawn(ch.send, 2) 581 | h.spawn(ch.send, 1) 582 | h.sleep(1) 583 | 584 | assert ch.recv() == 3 585 | assert ch.recv() == 2 586 | assert ch.recv() == 1 587 | 588 | def test_recv_then_send(self): 589 | h = vanilla.Hub() 590 | 591 | d = h.dealer() 592 | q = h.queue(10) 593 | 594 | h.spawn(lambda: q.send(d.recv())) 595 | h.spawn(lambda: q.send(d.recv())) 596 | h.spawn(lambda: q.send(d.recv())) 597 | h.sleep(1) 598 | 599 | d.send(1) 600 | assert q.recv() == 1 601 | d.send(2) 602 | d.send(3) 603 | assert q.recv() == 2 604 | assert q.recv() == 3 605 | 606 | def test_no_queue(self): 607 | h = vanilla.Hub() 608 | ch = h.channel() 609 | pytest.raises(vanilla.Timeout, ch.send, 1, timeout=0) 610 | 611 | def test_queue(self): 612 | h = vanilla.Hub() 613 | ch = h.channel(2) 614 | ch.send(1, timeout=0) 615 | ch.send(2, timeout=0) 616 | pytest.raises(vanilla.Timeout, ch.send, 3, timeout=0) 617 | assert ch.recv() == 1 618 | ch.send(3, timeout=0) 619 | assert ch.recv() == 2 620 | assert ch.recv() == 3 621 | 622 | 623 | class TestBroadcast(object): 624 | def test_broadcast(self): 625 | h = vanilla.Hub() 626 | 627 | b = h.broadcast() 628 | check = h.queue(10) 629 | 630 | b.onempty(check.send, 'empty') 631 | 632 | def subscriber(s, name): 633 | for item in s: 634 | check.send((name, item)) 635 | 636 | s1 = b.subscribe() 637 | s2 = b.subscribe() 638 | h.spawn(subscriber, s1, 's1') 639 | h.spawn(subscriber, s2, 's2') 640 | h.sleep(1) 641 | 642 | b.send(1) 643 | assert check.recv() == ('s1', 1) 644 | assert check.recv() == ('s2', 1) 645 | 646 | b.send(2) 647 | assert check.recv() == ('s1', 2) 648 | assert check.recv() == ('s2', 2) 649 | 650 | s1.close() 651 | b.send(3) 652 | assert check.recv() == ('s2', 3) 653 | pytest.raises(vanilla.Timeout, check.recv, timeout=0) 654 | 655 | s2.close() 656 | assert check.recv() == 'empty' 657 | 658 | def test_pipe(self): 659 | h = vanilla.Hub() 660 | 661 | b = h.broadcast() 662 | source = h.pulse(20) 663 | source.pipe(b) 664 | check = h.queue(10) 665 | 666 | def subscriber(s, name): 667 | for item in s: 668 | check.send((name, item)) 669 | 670 | s1 = b.subscribe() 671 | s2 = b.subscribe() 672 | 673 | h.spawn(subscriber, s1, 's1') 674 | h.spawn(subscriber, s2, 's2') 675 | h.sleep(1) 676 | 677 | assert check.recv() == ('s1', True) 678 | assert check.recv() == ('s2', True) 679 | 680 | 681 | class TestState(object): 682 | def test_state(self): 683 | h = vanilla.Hub() 684 | 685 | s = h.state() 686 | pytest.raises(vanilla.Timeout, s.recv, 10) 687 | 688 | h.spawn_later(10, s.send, 'Toby') 689 | assert s.recv() == 'Toby' 690 | assert s.recv() == 'Toby' 691 | 692 | s.clear() 693 | pytest.raises(vanilla.Timeout, s.recv, 10) 694 | s.send('Toby') 695 | assert s.recv() == 'Toby' 696 | 697 | s.close() 698 | pytest.raises(vanilla.Closed, s.send, 'update') 699 | pytest.raises(vanilla.Closed, s.recv, 10) 700 | 701 | def test_pipe(self): 702 | h = vanilla.Hub() 703 | p = h.pipe() \ 704 | .map(lambda x: x*2) \ 705 | .pipe(h.state('Toby')) \ 706 | .map(lambda x: x+'.') 707 | 708 | assert p.recv() == 'Toby.' 709 | 710 | p.send('foo') 711 | assert p.recv() == 'foofoo.' 712 | assert p.recv() == 'foofoo.' 713 | # TODO: should clear be able to be passed through map? 714 | 715 | 716 | class TestSerialize(object): 717 | def test_serialize(self): 718 | h = vanilla.Hub() 719 | 720 | out = h.pipe() 721 | 722 | @h.serialize 723 | def go(i): 724 | h.sleep(40-(10*i)) 725 | out.send(i) 726 | 727 | for i in xrange(3): 728 | h.spawn(go, i) 729 | 730 | assert list(out.recver) == [0, 1, 2] 731 | 732 | def test_exception(self): 733 | h = vanilla.Hub() 734 | 735 | @h.serialize 736 | def go(): 737 | raise AssertionError('foo') 738 | 739 | pytest.raises(AssertionError, go) 740 | 741 | 742 | class TestStream(object): 743 | def test_stream(self): 744 | h = vanilla.Hub() 745 | 746 | sender, recver = h.pipe() 747 | recver = vanilla.message.Stream(recver) 748 | 749 | @h.spawn 750 | def _(): 751 | sender.send('foo') 752 | sender.send('123') 753 | sender.send('456') 754 | sender.send('TobyTobyToby') 755 | sender.send('foo\n') 756 | sender.send('bar\nend.') 757 | sender.close() 758 | 759 | assert recver.recv() == 'foo' 760 | assert recver.recv_n(2) == '12' 761 | assert recver.recv_n(2) == '34' 762 | assert recver.recv_partition('y') == '56Tob' 763 | assert recver.recv_partition('y') == 'Tob' 764 | assert recver.recv_line() == 'Tobyfoo' 765 | assert recver.recv_line() == 'bar' 766 | assert recver.recv() == 'end.' 767 | pytest.raises(vanilla.Closed, recver.recv_n, 2) 768 | -------------------------------------------------------------------------------- /tests/unit/test_poll.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import vanilla.poll 4 | 5 | 6 | class TestPoll(object): 7 | def test_poll(self): 8 | poll = vanilla.poll.Poll() 9 | 10 | r, w = os.pipe() 11 | 12 | poll.register(r, vanilla.poll.POLLIN) 13 | assert poll.poll(timeout=0) == [] 14 | 15 | os.write(w, '1') 16 | assert poll.poll() == [(r, vanilla.poll.POLLIN)] 17 | # test event is cleared 18 | assert poll.poll(timeout=0) == [] 19 | 20 | # test event is reset on new write after read 21 | assert os.read(r, 4096) == '1' 22 | assert poll.poll(timeout=0) == [] 23 | os.write(w, '2') 24 | assert poll.poll() == [(r, vanilla.poll.POLLIN)] 25 | assert poll.poll(timeout=0) == [] 26 | 27 | # test event is reset on new write without read 28 | os.write(w, '3') 29 | assert poll.poll() == [(r, vanilla.poll.POLLIN)] 30 | assert poll.poll(timeout=0) == [] 31 | 32 | assert os.read(r, 4096) == '23' 33 | 34 | def test_write_close(self): 35 | poll = vanilla.poll.Poll() 36 | r, w = os.pipe() 37 | 38 | poll.register(r, vanilla.poll.POLLIN) 39 | poll.register(w, vanilla.poll.POLLOUT) 40 | assert poll.poll() == [(w, vanilla.poll.POLLOUT)] 41 | assert poll.poll(timeout=0) == [] 42 | 43 | os.close(w) 44 | assert poll.poll() == [(r, vanilla.poll.POLLERR)] 45 | assert poll.poll(timeout=0) == [] 46 | 47 | def test_read_close(self): 48 | poll = vanilla.poll.Poll() 49 | r, w = os.pipe() 50 | 51 | poll.register(r, vanilla.poll.POLLIN) 52 | poll.register(w, vanilla.poll.POLLOUT) 53 | assert poll.poll() == [(w, vanilla.poll.POLLOUT)] 54 | assert poll.poll(timeout=0) == [] 55 | 56 | os.close(r) 57 | got = poll.poll() 58 | assert got == [(w, vanilla.poll.POLLOUT), (w, vanilla.poll.POLLERR)] 59 | assert poll.poll(timeout=0) == [] 60 | -------------------------------------------------------------------------------- /tests/unit/test_process.py: -------------------------------------------------------------------------------- 1 | import vanilla 2 | import signal 3 | import os 4 | 5 | import pytest 6 | 7 | 8 | class TestProcess(object): 9 | def test_basic(self): 10 | h = vanilla.Hub() 11 | 12 | child = h.process.execv( 13 | ['/usr/bin/env', 'grep', '--line-buffered', 'foo']) 14 | assert child.check_liveness() 15 | pytest.raises(vanilla.Timeout, child.done.recv, timeout=0) 16 | 17 | child.stdin.send('foo1\n') 18 | assert child.stdout.recv() == 'foo1\n' 19 | child.stdin.send('bar1\n') 20 | child.stdin.send('foo2\n') 21 | assert child.stdout.recv() == 'foo2\n' 22 | 23 | child.terminate() 24 | child.done.recv() 25 | assert not child.check_liveness() 26 | 27 | # spawn a new process to ensure the reaper watch is rebooted 28 | h.sleep(1) 29 | child = h.process.execv( 30 | ['/usr/bin/env', 'grep', '--line-buffered', 'foo']) 31 | assert child.check_liveness() 32 | pytest.raises(vanilla.Timeout, child.done.recv, timeout=0) 33 | 34 | child.stdin.send('foo1\n') 35 | assert child.stdout.recv() == 'foo1\n' 36 | child.stdin.send('bar1\n') 37 | child.stdin.send('foo2\n') 38 | assert child.stdout.recv() == 'foo2\n' 39 | 40 | child.terminate() 41 | child.done.recv() 42 | assert not child.check_liveness() 43 | 44 | def test_unexecutable(self): 45 | h = vanilla.Hub() 46 | child = h.process.execv(['foobar123']) 47 | child.done.recv() 48 | assert child.exitcode == 127 49 | child = h.process.execv(['/dev/null']) 50 | child.done.recv() 51 | assert child.exitcode == 126 52 | 53 | def test_stderr(self): 54 | h = vanilla.Hub() 55 | child = h.process.execv(['/usr/bin/env', 'grep', '-g']) 56 | assert child.stderr.recv() 57 | 58 | def test_stderrtoout(self): 59 | h = vanilla.Hub() 60 | child = h.process.execv( 61 | ['/usr/bin/env', 'grep', '-g'], stderrtoout=True) 62 | assert child.stdout.recv() 63 | 64 | def test_signal(self): 65 | h = vanilla.Hub() 66 | child = h.process.execv( 67 | ['/usr/bin/env', 'grep', '--line-buffered', 'foo']) 68 | child.signal(signal.SIGTERM) 69 | child.done.recv() 70 | assert not child.check_liveness() 71 | 72 | def test_env(self): 73 | h = vanilla.Hub() 74 | 75 | VAR1 = 'VANILLA_%s_VAR1' % os.getpid() 76 | VAR2 = 'VANILLA_%s_VAR2' % os.getpid() 77 | 78 | os.putenv(VAR1, 'VAR1') 79 | 80 | child = h.process.execv( 81 | ['/usr/bin/env', 'sh', '-c', 'echo $%s $%s' % (VAR1, VAR2)]) 82 | assert child.stdout.recv() == 'VAR1\n' 83 | 84 | child = h.process.execv( 85 | ['/usr/bin/env', 'sh', '-c', 'echo $%s $%s' % (VAR1, VAR2)], 86 | env={VAR2: 'VAR2'}) 87 | assert child.stdout.recv() == 'VAR2\n' 88 | 89 | def test_spawn(self): 90 | h = vanilla.Hub() 91 | 92 | def worker(): 93 | import sys 94 | while True: 95 | line = sys.stdin.readline() 96 | sys.stdout.write('worker: %s' % line) 97 | 98 | child = h.process.spawn(worker) 99 | child.stdin.send('line1\n') 100 | assert child.stdout.recv_partition('\n') == 'worker: line1' 101 | child.stdin.send('line2\n') 102 | assert child.stdout.recv_partition('\n') == 'worker: line2' 103 | -------------------------------------------------------------------------------- /tests/unit/test_signal.py: -------------------------------------------------------------------------------- 1 | import signal 2 | import os 3 | 4 | import pytest 5 | 6 | import vanilla 7 | 8 | 9 | class TestSignal(object): 10 | # TODO: test abandoned 11 | def test_signal(self): 12 | h = vanilla.Hub() 13 | 14 | signal.setitimer(signal.ITIMER_REAL, 50.0/1000) 15 | 16 | s1 = h.signal.subscribe(signal.SIGALRM) 17 | s2 = h.signal.subscribe(signal.SIGALRM) 18 | 19 | assert s1.recv() == signal.SIGALRM 20 | assert s2.recv() == signal.SIGALRM 21 | 22 | signal.setitimer(signal.ITIMER_REAL, 10.0/1000) 23 | s1.close() 24 | 25 | pytest.raises(vanilla.Halt, s1.recv) 26 | assert s2.recv() == signal.SIGALRM 27 | 28 | # assert that removing the last listener for a signal cleans up the 29 | # registered file descriptor 30 | s2.close() 31 | assert not h.registered 32 | 33 | def test_stop_on_term(self): 34 | h = vanilla.Hub() 35 | h.spawn_later(10, os.kill, os.getpid(), signal.SIGINT) 36 | h.stop_on_term() 37 | -------------------------------------------------------------------------------- /tests/unit/test_tcp.py: -------------------------------------------------------------------------------- 1 | import vanilla 2 | 3 | 4 | class TestTCP(object): 5 | def test_tcp(self): 6 | h = vanilla.Hub() 7 | server = h.tcp.listen() 8 | 9 | @h.spawn 10 | def _(): 11 | conn = server.recv() 12 | message = conn.recv() 13 | conn.send('Echo: ' + message) 14 | 15 | client = h.tcp.connect(server.port) 16 | client.send('Toby') 17 | assert client.recv() == 'Echo: Toby' 18 | 19 | h.stop() 20 | assert not h.registered 21 | -------------------------------------------------------------------------------- /tests/unit/test_thread.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | import vanilla 4 | 5 | 6 | # TODO: test shutdown 7 | 8 | 9 | def test_pipe(): 10 | h = vanilla.Hub() 11 | sender, recver = h.thread.pipe() 12 | sender.send(1) 13 | sender.send(2) 14 | assert recver.recv() == 1 15 | assert recver.recv() == 2 16 | 17 | 18 | def test_call(): 19 | def add(a, b): 20 | return a + b 21 | 22 | h = vanilla.Hub() 23 | assert h.thread.call(add, 2, 3).recv() == 5 24 | 25 | 26 | def test_pool(): 27 | h = vanilla.Hub() 28 | 29 | def sleeper(x): 30 | time.sleep(x) 31 | return x 32 | 33 | p = h.thread.pool(2) 34 | check = h.router() 35 | 36 | p.call(sleeper, 0.2).pipe(check) 37 | p.call(sleeper, 0.1).pipe(check) 38 | p.call(sleeper, 0.05).pipe(check) 39 | 40 | assert check.recv() == 0.1 41 | assert check.recv() == 0.05 42 | assert check.recv() == 0.2 43 | 44 | 45 | def test_wrap(): 46 | 47 | class Target(object): 48 | def __init__(self): 49 | class Two(object): 50 | def two(self): 51 | return "two" 52 | self.child = Two() 53 | 54 | def one(self): 55 | return "one" 56 | 57 | target = Target() 58 | assert target.one() == "one" 59 | assert target.child.two() == "two" 60 | 61 | h = vanilla.Hub() 62 | 63 | p = h.thread.pool(2) 64 | w = p.wrap(target) 65 | 66 | assert w.one().recv() == 'one' 67 | assert w.child.two().recv() == 'two' 68 | 69 | 70 | def test_spawn(): 71 | def producer(h, extra): 72 | assert extra == 'foo' 73 | for n in h.parent.recver: 74 | for i in xrange(n): 75 | h.parent.send(i) 76 | 77 | h = vanilla.Hub() 78 | 79 | child = h.thread.spawn(producer, 'foo') 80 | child.send(3) 81 | 82 | assert child.recv() == 0 83 | assert child.recv() == 1 84 | assert child.recv() == 2 85 | -------------------------------------------------------------------------------- /tests/unit/test_udp.py: -------------------------------------------------------------------------------- 1 | import vanilla 2 | 3 | 4 | def test_core(): 5 | h = vanilla.Hub() 6 | 7 | serve = h.udp.listen() 8 | 9 | client = h.udp.create() 10 | client.send(('foo', ('127.0.0.1', serve.port))) 11 | 12 | data, addr = serve.recv() 13 | assert data == 'foo' 14 | serve.send(('Echo: ' + data, addr)) 15 | 16 | data, addr = client.recv() 17 | assert data == 'Echo: foo' 18 | assert addr == ('127.0.0.1', serve.port) 19 | 20 | serve.close() 21 | client.close() 22 | assert h.registered == {} 23 | 24 | 25 | def test_send(): 26 | h = vanilla.Hub() 27 | 28 | N = 5 29 | serve = h.udp.listen() 30 | 31 | @h.spawn 32 | def _(): 33 | client = h.udp.create() 34 | for i in xrange(N): 35 | if not i % 2000: 36 | h.sleep(1) 37 | client.send((str(i), ('127.0.0.1', serve.port))) 38 | 39 | for i in xrange(N): 40 | data, addr = serve.recv() 41 | assert int(data) == i 42 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = flake8, py27, pypy, coverage 3 | 4 | [flake8] 5 | ignore = F811,E226 6 | exclude = .tox/,xx,__*,docs 7 | 8 | [testenv] 9 | deps = pytest 10 | commands = 11 | py.test vanilla tests/unit 12 | 13 | [testenv:flake8] 14 | deps = flake8 15 | commands = flake8 16 | 17 | [testenv:coverage] 18 | passenv = CI TRAVIS_BUILD_ID TRAVIS TRAVIS_BRANCH TRAVIS_JOB_NUMBER TRAVIS_PULL_REQUEST TRAVIS_JOB_ID TRAVIS_REPO_SLUG TRAVIS_COMMIT 19 | deps = 20 | pytest-cov 21 | codecov 22 | commands = 23 | py.test --cov {envsitepackagesdir}/vanilla tests/unit 24 | codecov -e TOXENV 25 | -------------------------------------------------------------------------------- /vanilla/__init__.py: -------------------------------------------------------------------------------- 1 | __import__('pkg_resources').declare_namespace(__name__) 2 | 3 | from vanilla.core import Hub 4 | 5 | from vanilla.exception import ConnectionLost 6 | from vanilla.exception import Abandoned 7 | from vanilla.exception import Timeout 8 | from vanilla.exception import Closed 9 | from vanilla.exception import Stop 10 | from vanilla.exception import Halt 11 | -------------------------------------------------------------------------------- /vanilla/core.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import collections 4 | import functools 5 | import importlib 6 | import logging 7 | import signal 8 | import heapq 9 | import time 10 | 11 | 12 | from greenlet import getcurrent 13 | from greenlet import greenlet 14 | 15 | import vanilla.exception 16 | import vanilla.message 17 | import vanilla.poll 18 | 19 | 20 | log = logging.getLogger(__name__) 21 | 22 | 23 | class lazy(object): 24 | def __init__(self, f): 25 | self.f = f 26 | 27 | def __get__(self, ob, type_=None): 28 | value = self.f(ob) 29 | setattr(ob, self.f.__name__, value) 30 | return value 31 | 32 | 33 | class Scheduler(object): 34 | Item = collections.namedtuple('Item', ['due', 'action', 'args']) 35 | 36 | def __init__(self): 37 | self.count = 0 38 | self.queue = [] 39 | self.removed = {} 40 | 41 | def add(self, delay, action, *args): 42 | due = time.time() + (delay / 1000.0) 43 | item = self.Item(due, action, args) 44 | heapq.heappush(self.queue, item) 45 | self.count += 1 46 | return item 47 | 48 | def __len__(self): 49 | return self.count 50 | 51 | def remove(self, item): 52 | self.removed[item] = True 53 | self.count -= 1 54 | 55 | def prune(self): 56 | while True: 57 | if self.queue[0] not in self.removed: 58 | break 59 | item = heapq.heappop(self.queue) 60 | del self.removed[item] 61 | 62 | def timeout(self): 63 | self.prune() 64 | return self.queue[0].due - time.time() 65 | 66 | def pop(self): 67 | self.prune() 68 | item = heapq.heappop(self.queue) 69 | self.count -= 1 70 | return item.action, item.args 71 | 72 | 73 | class Hub(object): 74 | """ 75 | A Vanilla Hub is a handle to a self contained world of interwoven 76 | coroutines. It includes an event loop which is responsibile for scheduling 77 | which green thread should have context. Unlike most asynchronous libraries 78 | this Hub is explicit and must be passed to coroutines that need to interact 79 | with it. This is particularly nice for testing, as it makes it clear what's 80 | going on, and other tests can't inadvertently effect each other. 81 | """ 82 | def __init__(self): 83 | self.log = logging.getLogger('%s.%s' % (__name__, self.__class__)) 84 | 85 | self.ready = collections.deque() 86 | self.scheduled = Scheduler() 87 | 88 | self.stopped = self.state() 89 | 90 | self.registered = {} 91 | self.poll = vanilla.poll.Poll() 92 | self.loop = greenlet(self.main) 93 | 94 | def __getattr__(self, name): 95 | # facilitates dynamic plugin look up 96 | try: 97 | package = '.'.join(__name__.split('.')[:-1]) 98 | module = importlib.import_module('.'+name, package=package) 99 | plugin = module.__plugin__(self) 100 | setattr(self, name, plugin) 101 | return plugin 102 | except Exception, e: 103 | log.exception(e) 104 | raise AttributeError( 105 | "'Hub' object has no attribute '{name}'\n" 106 | "You may be trying to use a plugin named vanilla.{name}. " 107 | "If you are, you still need to install it".format( 108 | name=name)) 109 | 110 | def pipe(self): 111 | """ 112 | Returns a `Pipe`_ `Pair`_. 113 | """ 114 | return vanilla.message.Pipe(self) 115 | 116 | def producer(self, f): 117 | """ 118 | Convenience to create a `Pipe`_. *f* is a callable that takes the 119 | `Sender`_ end of this Pipe and the corresponding `Recver`_ is 120 | returned:: 121 | 122 | def counter(sender): 123 | i = 0 124 | while True: 125 | i += 1 126 | sender.send(i) 127 | 128 | recver = h.producer(counter) 129 | 130 | recver.recv() # returns 1 131 | recver.recv() # returns 2 132 | """ 133 | sender, recver = self.pipe() 134 | self.spawn(f, sender) 135 | return recver 136 | 137 | def consumer(self, f): 138 | # TODO: this isn't symmetric with producer. need to rethink 139 | # TODO: don't form a closure 140 | # TODO: test 141 | sender, recver = self.pipe() 142 | 143 | @self.spawn 144 | def _(): 145 | for item in recver: 146 | f(item) 147 | return sender 148 | 149 | def pulse(self, ms, item=True): 150 | """ 151 | Convenience to create a `Pipe`_ that will have *item* sent on it every 152 | *ms* milliseconds. The `Recver`_ end of the Pipe is returned. 153 | 154 | Note that since sends to a Pipe block until the Recver is ready, the 155 | pulses will be throttled if the Recver is unable to keep up:: 156 | 157 | recver = h.pulse(500) 158 | 159 | for _ in recver: 160 | log.info('hello') # logs 'hello' every half a second 161 | """ 162 | @self.producer 163 | def _(sender): 164 | while True: 165 | try: 166 | self.sleep(ms) 167 | except vanilla.exception.Halt: 168 | break 169 | sender.send(item) 170 | sender.close() 171 | return _ 172 | 173 | def trigger(self, f): 174 | def consume(recver, f): 175 | for item in recver: 176 | f() 177 | sender, recver = self.pipe() 178 | self.spawn(consume, recver, f) 179 | sender.trigger = functools.partial(sender.send, True) 180 | return sender 181 | 182 | def dealer(self): 183 | """ 184 | Returns a `Dealer`_ `Pair`_. 185 | """ 186 | return vanilla.message.Dealer(self) 187 | 188 | def router(self): 189 | """ 190 | Returns a `Router`_ `Pair`_. 191 | """ 192 | return vanilla.message.Router(self) 193 | 194 | def queue(self, size): 195 | """ 196 | Returns a `Queue`_ `Pair`_. 197 | """ 198 | return vanilla.message.Queue(self, size) 199 | 200 | def channel(self, size=-1): 201 | """ 202 | :: 203 | 204 | send --\ +---------+ /--> recv 205 | +-> | Channel | -+ 206 | send --/ +---------+ \--> recv 207 | 208 | A Channel can have many senders and many recvers. By default it is 209 | unbuffered, but you can create buffered Channels by specifying a size. 210 | They're structurally equivalent to channels in Go. It's implementation 211 | is *literally* a `Router`_ piped to a `Dealer`_, with an optional 212 | `Queue`_ in between. 213 | """ 214 | sender, recver = self.router() 215 | if size > 0: 216 | recver = recver.pipe(self.queue(size)) 217 | return vanilla.message.Pair(sender, recver.pipe(self.dealer())) 218 | 219 | def serialize(self, f): 220 | """ 221 | Decorator to serialize access to a callable *f* 222 | """ 223 | s = self.router() 224 | 225 | @self.spawn 226 | def _(): 227 | for f, a, kw, r in s.recver: 228 | try: 229 | r.send(f(*a, **kw)) 230 | except Exception, e: 231 | r.send(e) 232 | 233 | def _(*a, **kw): 234 | r = self.pipe() 235 | s.send((f, a, kw, r)) 236 | return r.recv() 237 | 238 | return _ 239 | 240 | def broadcast(self): 241 | return vanilla.message.Broadcast(self) 242 | 243 | def state(self, state=vanilla.message.NoState): 244 | """ 245 | Returns a `State`_ `Pair`_. 246 | 247 | *state* if supplied sets the intial state. 248 | """ 249 | return vanilla.message.State(self, state=state) 250 | 251 | def value(self): 252 | return vanilla.message.Value(self) 253 | 254 | def select(self, ends, timeout=-1): 255 | """ 256 | An end is either a `Sender`_ or a `Recver`_. select takes a list of 257 | *ends* and blocks until *one* of them is ready. The select will block 258 | either forever, or until the optional *timeout* is reached. *timeout* 259 | is in milliseconds. 260 | 261 | It returns of tuple of (*end*, *value*) where *end* is the end that has 262 | become ready. If the *end* is a `Recver`_, then it will have already 263 | been *recv*'d on which will be available as *value*. For `Sender`_'s 264 | however the sender is still in a ready state waiting for a *send* and 265 | *value* is None. 266 | 267 | For example, the following is an appliance that takes an upstream 268 | `Recver`_ and a downstream `Sender`_. Sending to its upstream will 269 | alter it's current state. This state can be read at anytime by 270 | receiving on its downstream:: 271 | 272 | def state(h, upstream, downstream): 273 | current = None 274 | while True: 275 | end, value = h.select([upstream, downstream]) 276 | if end == upstream: 277 | current = value 278 | elif end == downstream: 279 | end.send(current) 280 | """ 281 | for end in ends: 282 | if end.ready: 283 | return end, isinstance( 284 | end, vanilla.message.Recver) and end.recv() or None 285 | 286 | for end in ends: 287 | end.select() 288 | 289 | try: 290 | fired, item = self.pause(timeout=timeout) 291 | finally: 292 | for end in ends: 293 | end.unselect() 294 | 295 | return fired, item 296 | 297 | def pause(self, timeout=-1): 298 | if timeout > -1: 299 | item = self.scheduled.add( 300 | timeout, 301 | getcurrent(), 302 | vanilla.exception.Timeout('timeout: %s' % timeout)) 303 | 304 | assert getcurrent() != self.loop, "cannot pause the main loop" 305 | 306 | resume = None 307 | try: 308 | resume = self.loop.switch() 309 | finally: 310 | if timeout > -1: 311 | if isinstance(resume, vanilla.exception.Timeout): 312 | raise resume 313 | # since we didn't timeout, remove ourselves from scheduled 314 | self.scheduled.remove(item) 315 | 316 | # TODO: rework State's is set test to be more natural 317 | if self.stopped.recver.ready: 318 | raise vanilla.exception.Stop( 319 | 'Hub stopped while we were paused. There must be a deadlock.') 320 | 321 | return resume 322 | 323 | def switch_to(self, target, *a): 324 | self.ready.append((getcurrent(), ())) 325 | return target.switch(*a) 326 | 327 | def throw_to(self, target, *a): 328 | self.ready.append((getcurrent(), ())) 329 | """ 330 | if len(a) == 1 and isinstance(a[0], preserve_exception): 331 | return target.throw(a[0].typ, a[0].val, a[0].tb) 332 | """ 333 | return target.throw(*a) 334 | 335 | def spawn(self, f, *a): 336 | """ 337 | Schedules a new green thread to be created to run *f(\*a)* on the next 338 | available tick:: 339 | 340 | def echo(pipe, s): 341 | pipe.send(s) 342 | 343 | p = h.pipe() 344 | h.spawn(echo, p, 'hi') 345 | p.recv() # returns 'hi' 346 | """ 347 | self.ready.append((f, a)) 348 | 349 | def spawn_later(self, ms, f, *a): 350 | """ 351 | Spawns a callable on a new green thread, scheduled for *ms* 352 | milliseconds in the future:: 353 | 354 | def echo(pipe, s): 355 | pipe.send(s) 356 | 357 | p = h.pipe() 358 | h.spawn_later(50, echo, p, 'hi') 359 | p.recv() # returns 'hi' after 50ms 360 | """ 361 | self.scheduled.add(ms, f, *a) 362 | 363 | def sleep(self, ms=1): 364 | """ 365 | Pauses the current green thread for *ms* milliseconds:: 366 | 367 | p = h.pipe() 368 | 369 | @h.spawn 370 | def _(): 371 | p.send('1') 372 | h.sleep(50) 373 | p.send('2') 374 | 375 | p.recv() # returns '1' 376 | p.recv() # returns '2' after 50 ms 377 | """ 378 | self.scheduled.add(ms, getcurrent()) 379 | self.loop.switch() 380 | 381 | def register(self, fd, *masks): 382 | ret = [] 383 | self.registered[fd] = {} 384 | for mask in masks: 385 | sender, recver = self.pipe() 386 | self.registered[fd][mask] = sender 387 | ret.append(recver) 388 | self.poll.register(fd, *masks) 389 | if len(ret) == 1: 390 | return ret[0] 391 | return ret 392 | 393 | def unregister(self, fd): 394 | if fd in self.registered: 395 | masks = self.registered.pop(fd) 396 | try: 397 | self.poll.unregister(fd, *(masks.keys())) 398 | except: 399 | pass 400 | for mask in masks: 401 | masks[mask].close() 402 | 403 | def stop(self): 404 | self.sleep(1) 405 | 406 | for fd, masks in self.registered.items(): 407 | for mask, sender in masks.items(): 408 | sender.stop() 409 | 410 | while self.scheduled: 411 | task, a = self.scheduled.pop() 412 | self.throw_to(task, vanilla.exception.Stop('stop')) 413 | 414 | try: 415 | self.stopped.recv() 416 | except vanilla.exception.Halt: 417 | return 418 | 419 | def stop_on_term(self): 420 | self.signal.subscribe(signal.SIGINT, signal.SIGTERM).recv() 421 | self.stop() 422 | 423 | def run_task(self, task, *a): 424 | try: 425 | if isinstance(task, greenlet): 426 | task.switch(*a) 427 | else: 428 | greenlet(task).switch(*a) 429 | except Exception, e: 430 | self.log.warn('Exception leaked back to main loop', exc_info=e) 431 | 432 | def dispatch_events(self, events): 433 | for fd, mask in events: 434 | if fd in self.registered: 435 | masks = self.registered[fd] 436 | if mask == vanilla.poll.POLLERR: 437 | for sender in masks.values(): 438 | sender.close() 439 | else: 440 | if masks[mask].ready: 441 | masks[mask].send(True) 442 | 443 | def main(self): 444 | """ 445 | Scheduler steps: 446 | - run ready until exhaustion 447 | 448 | - if there's something scheduled 449 | - run overdue scheduled immediately 450 | - or if there's nothing registered, sleep until next scheduled 451 | and then go back to ready 452 | 453 | - if there's nothing registered and nothing scheduled, we've 454 | deadlocked, so stopped 455 | 456 | - poll on registered, with timeout of next scheduled, if something 457 | is scheduled 458 | """ 459 | 460 | while True: 461 | while self.ready: 462 | task, a = self.ready.popleft() 463 | self.run_task(task, *a) 464 | 465 | if self.scheduled: 466 | timeout = self.scheduled.timeout() 467 | # run overdue scheduled immediately 468 | if timeout < 0: 469 | task, a = self.scheduled.pop() 470 | self.run_task(task, *a) 471 | continue 472 | 473 | # if nothing registered, just sleep until next scheduled 474 | if not self.registered: 475 | time.sleep(timeout) 476 | task, a = self.scheduled.pop() 477 | self.run_task(task, *a) 478 | continue 479 | else: 480 | timeout = -1 481 | 482 | # TODO: add better handling for deadlock 483 | if not self.registered: 484 | self.stopped.send(True) 485 | return 486 | 487 | # run poll 488 | events = None 489 | try: 490 | events = self.poll.poll(timeout=timeout) 491 | # IOError from a signal interrupt 492 | except IOError: 493 | pass 494 | if events: 495 | self.spawn(self.dispatch_events, events) 496 | -------------------------------------------------------------------------------- /vanilla/exception.py: -------------------------------------------------------------------------------- 1 | class Timeout(Exception): 2 | pass 3 | 4 | 5 | class Halt(Exception): 6 | pass 7 | 8 | 9 | class Stop(Halt): 10 | pass 11 | 12 | 13 | class Closed(Halt): 14 | pass 15 | 16 | 17 | class Abandoned(Halt): 18 | pass 19 | 20 | 21 | # TODO: think through HTTP Exceptions 22 | class ConnectionLost(Exception): 23 | pass 24 | -------------------------------------------------------------------------------- /vanilla/http.py: -------------------------------------------------------------------------------- 1 | import collections 2 | import functools 3 | import urlparse 4 | import logging 5 | import hashlib 6 | import base64 7 | import urllib 8 | import struct 9 | import json 10 | import time 11 | import uuid 12 | import ssl 13 | import os 14 | 15 | import vanilla.exception 16 | import vanilla.message 17 | import vanilla.meta 18 | 19 | 20 | HTTP_VERSION = 'HTTP/1.1' 21 | 22 | 23 | log = logging.getLogger(__name__) 24 | 25 | 26 | class __plugin__(object): 27 | def __init__(self, hub): 28 | self.hub = hub 29 | 30 | def connect(self, url): 31 | return HTTPClient(self.hub, url) 32 | 33 | # TODO: hacking in convenience for example, still need to add test 34 | # TODO: ensure connection is closed after the get is done 35 | def get(self, uri, params=None, headers=None): 36 | parsed = urlparse.urlsplit(uri) 37 | conn = self.connect('%s://%s' % (parsed.scheme, parsed.netloc)) 38 | return conn.get(parsed.path, params=params, headers=headers) 39 | 40 | def listen(self, port=0, host='127.0.0.1'): 41 | server = self.hub.tcp.listen(host=host, port=port) 42 | ret = server.map( 43 | lambda conn: HTTPServer(self.hub, conn)) 44 | ret.port = server.port 45 | return ret 46 | 47 | 48 | REASON_PHRASES = { 49 | 100: 'CONTINUE', 50 | 101: 'SWITCHING PROTOCOLS', 51 | 102: 'PROCESSING', 52 | 200: 'OK', 53 | 201: 'CREATED', 54 | 202: 'ACCEPTED', 55 | 203: 'NON-AUTHORITATIVE INFORMATION', 56 | 204: 'NO CONTENT', 57 | 205: 'RESET CONTENT', 58 | 206: 'PARTIAL CONTENT', 59 | 207: 'MULTI-STATUS', 60 | 208: 'ALREADY REPORTED', 61 | 226: 'IM USED', 62 | 300: 'MULTIPLE CHOICES', 63 | 301: 'MOVED PERMANENTLY', 64 | 302: 'FOUND', 65 | 303: 'SEE OTHER', 66 | 304: 'NOT MODIFIED', 67 | 305: 'USE PROXY', 68 | 306: 'RESERVED', 69 | 307: 'TEMPORARY REDIRECT', 70 | 308: 'PERMANENT REDIRECT', 71 | 400: 'BAD REQUEST', 72 | 401: 'UNAUTHORIZED', 73 | 402: 'PAYMENT REQUIRED', 74 | 403: 'FORBIDDEN', 75 | 404: 'NOT FOUND', 76 | 405: 'METHOD NOT ALLOWED', 77 | 406: 'NOT ACCEPTABLE', 78 | 407: 'PROXY AUTHENTICATION REQUIRED', 79 | 408: 'REQUEST TIMEOUT', 80 | 409: 'CONFLICT', 81 | 410: 'GONE', 82 | 411: 'LENGTH REQUIRED', 83 | 412: 'PRECONDITION FAILED', 84 | 413: 'REQUEST ENTITY TOO LARGE', 85 | 414: 'REQUEST-URI TOO LONG', 86 | 415: 'UNSUPPORTED MEDIA TYPE', 87 | 416: 'REQUESTED RANGE NOT SATISFIABLE', 88 | 417: 'EXPECTATION FAILED', 89 | 418: "I'M A TEAPOT", 90 | 422: 'UNPROCESSABLE ENTITY', 91 | 423: 'LOCKED', 92 | 424: 'FAILED DEPENDENCY', 93 | 426: 'UPGRADE REQUIRED', 94 | 428: 'PRECONDITION REQUIRED', 95 | 429: 'TOO MANY REQUESTS', 96 | 431: 'REQUEST HEADER FIELDS TOO LARGE', 97 | 500: 'INTERNAL SERVER ERROR', 98 | 501: 'NOT IMPLEMENTED', 99 | 502: 'BAD GATEWAY', 100 | 503: 'SERVICE UNAVAILABLE', 101 | 504: 'GATEWAY TIMEOUT', 102 | 505: 'HTTP VERSION NOT SUPPORTED', 103 | 506: 'VARIANT ALSO NEGOTIATES', 104 | 507: 'INSUFFICIENT STORAGE', 105 | 508: 'LOOP DETECTED', 106 | 510: 'NOT EXTENDED', 107 | 511: 'NETWORK AUTHENTICATION REQUIRED', 108 | } 109 | 110 | 111 | def Status(code): 112 | return code, REASON_PHRASES[code] 113 | 114 | 115 | class Headers(object): 116 | Value = collections.namedtuple('Value', ['key', 'value']) 117 | 118 | def __init__(self): 119 | self.store = {} 120 | 121 | def __setitem__(self, key, value): 122 | self.store[key.lower()] = self.Value(key, value) 123 | 124 | def __contains__(self, key): 125 | return key.lower() in self.store 126 | 127 | def __getitem__(self, key): 128 | return self.store[key.lower()].value 129 | 130 | def __repr__(self): 131 | return repr(dict(self.store.itervalues())) 132 | 133 | def get(self, key, default=None): 134 | try: 135 | return self[key] 136 | except KeyError: 137 | return default 138 | 139 | 140 | class HTTPSocket(object): 141 | 142 | def recv_headers(self): 143 | headers = Headers() 144 | while True: 145 | line = self.socket.recv_line() 146 | if not line: 147 | break 148 | k, v = line.split(': ', 1) 149 | headers[k] = v.strip() 150 | return headers 151 | 152 | def send_headers(self, headers): 153 | headers = '\r\n'.join( 154 | '%s: %s' % (k, v) for k, v in headers.iteritems()) 155 | self.socket.send(headers+'\r\n'+'\r\n') 156 | 157 | def recv_chunk(self): 158 | length = int(self.socket.recv_line(), 16) 159 | if length: 160 | chunk = self.socket.recv_n(length) 161 | else: 162 | chunk = '' 163 | assert self.socket.recv_n(2) == '\r\n' 164 | return chunk 165 | 166 | def send_chunk(self, chunk): 167 | self.socket.send('%s\r\n%s\r\n' % (hex(len(chunk))[2:], chunk)) 168 | 169 | 170 | class HTTPClient(HTTPSocket): 171 | 172 | Status = collections.namedtuple('Status', ['version', 'code', 'message']) 173 | 174 | class Response(object): 175 | def __init__(self, status, headers, body): 176 | self.status = status 177 | self.headers = headers 178 | self.body = body 179 | 180 | def consume(self): 181 | return ''.join(self.body) 182 | 183 | def json(self): 184 | return json.loads(self.consume()) 185 | 186 | def __repr__(self): 187 | return 'HTTPClient.Response(status=%r)' % (self.status,) 188 | 189 | def __init__(self, hub, url): 190 | self.hub = hub 191 | 192 | parsed = urlparse.urlsplit(url) 193 | assert parsed.query == '' 194 | assert parsed.fragment == '' 195 | 196 | default_port = 443 if parsed.scheme == 'https' else 80 197 | host, port = urllib.splitnport(parsed.netloc, default_port) 198 | 199 | self.socket = self.hub.tcp.connect(host=host, port=port) 200 | 201 | # TODO: this shouldn't block on the SSL handshake 202 | if parsed.scheme == 'https': 203 | # TODO: what a mess 204 | conn = self.socket.sender.fd.conn 205 | conn = ssl.wrap_socket(conn) 206 | conn.setblocking(0) 207 | self.socket.sender.fd.conn = conn 208 | 209 | self.socket.recver.sep = '\r\n' 210 | 211 | self.agent = 'vanilla/%s' % vanilla.meta.__version__ 212 | 213 | self.default_headers = dict([ 214 | ('Accept', '*/*'), 215 | ('User-Agent', self.agent), 216 | ('Host', parsed.netloc), ]) 217 | 218 | self.requests = self.hub.router().pipe(self.hub.queue(10)) 219 | self.requests.pipe(self.hub.consumer(self.writer)) 220 | 221 | self.responses = self.hub.router().pipe(self.hub.queue(10)) 222 | self.responses.pipe(self.hub.consumer(self.reader)) 223 | 224 | def reader(self, response): 225 | try: 226 | version, code, message = self.socket.recv_line().split(' ', 2) 227 | except vanilla.exception.Halt: 228 | # TODO: could we offer the ability to auto-reconnect? 229 | try: 230 | response.send(vanilla.exception.ConnectionLost()) 231 | except vanilla.exception.Abandoned: 232 | # TODO: super need to think this through 233 | pass 234 | return 235 | 236 | code = int(code) 237 | status = self.Status(version, code, message) 238 | # TODO: 239 | # if status.code == 408: 240 | 241 | headers = self.recv_headers() 242 | sender, recver = self.hub.pipe() 243 | 244 | response.send(self.Response(status, headers, recver)) 245 | 246 | if headers.get('Connection') == 'Upgrade': 247 | sender.close() 248 | return 249 | 250 | try: 251 | if headers.get('transfer-encoding') == 'chunked': 252 | while True: 253 | chunk = self.recv_chunk() 254 | if not chunk: 255 | break 256 | sender.send(chunk) 257 | else: 258 | # TODO: 259 | # http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.4 260 | length = headers.get('content-length') 261 | if length is None: 262 | """ 263 | TODO: 264 | content-length isn't in header, assume body is marked by 265 | connection closed 266 | """ 267 | body = '' 268 | while True: 269 | try: 270 | body += self.socket.recv() 271 | except vanilla.exception.Closed: 272 | break 273 | sender.send(body) 274 | else: 275 | body = self.socket.recv_n(int(length)) 276 | sender.send(body) 277 | 278 | except vanilla.exception.Halt: 279 | # TODO: could we offer the ability to auto-reconnect? 280 | sender.send(vanilla.exception.ConnectionLost()) 281 | 282 | sender.close() 283 | 284 | def request( 285 | self, 286 | method, 287 | path='/', 288 | params=None, 289 | headers=None, 290 | data=None): 291 | 292 | self.requests.send((method, path, params, headers, data)) 293 | sender, recver = self.hub.pipe() 294 | self.responses.send(sender) 295 | return recver 296 | 297 | def writer(self, request): 298 | method, path, params, headers, data = request 299 | 300 | request_headers = {} 301 | request_headers.update(self.default_headers) 302 | if headers: 303 | request_headers.update(headers) 304 | 305 | if params: 306 | path += '?' + urllib.urlencode(params) 307 | 308 | request = '%s %s %s\r\n' % (method, path, HTTP_VERSION) 309 | self.socket.send(request) 310 | 311 | # TODO: handle chunked transfers 312 | if data is not None: 313 | 314 | if isinstance(data, dict): 315 | request_headers['Content-Type'] = \ 316 | 'application/x-www-form-urlencoded' 317 | data = urllib.urlencode(data) 318 | 319 | request_headers['Content-Length'] = len(data) 320 | 321 | self.send_headers(request_headers) 322 | 323 | # TODO: handle chunked transfers 324 | if data is not None: 325 | self.socket.send(data) 326 | 327 | def get(self, path='/', params=None, headers=None, auth=None): 328 | if auth: 329 | if not headers: 330 | headers = {} 331 | headers['Authorization'] = \ 332 | 'Basic ' + base64.b64encode('%s:%s' % auth) 333 | return self.request('GET', path, params, headers, None) 334 | 335 | def post(self, path='/', params=None, headers=None, data=''): 336 | return self.request('POST', path, params, headers, data) 337 | 338 | def put(self, path='/', params=None, headers=None, data=''): 339 | return self.request('PUT', path, params, headers, data) 340 | 341 | def delete(self, path='/', params=None, headers=None): 342 | return self.request('DELETE', path, params, headers, None) 343 | 344 | def websocket(self, path='/', params=None, headers=None): 345 | key = base64.b64encode(uuid.uuid4().bytes) 346 | 347 | headers = headers or {} 348 | headers.update({ 349 | 'Upgrade': 'WebSocket', 350 | 'Connection': 'Upgrade', 351 | 'Sec-WebSocket-Key': key, 352 | 'Sec-WebSocket-Version': 13, }) 353 | 354 | response = self.request('GET', path, params, headers, None).recv() 355 | assert response.status.code == 101 356 | assert response.headers['Upgrade'].lower() == 'websocket' 357 | assert response.headers['Sec-WebSocket-Accept'] == \ 358 | WebSocket.accept_key(key) 359 | 360 | return WebSocket(self.hub, self.socket) 361 | 362 | def close(self): 363 | # TODO: handle inflight requests? 364 | self.socket.close() 365 | 366 | 367 | class HTTPServer(HTTPSocket): 368 | def __init__(self, hub, socket): 369 | self.hub = hub 370 | 371 | self.socket = socket 372 | self.socket.recver.sep = '\r\n' 373 | 374 | self.responses = self.hub.router() 375 | 376 | @self.responses.consume 377 | def writer(response): 378 | status, headers, body = response 379 | 380 | self.socket.send('HTTP/1.1 %s %s\r\n' % status) 381 | 382 | if headers.get('Connection') == 'Upgrade': 383 | self.send_headers(headers) 384 | self.responses.close() 385 | return 386 | 387 | headers.setdefault( 388 | 'Date', 389 | time.strftime('%a, %d %b %Y %H:%M:%S GMT', time.gmtime())) 390 | 391 | # if body is a pipe, use chunked encoding 392 | if hasattr(body, 'recv'): 393 | headers['Transfer-Encoding'] = 'chunked' 394 | self.send_headers(headers) 395 | for chunk in body: 396 | self.send_chunk(chunk) 397 | self.send_chunk('') 398 | 399 | # otherwise send in oneshot 400 | else: 401 | headers['Content-Length'] = len(body) 402 | self.send_headers(headers) 403 | self.socket.send(body) 404 | 405 | Request = collections.namedtuple( 406 | 'Request', ['method', 'path', 'version', 'headers']) 407 | 408 | class Request(Request): 409 | 410 | @property 411 | def form(self): 412 | if self.headers.get('Content-Type') != \ 413 | 'application/x-www-form-urlencoded': 414 | raise AttributeError('not a form encoded request') 415 | return dict(urlparse.parse_qsl(self.body)) 416 | 417 | @property 418 | def form_multi(self): 419 | if self.headers.get('Content-Type') != \ 420 | 'application/x-www-form-urlencoded': 421 | raise AttributeError('not a form encoded request') 422 | return urlparse.parse_qs(self.body) 423 | 424 | def consume(self): 425 | return self.body 426 | 427 | def reply(self, status, headers, body): 428 | self.server.responses.send((status, headers, body)) 429 | 430 | def upgrade(self): 431 | # TODO: the connection header can be a list of tokens, this should 432 | # be handled more comprehensively 433 | connection_tokens = [ 434 | x.strip().lower() 435 | for x in self.headers['Connection'].split(',')] 436 | assert 'upgrade' in connection_tokens 437 | 438 | assert self.headers['Upgrade'].lower() == 'websocket' 439 | 440 | key = self.headers['Sec-WebSocket-Key'] 441 | accept = WebSocket.accept_key(key) 442 | headers = { 443 | "Upgrade": "websocket", 444 | "Connection": "Upgrade", 445 | "Sec-WebSocket-Accept": accept, } 446 | 447 | self.reply(Status(101), headers, None) 448 | 449 | return WebSocket( 450 | self.server.hub, self.server.socket, is_client=False) 451 | 452 | def recv(self, timeout=None): 453 | method, path, version = self.socket.recv_line().split(' ', 2) 454 | headers = self.recv_headers() 455 | request = self.Request(method, path, version, headers) 456 | # TODO: handle chunked transfers 457 | length = int(headers.get('content-length', 0)) 458 | request.body = self.socket.recv_n(length) 459 | request.server = self 460 | return request 461 | 462 | # TODO: we should provide the standard Recver API 463 | def __iter__(self): 464 | while True: 465 | try: 466 | yield self.recv() 467 | except vanilla.exception.Halt: 468 | break 469 | 470 | 471 | class WebSocket(object): 472 | MASK = FIN = 0b10000000 473 | RSV = 0b01110000 474 | OP = 0b00001111 475 | CONTROL = 0b00001000 476 | PAYLOAD = 0b01111111 477 | 478 | OP_TEXT = 0x1 479 | OP_BIN = 0x2 480 | OP_CLOSE = 0x8 481 | OP_PING = 0x9 482 | OP_PONG = 0xA 483 | 484 | SANITY = 1024**3 # limit fragments to 1GB 485 | 486 | def __new__(cls, hub, socket, is_client=True): 487 | sender = hub.pipe() \ 488 | .map(functools.partial(WebSocket.send, is_client)) \ 489 | .pipe(socket).sender 490 | 491 | recver = socket \ 492 | .pipe(functools.partial(WebSocket.recv, is_client)).recver 493 | 494 | @recver.onclose 495 | def close(): 496 | MASK = WebSocket.MASK if is_client else 0 497 | header = struct.pack( 498 | '!BB', 499 | WebSocket.OP_CLOSE | WebSocket.FIN, 500 | MASK) 501 | socket.send(header) 502 | socket.close() 503 | 504 | return vanilla.message.Pair(sender, recver) 505 | 506 | @staticmethod 507 | def mask(mask, s): 508 | mask_bytes = [ord(c) for c in mask] 509 | return ''.join( 510 | chr(mask_bytes[i % 4] ^ ord(c)) for i, c in enumerate(s)) 511 | 512 | @staticmethod 513 | def accept_key(key): 514 | value = key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" 515 | return base64.b64encode(hashlib.sha1(value).digest()) 516 | 517 | @staticmethod 518 | def send(is_client, data): 519 | length = len(data) 520 | 521 | MASK = WebSocket.MASK if is_client else 0 522 | 523 | if length <= 125: 524 | header = struct.pack( 525 | '!BB', 526 | WebSocket.OP_TEXT | WebSocket.FIN, 527 | length | MASK) 528 | 529 | elif length <= 65535: 530 | header = struct.pack( 531 | '!BBH', 532 | WebSocket.OP_TEXT | WebSocket.FIN, 533 | 126 | MASK, 534 | length) 535 | else: 536 | assert length < WebSocket.SANITY, \ 537 | "Frames limited to 1Gb for sanity" 538 | header = struct.pack( 539 | '!BBQ', 540 | WebSocket.OP_TEXT | WebSocket.FIN, 541 | 127 | MASK, 542 | length) 543 | 544 | if is_client: 545 | mask = os.urandom(4) 546 | return header + mask + WebSocket.mask(mask, data) 547 | else: 548 | return header + data 549 | 550 | @staticmethod 551 | def recv(is_client, upstream, downstream): 552 | while True: 553 | b1, length, = struct.unpack('!BB', upstream.recv_n(2)) 554 | assert b1 & WebSocket.FIN, "Fragmented messages not supported yet" 555 | 556 | if is_client: 557 | assert not length & WebSocket.MASK 558 | else: 559 | assert length & WebSocket.MASK 560 | length = length & WebSocket.PAYLOAD 561 | 562 | # TODO: support binary 563 | opcode = b1 & WebSocket.OP 564 | 565 | if opcode & WebSocket.CONTROL: 566 | # this is a control frame 567 | assert length <= 125 568 | if opcode == WebSocket.OP_CLOSE: 569 | upstream.recv_n(length) 570 | upstream.close() 571 | raise vanilla.exception.Closed 572 | 573 | if length == 126: 574 | length, = struct.unpack('!H', upstream.recv_n(2)) 575 | 576 | elif length == 127: 577 | length, = struct.unpack('!Q', upstream.recv_n(8)) 578 | 579 | assert length < WebSocket.SANITY, \ 580 | "Frames limited to 1Gb for sanity" 581 | 582 | if is_client: 583 | downstream.send(upstream.recv_n(length)) 584 | else: 585 | mask = upstream.recv_n(4) 586 | downstream.send(WebSocket.mask(mask, upstream.recv_n(length))) 587 | -------------------------------------------------------------------------------- /vanilla/io.py: -------------------------------------------------------------------------------- 1 | import socket 2 | import fcntl 3 | import errno 4 | import ssl 5 | import os 6 | 7 | import vanilla.exception 8 | import vanilla.message 9 | import vanilla.poll 10 | 11 | 12 | class __plugin__(object): 13 | def __init__(self, hub): 14 | self.hub = hub 15 | 16 | def fd_in(self, fd): 17 | return Recver(FD_from_fileno_in(self.hub, fd)) 18 | 19 | def fd_out(self, fd): 20 | return Sender(FD_from_fileno_out(self.hub, fd)) 21 | 22 | def pipe(self): 23 | r, w = os.pipe() 24 | recver = Recver(FD_from_fileno_in(self.hub, r)) 25 | sender = Sender(FD_from_fileno_out(self.hub, w)) 26 | return vanilla.message.Pair(sender, recver) 27 | 28 | def socket(self, conn): 29 | fd = FD_from_socket(self.hub, conn) 30 | recver = vanilla.io.Recver(fd) 31 | sender = vanilla.io.Sender(fd) 32 | return vanilla.message.Pair(sender, recver) 33 | 34 | 35 | def unblock(fileno): 36 | flags = fcntl.fcntl(fileno, fcntl.F_GETFL, 0) 37 | flags = flags | os.O_NONBLOCK 38 | fcntl.fcntl(fileno, fcntl.F_SETFL, flags) 39 | return fileno 40 | 41 | 42 | class FD_from_fileno_in(object): 43 | def __init__(self, hub, fileno): 44 | self.hub = hub 45 | self.fileno = fileno 46 | unblock(self.fileno) 47 | self.pollin = hub.register(self.fileno, vanilla.poll.POLLIN) 48 | 49 | def read(self, n): 50 | return os.read(self.fileno, n) 51 | 52 | def close(self): 53 | try: 54 | os.close(self.fileno) 55 | except OSError: 56 | pass 57 | self.hub.unregister(self.fileno) 58 | 59 | 60 | class FD_from_fileno_out(object): 61 | def __init__(self, hub, fileno): 62 | self.hub = hub 63 | self.fileno = fileno 64 | unblock(self.fileno) 65 | self.pollout = hub.register(self.fileno, vanilla.poll.POLLOUT) 66 | 67 | def write(self, data): 68 | return os.write(self.fileno, data) 69 | 70 | def close(self): 71 | try: 72 | os.close(self.fileno) 73 | except OSError: 74 | pass 75 | self.hub.unregister(self.fileno) 76 | 77 | 78 | class FD_from_socket(object): 79 | def __init__(self, hub, conn): 80 | self.hub = hub 81 | self.conn = conn 82 | self.closed = False 83 | self.fileno = self.conn.fileno() 84 | unblock(self.fileno) 85 | self.pollin, self.pollout = hub.register( 86 | self.fileno, vanilla.poll.POLLIN, vanilla.poll.POLLOUT) 87 | 88 | def read(self, n): 89 | return self.conn.recv(n) 90 | 91 | def write(self, data): 92 | return self.conn.send(data) 93 | 94 | def close(self): 95 | if self.closed: 96 | return 97 | self.closed = True 98 | self.conn.close() 99 | self.hub.unregister(self.fileno) 100 | 101 | 102 | class Sender(object): 103 | def __init__(self, fd): 104 | self.fd = fd 105 | self.hub = fd.hub 106 | 107 | self.gate = self.hub.router().pipe(self.hub.state()) 108 | self.fd.pollout.pipe(self.gate) 109 | self.fd.pollout.onclose(self.close) 110 | 111 | @self.hub.serialize 112 | def send(data, timeout=-1): 113 | # TODO: test timeout 114 | while True: 115 | try: 116 | n = self.fd.write(data) 117 | except (socket.error, OSError), e: 118 | if e.errno == errno.EAGAIN: 119 | self.gate.clear().recv() 120 | continue 121 | self.close() 122 | raise vanilla.exception.Closed() 123 | if n == len(data): 124 | break 125 | data = data[n:] 126 | self.send = send 127 | 128 | def connect(self, recver): 129 | recver.consume(self.send) 130 | 131 | def close(self): 132 | self.gate.close() 133 | self.fd.close() 134 | 135 | 136 | def Recver(fd): 137 | hub = fd.hub 138 | sender, recver = hub.pipe() 139 | 140 | recver.onclose(fd.close) 141 | 142 | @hub.spawn 143 | def _(): 144 | for _ in fd.pollin: 145 | while True: 146 | try: 147 | data = fd.read(16384) 148 | except (socket.error, OSError), e: 149 | if e.errno == errno.EAGAIN: 150 | break 151 | """ 152 | # TODO: investigate handling non-blocking ssl correctly 153 | # perhaps SSL_set_fd() ?? 154 | """ 155 | if isinstance(e, ssl.SSLError): 156 | break 157 | sender.close() 158 | return 159 | 160 | if not data: 161 | sender.close() 162 | return 163 | 164 | sender.send(data) 165 | sender.close() 166 | 167 | return vanilla.message.Stream(recver) 168 | -------------------------------------------------------------------------------- /vanilla/message.py: -------------------------------------------------------------------------------- 1 | import collections 2 | import weakref 3 | 4 | from greenlet import getcurrent 5 | 6 | import vanilla.exception 7 | 8 | 9 | Pair = collections.namedtuple('Pair', ['sender', 'recver']) 10 | 11 | 12 | class NoState(object): 13 | """a marker to indicate no state""" 14 | 15 | 16 | class Pair(Pair): 17 | """ 18 | A Pair is a tuple of a `Sender`_ and a `Recver`_. The pair only share a 19 | weakref to each other so unless a reference is kept to both ends, the 20 | remaining end will be *abandoned* and the entire pair will be garbage 21 | collected. 22 | 23 | It's possible to call methods directly on the Pair tuple. A common pattern 24 | though is to split up the tuple with the `Sender`_ used in one closure and 25 | the `Recver`_ in another:: 26 | 27 | # create a Pipe Pair 28 | p = h.pipe() 29 | 30 | # call the Pair tuple directly 31 | h.spawn(p.send, '1') 32 | p.recv() # returns '1' 33 | 34 | # split the sender and recver 35 | sender, recver = p 36 | sender.send('2') 37 | recver.recv() # returns '2' 38 | """ 39 | def send(self, item, timeout=-1): 40 | """ 41 | Send an *item* on this pair. This will block unless our Rever is ready, 42 | either forever or until *timeout* milliseconds. 43 | """ 44 | return self.sender.send(item, timeout=timeout) 45 | 46 | def clear(self): 47 | self.sender.clear() 48 | return self 49 | 50 | def recv(self, timeout=-1): 51 | """ 52 | Receive and item from our Sender. This will block unless our Sender is 53 | ready, either forever or unless *timeout* milliseconds. 54 | """ 55 | return self.recver.recv(timeout=timeout) 56 | 57 | def recv_n(self, n, timeout=-1): 58 | return self.recver.recv_n(n, timeout=timeout) 59 | 60 | def recv_partition(self, sep, timeout=-1): 61 | return self.recver.recv_partition(sep, timeout=timeout) 62 | 63 | def recv_line(self, timeout=-1): 64 | return self.recver.recv_line(timeout=timeout) 65 | 66 | def pipe(self, target): 67 | """ 68 | Pipes are Recver to the target; see :meth:`vanilla.core.Recver.pipe` 69 | 70 | Returns a new Pair of our current Sender and the target's Recver. 71 | """ 72 | return self._replace(recver=self.recver.pipe(target)) 73 | 74 | def map(self, f): 75 | """ 76 | Maps this Pair with *f*'; see :meth:`vanilla.core.Recver.map` 77 | 78 | Returns a new Pair of our current Sender and the mapped target's 79 | Recver. 80 | """ 81 | return self._replace(recver=self.recver.map(f)) 82 | 83 | def consume(self, f): 84 | """ 85 | Consumes this Pair with *f*; see :meth:`vanilla.core.Recver.consume`. 86 | 87 | Returns only our Sender 88 | """ 89 | self.recver.consume(f) 90 | return self.sender 91 | 92 | def connect(self, recver): 93 | # TODO: shouldn't this return a new Pair? 94 | return self.sender.connect(recver) 95 | 96 | def onclose(self, f, *a, **kw): 97 | self.recver.onclose(f, *a, **kw) 98 | 99 | def close(self): 100 | """ 101 | Closes both ends of this Pair 102 | """ 103 | self.sender.close() 104 | self.recver.close() 105 | 106 | 107 | class Pipe(object): 108 | """ 109 | :: 110 | 111 | +------+ 112 | send --> | Pipe | --> recv 113 | +------+ 114 | 115 | The most basic primitive is the Pipe. A Pipe has exactly one sender and 116 | exactly one recver. A Pipe has no buffering, so send and recvs will block 117 | until there is a corresponding send or recv. 118 | 119 | For example, the following code will deadlock as the sender will block, 120 | preventing the recv from ever being called:: 121 | 122 | h = vanilla.Hub() 123 | p = h.pipe() 124 | p.send(1) # deadlock 125 | p.recv() 126 | 127 | The following is OK as the send is spawned to a background green thread:: 128 | 129 | h = vanilla.Hub() 130 | p = h.pipe() 131 | h.spawn(p.send, 1) 132 | p.recv() # returns 1 133 | """ 134 | def __new__(cls, hub): 135 | self = super(Pipe, cls).__new__(cls) 136 | self.hub = hub 137 | self.closed = False 138 | 139 | recver = Recver(self) 140 | self.recver = weakref.ref(recver, self.on_abandoned) 141 | self.recver_current = None 142 | 143 | sender = Sender(self) 144 | self.sender = weakref.ref(sender, self.on_abandoned) 145 | self.sender_current = None 146 | 147 | return Pair(sender, recver) 148 | 149 | def on_abandoned(self, *a, **kw): 150 | remaining = self.recver() or self.sender() 151 | if remaining: 152 | # this is running from a preemptive callback triggered by the 153 | # garbage collector. we spawn the abandon clean up in order to pull 154 | # execution back under a green thread owned by our hub, and to 155 | # minimize the amount of code running while preempted. note this 156 | # means spawning needs to be atomic. 157 | self.hub.spawn(remaining.abandoned) 158 | 159 | 160 | class End(object): 161 | def __init__(self, pipe): 162 | self.middle = pipe 163 | 164 | @property 165 | def hub(self): 166 | return self.middle.hub 167 | 168 | @property 169 | def halted(self): 170 | return bool(self.middle.closed or self.other is None) 171 | 172 | @property 173 | def ready(self): 174 | if self.middle.closed: 175 | raise vanilla.exception.Closed 176 | if self.other is None: 177 | raise vanilla.exception.Abandoned 178 | return bool(self.other.current) 179 | 180 | def select(self): 181 | assert self.current is None 182 | self.current = getcurrent() 183 | 184 | def unselect(self): 185 | assert self.current == getcurrent() 186 | self.current = None 187 | 188 | def abandoned(self): 189 | if self.current: 190 | self.hub.throw_to(self.current, vanilla.exception.Abandoned) 191 | 192 | @property 193 | def peak(self): 194 | return self.current 195 | 196 | def pause(self, timeout=-1): 197 | self.select() 198 | try: 199 | _, ret = self.hub.pause(timeout=timeout) 200 | finally: 201 | self.unselect() 202 | return ret 203 | 204 | def onclose(self, f, *a, **kw): 205 | if not hasattr(self.middle, 'closers'): 206 | self.middle.closers = [(f, a, kw)] 207 | else: 208 | self.middle.closers.append((f, a, kw)) 209 | 210 | def close(self, exception=vanilla.exception.Closed): 211 | closers = getattr(self.middle, 'closers', []) 212 | if closers: 213 | del self.middle.closers 214 | 215 | self.middle.closed = True 216 | 217 | if self.other is not None and bool(self.other.current): 218 | self.hub.throw_to(self.other.current, exception) 219 | 220 | for f, a, kw in closers: 221 | try: 222 | f(*a, **kw) 223 | except vanilla.exception.Halt: 224 | pass 225 | 226 | def stop(self): 227 | self.close(exception=vanilla.exception.Stop) 228 | 229 | 230 | class Sender(End): 231 | @property 232 | def current(self): 233 | return self.middle.sender_current 234 | 235 | @current.setter 236 | def current(self, value): 237 | self.middle.sender_current = value 238 | 239 | @property 240 | def other(self): 241 | return self.middle.recver() 242 | 243 | def send(self, item, timeout=-1): 244 | """ 245 | Send an *item* on this pair. This will block unless our Rever is ready, 246 | either forever or until *timeout* milliseconds. 247 | """ 248 | if not self.ready: 249 | self.pause(timeout=timeout) 250 | 251 | if isinstance(item, Exception): 252 | return self.hub.throw_to(self.other.peak, item) 253 | 254 | return self.hub.switch_to(self.other.peak, self.other, item) 255 | 256 | def handover(self, recver): 257 | assert recver.ready 258 | recver.select() 259 | # switch directly, as we need to pause 260 | _, ret = recver.other.peak.switch(recver.other, None) 261 | recver.unselect() 262 | return ret 263 | 264 | def clear(self): 265 | self.send(NoState) 266 | 267 | def connect(self, recver): 268 | """ 269 | Rewire: 270 | s1 -> m1 <- r1 --> s2 -> m2 <- r2 271 | To: 272 | s1 -> m1 <- r2 273 | """ 274 | r1 = recver 275 | m1 = r1.middle 276 | s2 = self 277 | m2 = self.middle 278 | r2 = self.other 279 | 280 | r2.middle = m1 281 | del m2.sender 282 | del m2.recver 283 | 284 | del m1.recver 285 | m1.recver = weakref.ref(r2, m1.on_abandoned) 286 | m1.recver_current = m2.recver_current 287 | 288 | del r1.middle 289 | del s2.middle 290 | 291 | # if we are currently a chain, return the last recver of our chain 292 | while True: 293 | if getattr(r2, 'downstream', None) is None: 294 | break 295 | r2 = r2.downstream.other 296 | return r2 297 | 298 | 299 | class Recver(End): 300 | @property 301 | def current(self): 302 | return self.middle.recver_current 303 | 304 | @current.setter 305 | def current(self, value): 306 | self.middle.recver_current = value 307 | 308 | @property 309 | def other(self): 310 | return self.middle.sender() 311 | 312 | def recv(self, timeout=-1): 313 | """ 314 | Receive and item from our Sender. This will block unless our Sender is 315 | ready, either forever or unless *timeout* milliseconds. 316 | """ 317 | if self.ready: 318 | return self.other.handover(self) 319 | 320 | return self.pause(timeout=timeout) 321 | 322 | def __iter__(self): 323 | while True: 324 | try: 325 | yield self.recv() 326 | except vanilla.exception.Halt: 327 | break 328 | 329 | def pipe(self, target): 330 | """ 331 | Pipes this Recver to *target*. *target* can either be `Sender`_ (or 332 | `Pair`_) or a callable. 333 | 334 | If *target* is a Sender, the two pairs are rewired so that sending on 335 | this Recver's Sender will now be directed to the target's Recver:: 336 | 337 | sender1, recver1 = h.pipe() 338 | sender2, recver2 = h.pipe() 339 | 340 | recver1.pipe(sender2) 341 | 342 | h.spawn(sender1.send, 'foo') 343 | recver2.recv() # returns 'foo' 344 | 345 | If *target* is a callable, a new `Pipe`_ will be created. This Recver 346 | and the new Pipe's Sender are passed to the target callable to act as 347 | upstream and downstream. The callable can then do any processing 348 | desired including filtering, mapping and duplicating packets:: 349 | 350 | sender, recver = h.pipe() 351 | 352 | def pipeline(upstream, downstream): 353 | for i in upstream: 354 | if i % 2: 355 | downstream.send(i*2) 356 | 357 | recver = recver.pipe(pipeline) 358 | 359 | @h.spawn 360 | def _(): 361 | for i in xrange(10): 362 | sender.send(i) 363 | 364 | recver.recv() # returns 2 (0 is filtered, so 1*2) 365 | recver.recv() # returns 6 (2 is filtered, so 3*2) 366 | """ 367 | if callable(target): 368 | sender, recver = self.hub.pipe() 369 | 370 | # link the two ends in the closure with a strong reference to 371 | # prevent them from being garbage collected if this piped section 372 | # is used in a chain 373 | self.downstream = sender 374 | sender.upstream = self 375 | 376 | @self.hub.spawn 377 | def _(): 378 | try: 379 | target(self, sender) 380 | except vanilla.exception.Halt: 381 | sender.close() 382 | 383 | return recver 384 | 385 | else: 386 | return target.connect(self) 387 | 388 | def map(self, f): 389 | """ 390 | *f* is a callable that takes a single argument. All values sent on this 391 | Recver's Sender will be passed to *f* to be transformed:: 392 | 393 | def double(i): 394 | return i * 2 395 | 396 | sender, recver = h.pipe() 397 | recver.map(double) 398 | 399 | h.spawn(sender.send, 2) 400 | recver.recv() # returns 4 401 | """ 402 | @self.pipe 403 | def recver(recver, sender): 404 | for item in recver: 405 | try: 406 | sender.send(f(item)) 407 | except Exception, e: 408 | sender.send(e) 409 | return recver 410 | 411 | def consume(self, f): 412 | """ 413 | Creates a sink which consumes all values for this Recver. *f* is a 414 | callable which takes a single argument. All values sent on this 415 | Recver's Sender will be passed to *f* for processing. Unlike *map* 416 | however consume terminates this chain:: 417 | 418 | sender, recver = h.pipe 419 | 420 | @recver.consume 421 | def _(data): 422 | logging.info(data) 423 | 424 | sender.send('Hello') # logs 'Hello' 425 | """ 426 | @self.hub.spawn 427 | def _(): 428 | for item in self: 429 | # TODO: think through whether trapping for HALT here is a good 430 | # idea 431 | try: 432 | f(item) 433 | except vanilla.exception.Halt: 434 | self.close() 435 | break 436 | 437 | 438 | def Queue(hub, size): 439 | """ 440 | :: 441 | 442 | +----------+ 443 | send --> | Queue | 444 | | (buffer) | --> recv 445 | +----------+ 446 | 447 | A Queue may also only have exactly one sender and recver. A Queue however 448 | has a fifo buffer of a custom size. Sends to the Queue won't block until 449 | the buffer becomes full:: 450 | 451 | h = vanilla.Hub() 452 | q = h.queue(1) 453 | q.send(1) # safe from deadlock 454 | # q.send(1) # this would deadlock however as the queue only has a 455 | # buffer size of 1 456 | q.recv() # returns 1 457 | """ 458 | assert size > 0 459 | 460 | def main(upstream, downstream, size): 461 | queue = collections.deque() 462 | 463 | while True: 464 | if downstream.halted: 465 | # no one is downstream, so shutdown 466 | upstream.close() 467 | return 468 | 469 | watch = [] 470 | if queue: 471 | watch.append(downstream) 472 | else: 473 | # if the buffer is empty, and no one is upstream, shutdown 474 | if upstream.halted: 475 | downstream.close() 476 | return 477 | 478 | # if are upstream is still available, and there is spare room in 479 | # the buffer, watch upstream as well 480 | if not upstream.halted and len(queue) < size: 481 | watch.append(upstream) 482 | 483 | try: 484 | ch, item = hub.select(watch) 485 | except vanilla.exception.Halt: 486 | continue 487 | 488 | if ch == upstream: 489 | queue.append(item) 490 | 491 | elif ch == downstream: 492 | item = queue.popleft() 493 | downstream.send(item) 494 | 495 | upstream = hub.pipe() 496 | downstream = hub.pipe() 497 | 498 | # TODO: rethink this 499 | old_connect = upstream.sender.connect 500 | 501 | def connect(recver): 502 | old_connect(recver) 503 | return downstream.recver 504 | 505 | upstream.sender.connect = connect 506 | 507 | hub.spawn(main, upstream.recver, downstream.sender, size) 508 | return Pair(upstream.sender, downstream.recver) 509 | 510 | 511 | class Dealer(object): 512 | """ 513 | :: 514 | 515 | +--------+ /--> recv 516 | send --> | Dealer | -+ 517 | +--------+ \--> recv 518 | 519 | A Dealer has exactly one sender but can have many recvers. It has no 520 | buffer, so sends and recvs block until a corresponding green thread is 521 | ready. Sends are round robined to waiting recvers on a first come first 522 | serve basis:: 523 | 524 | h = vanilla.Hub() 525 | d = h.dealer() 526 | # d.send(1) # this would deadlock as there are no recvers 527 | h.spawn(lambda: 'recv 1: %s' % d.recv()) 528 | h.spawn(lambda: 'recv 2: %s' % d.recv()) 529 | d.send(1) 530 | d.send(2) 531 | """ 532 | class Recver(Recver): 533 | def select(self): 534 | assert getcurrent() not in self.current 535 | self.current.append(getcurrent()) 536 | 537 | def unselect(self): 538 | self.current.remove(getcurrent()) 539 | 540 | @property 541 | def peak(self): 542 | return self.current[0] 543 | 544 | def abandoned(self): 545 | waiters = list(self.current) 546 | for current in waiters: 547 | self.hub.throw_to(current, vanilla.exception.Abandoned) 548 | 549 | def __new__(cls, hub): 550 | sender, recver = hub.pipe() 551 | recver.__class__ = Dealer.Recver 552 | recver.current = collections.deque() 553 | return Pair(sender, recver) 554 | 555 | 556 | class Router(object): 557 | """ 558 | :: 559 | 560 | send --\ +--------+ 561 | +-> | Router | --> recv 562 | send --/ +--------+ 563 | 564 | A Router has exactly one recver but can have many senders. It has no 565 | buffer, so sends and recvs block until a corresponding thread is ready. 566 | Sends are accepted on a first come first servce basis:: 567 | 568 | h = vanilla.Hub() 569 | r = h.router() 570 | h.spawn(r.send, 3) 571 | h.spawn(r.send, 2) 572 | h.spawn(r.send, 1) 573 | r.recv() # returns 3 574 | r.recv() # returns 2 575 | r.recv() # returns 1 576 | """ 577 | class Sender(Sender): 578 | def select(self): 579 | assert getcurrent() not in self.current 580 | self.current.append(getcurrent()) 581 | 582 | def unselect(self): 583 | self.current.remove(getcurrent()) 584 | 585 | @property 586 | def peak(self): 587 | return self.current[0] 588 | 589 | def abandoned(self): 590 | waiters = list(self.current) 591 | for current in waiters: 592 | self.hub.throw_to(current, vanilla.exception.Abandoned) 593 | 594 | def connect(self, recver): 595 | self.onclose(recver.close) 596 | recver.consume(self.send) 597 | 598 | def __new__(cls, hub): 599 | sender, recver = hub.pipe() 600 | sender.__class__ = Router.Sender 601 | sender.current = collections.deque() 602 | return Pair(sender, recver) 603 | 604 | 605 | class Broadcast(object): 606 | def __init__(self, hub): 607 | self.hub = hub 608 | self.subscribers = [] 609 | self.emptiers = [] 610 | 611 | def onempty(self, f, *a, **kw): 612 | self.emptiers.append((f, a, kw)) 613 | 614 | def send(self, item): 615 | for subscriber in self.subscribers: 616 | subscriber.send(item) 617 | 618 | def unsubscribe(self, sender): 619 | self.subscribers.remove(sender) 620 | if not self.subscribers: 621 | emptiers = self.emptiers 622 | self.emptiers = [] 623 | for f, a, kw in emptiers: 624 | f(*a, **kw) 625 | 626 | def subscribe(self): 627 | sender, recver = self.hub.pipe() 628 | recver.onclose(self.unsubscribe, sender) 629 | self.subscribers.append(sender) 630 | return recver 631 | 632 | def connect(self, recver): 633 | # TODO: this probably should wire onclose to recver.close 634 | recver.consume(self.send) 635 | 636 | 637 | class State(object): 638 | """ 639 | State is a specialized `Pipe`_ which maintains the state of a previous 640 | send. Sends never block, but modify the object's current state. 641 | 642 | When the current state is unset, a recv will block until the state is set. 643 | 644 | If state is set, recvs never block as well, and return the current state. 645 | 646 | State can cleared using the *clear* method:: 647 | 648 | s = h.state() 649 | 650 | s.recv() # this will deadlock as state is not set 651 | 652 | s.send(3) # sets state, note the send doesn't block even though there 653 | # is no recver 654 | s.recv() # 3 655 | s.recv() # 3 - note subsequent recvs don't block 656 | 657 | s.clear() # clear the current state 658 | s.recv() # this will deadlock as state is not set 659 | """ 660 | class G(object): 661 | def __init__(self, hub, state): 662 | self.hub = hub 663 | self.state = state 664 | 665 | # ignore throws 666 | def throw(self, *a, **kw): 667 | self.hub.pause() 668 | 669 | def __nonzero__(self): 670 | return self.state != NoState 671 | 672 | class Sender(Sender): 673 | def init_state(self, item): 674 | self.current = State.G(self.hub, item) 675 | 676 | def send(self, item, timeout=-1): 677 | self.current.state = item 678 | if self.ready and self.current: 679 | return self.hub.switch_to(self.other.peak, self.other, item) 680 | 681 | def handover(self, recver): 682 | assert recver.ready 683 | return self.current.state 684 | 685 | def connect(self, recver): 686 | self.onclose(recver.close) 687 | recver.consume(self.send) 688 | return self.other 689 | 690 | def __new__(cls, hub, state=NoState): 691 | sender, recver = hub.pipe() 692 | sender.__class__ = State.Sender 693 | sender.init_state(state) 694 | return Pair(sender, recver) 695 | 696 | 697 | class Stream(object): 698 | """ 699 | A `Stream`_ is a specialized `Recver`_ which provides additional methods 700 | for working with streaming sources, particularly sockets and file 701 | descriptors. 702 | """ 703 | class Recver(Recver): 704 | def recv(self, timeout=-1): 705 | if self.extra: 706 | extra = self.extra 707 | self.extra = '' 708 | return extra 709 | return super(Stream.Recver, self).recv(timeout=timeout) 710 | 711 | def recv_n(self, n, timeout=-1): 712 | """ 713 | Blocks until *n* bytes of data are available, and then returns 714 | them. 715 | """ 716 | got = '' 717 | if n: 718 | while len(got) < n: 719 | got += self.recv(timeout=timeout) 720 | got, self.extra = got[:n], got[n:] 721 | return got 722 | 723 | def recv_partition(self, sep, timeout=-1): 724 | """ 725 | Blocks until the seperator *sep* is seen in the stream, and then 726 | returns all data received until *sep*. 727 | """ 728 | got = '' 729 | while True: 730 | got += self.recv(timeout=timeout) 731 | keep, matched, extra = got.partition(sep) 732 | if matched: 733 | self.extra = extra 734 | return keep 735 | 736 | def recv_line(self, timeout=-1): 737 | """ 738 | Short hand to receive a line from the stream. The line seperator 739 | defaults to '\\n' but can be changed by setting recver.sep on this 740 | recver. 741 | """ 742 | return self.recv_partition(self.sep, timeout=timeout) 743 | 744 | def __new__(cls, recver, sep='\n'): 745 | recver.__class__ = Stream.Recver 746 | recver.extra = '' 747 | recver.sep = sep 748 | return recver 749 | -------------------------------------------------------------------------------- /vanilla/meta.py: -------------------------------------------------------------------------------- 1 | __version__ = '0.1.16' 2 | -------------------------------------------------------------------------------- /vanilla/poll.py: -------------------------------------------------------------------------------- 1 | import operator 2 | import select 3 | import errno 4 | 5 | 6 | POLLIN = 1 7 | POLLOUT = 2 8 | POLLERR = 3 9 | 10 | 11 | if hasattr(select, 'kqueue'): 12 | class Poll(object): 13 | def __init__(self): 14 | self.q = select.kqueue() 15 | 16 | self.to_ = { 17 | select.KQ_FILTER_READ: POLLIN, 18 | select.KQ_FILTER_WRITE: POLLOUT, } 19 | 20 | self.from_ = dict((v, k) for k, v in self.to_.iteritems()) 21 | 22 | def register(self, fd, *masks): 23 | for mask in masks: 24 | event = select.kevent( 25 | fd, 26 | filter=self.from_[mask], 27 | flags=select.KQ_EV_ADD | select.KQ_EV_CLEAR) 28 | self.q.control([event], 0) 29 | 30 | def unregister(self, fd, *masks): 31 | for mask in masks: 32 | event = select.kevent( 33 | fd, filter=self.from_[mask], flags=select.KQ_EV_DELETE) 34 | self.q.control([event], 0) 35 | 36 | def poll(self, timeout=None): 37 | if timeout == -1: 38 | timeout = None 39 | while True: 40 | try: 41 | events = self.q.control(None, 4, timeout) 42 | break 43 | except OSError, err: 44 | if err.errno == errno.EINTR: 45 | continue 46 | raise 47 | 48 | ret = [] 49 | for e in events: 50 | if e.filter == select.KQ_FILTER_READ and e.data: 51 | ret.append((e.ident, POLLIN)) 52 | if e.filter == select.KQ_FILTER_WRITE: 53 | ret.append((e.ident, POLLOUT)) 54 | if e.flags & (select.KQ_EV_EOF | select.KQ_EV_ERROR): 55 | ret.append((e.ident, POLLERR)) 56 | return ret 57 | 58 | 59 | elif hasattr(select, 'epoll'): 60 | class Poll(object): 61 | def __init__(self): 62 | self.q = select.epoll() 63 | 64 | self.to_ = { 65 | select.EPOLLIN: POLLIN, 66 | select.EPOLLOUT: POLLOUT, } 67 | 68 | self.from_ = dict((v, k) for k, v in self.to_.iteritems()) 69 | 70 | def register(self, fd, *masks): 71 | masks = [self.from_[x] for x in masks] + [ 72 | select.EPOLLET, select.EPOLLERR, select.EPOLLHUP] 73 | self.q.register(fd, reduce(operator.or_, masks, 0)) 74 | 75 | def unregister(self, fd, *masks): 76 | self.q.unregister(fd) 77 | 78 | def poll(self, timeout=-1): 79 | events = self.q.poll(timeout=timeout) 80 | ret = [] 81 | for fd, event in events: 82 | for mask in self.to_: 83 | if event & mask: 84 | ret.append((fd, self.to_[mask])) 85 | if event & (select.EPOLLERR | select.EPOLLHUP): 86 | ret.append((fd, POLLERR)) 87 | return ret 88 | 89 | else: 90 | raise Exception('only epoll or kqueue supported') 91 | -------------------------------------------------------------------------------- /vanilla/process.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | 4 | import functools 5 | import logging 6 | import select 7 | import errno 8 | import signal 9 | import ctypes 10 | import sys 11 | import os 12 | 13 | import vanilla.exception 14 | 15 | 16 | log = logging.getLogger(__name__) 17 | 18 | 19 | # TODO: investigate the equivalent for BSD and OSX 20 | # TODO: should move this and poll into some kind of compat module 21 | # 22 | # Attempt to define a function to ensure out children are sent a SIGTERM when 23 | # our process dies, to avoid orphaned children. 24 | 25 | def set_pdeathsig(): 26 | pass 27 | 28 | if hasattr(select, 'epoll'): 29 | try: 30 | PR_SET_PDEATHSIG = 1 31 | libc = ctypes.CDLL('libc.so.6') 32 | 33 | def set_pdeathsig(): 34 | rc = libc.prctl(PR_SET_PDEATHSIG, signal.SIGTERM) 35 | assert not rc, 'PR_SET_PDEATHSIG failed: %s' % rc 36 | except: 37 | log.warn('unable to load libc: needed to set PR_SET_PDEATHSIG') 38 | 39 | 40 | class __plugin__(object): 41 | def __init__(self, hub): 42 | self.hub = hub 43 | self.children = [] 44 | self.sigchld = None 45 | 46 | class Child(object): 47 | def __init__(self, hub, pid): 48 | self.hub = hub 49 | self.pid = pid 50 | self.done = self.hub.state() 51 | 52 | def check_liveness(self): 53 | try: 54 | pid, code = os.waitpid(self.pid, os.WNOHANG) 55 | except OSError: 56 | return False 57 | 58 | if (pid, code) == (0, 0): 59 | return True 60 | 61 | self.exitcode = code >> 8 62 | self.exitsignal = code & (2**8-1) 63 | self.done.send(self) 64 | return False 65 | 66 | def terminate(self): 67 | self.signal(signal.SIGTERM) 68 | 69 | def signal(self, signum): 70 | os.kill(self.pid, signum) 71 | 72 | def watch(self): 73 | while self.children: 74 | try: 75 | self.sigchld.recv() 76 | except vanilla.exception.Stop: 77 | for child in self.children: 78 | child.terminate() 79 | continue 80 | self.children = [ 81 | child for child in self.children if child.check_liveness()] 82 | self.sigchld.close() 83 | self.sigchld = None 84 | 85 | def bootstrap(self, f, *a, **kw): 86 | import marshal 87 | import cPickle as pickle 88 | 89 | pipe_r, pipe_w = os.pipe() 90 | os.write(pipe_w, pickle.dumps((marshal.dumps(f.func_code), a, kw))) 91 | os.close(pipe_w) 92 | 93 | bootstrap = '\n'.join(x.strip() for x in (""" 94 | import cPickle as pickle 95 | import marshal 96 | import types 97 | import sys 98 | import os 99 | 100 | code, a, kw = pickle.loads(os.read(%(pipe_r)s, 4096)) 101 | os.close(%(pipe_r)s) 102 | 103 | f = types.FunctionType(marshal.loads(code), globals(), 'f') 104 | f(*a, **kw) 105 | """ % {'pipe_r': pipe_r}).split('\n') if x) 106 | 107 | argv = [sys.executable, '-u', '-c', bootstrap] 108 | os.execv(argv[0], argv) 109 | 110 | def launch(self, f, *a, **kw): 111 | stderrtoout = kw.pop('stderrtoout', False) 112 | 113 | if not self.sigchld: 114 | self.sigchld = self.hub.signal.subscribe(signal.SIGCHLD) 115 | self.hub.spawn(self.watch) 116 | 117 | inpipe_r, inpipe_w = os.pipe() 118 | outpipe_r, outpipe_w = os.pipe() 119 | 120 | if not stderrtoout: 121 | errpipe_r, errpipe_w = os.pipe() 122 | 123 | pid = os.fork() 124 | 125 | if pid == 0: 126 | # child process 127 | set_pdeathsig() 128 | 129 | os.close(inpipe_w) 130 | os.dup2(inpipe_r, 0) 131 | os.close(inpipe_r) 132 | 133 | os.close(outpipe_r) 134 | os.dup2(outpipe_w, 1) 135 | 136 | if stderrtoout: 137 | os.dup2(outpipe_w, 2) 138 | else: 139 | os.close(errpipe_r) 140 | os.dup2(errpipe_w, 2) 141 | os.close(errpipe_w) 142 | 143 | os.close(outpipe_w) 144 | 145 | f(*a, **kw) 146 | return 147 | 148 | # parent continues 149 | os.close(inpipe_r) 150 | os.close(outpipe_w) 151 | 152 | child = self.Child(self.hub, pid) 153 | child.stdin = self.hub.io.fd_out(inpipe_w) 154 | child.stdout = self.hub.io.fd_in(outpipe_r) 155 | 156 | if not stderrtoout: 157 | os.close(errpipe_w) 158 | child.stderr = self.hub.io.fd_in(errpipe_r) 159 | 160 | self.children.append(child) 161 | return child 162 | 163 | def spawn(self, f, *a, **kw): 164 | return self.launch(self.bootstrap, f, *a, **kw) 165 | 166 | def execv(self, args, env=None, stderrtoout=False): 167 | def tryexec(f, *a, **kw): 168 | try: 169 | f(*a, **kw) 170 | except OSError, e: 171 | if e.errno == errno.ENOENT: 172 | os._exit(127) 173 | if e.errno == errno.EACCES: 174 | os._exit(126) 175 | raise 176 | 177 | if env: 178 | return self.launch( 179 | functools.partial(tryexec, os.execve), 180 | args[0], 181 | args, 182 | env, 183 | stderrtoout=stderrtoout) 184 | 185 | return self.launch( 186 | functools.partial(tryexec, os.execv), 187 | args[0], 188 | args, 189 | stderrtoout=stderrtoout) 190 | -------------------------------------------------------------------------------- /vanilla/signal.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import signal 4 | import os 5 | 6 | 7 | class __plugin__(object): 8 | def __init__(self, hub): 9 | self.hub = hub 10 | self.fd_w = self.recver = None 11 | self.mapper = {} 12 | 13 | def start(self): 14 | assert not self.fd_w 15 | r, self.fd_w = os.pipe() 16 | self.recver = self.hub.io.fd_in(r) 17 | 18 | @self.hub.spawn 19 | def _(): 20 | for data in self.recver: 21 | for x in data: 22 | sig = ord(x) 23 | self.mapper[sig].send(sig) 24 | self.recver.close() 25 | self.fd_w = self.recver = None 26 | 27 | def capture(self, sig): 28 | if not self.fd_w: 29 | self.start() 30 | 31 | def handler(sig, frame): 32 | # this is running from a preemptive callback triggered by the 33 | # interrupt, so we write directly to a file descriptor instead of 34 | # using an io.pipe() 35 | if self.fd_w: 36 | os.write(self.fd_w, chr(sig)) 37 | 38 | self.mapper[sig] = self.hub.broadcast() 39 | self.mapper[sig].onempty(self.uncapture, sig) 40 | signal.signal(sig, handler) 41 | 42 | def uncapture(self, sig): 43 | assert not self.mapper[sig].subscribers 44 | signal.signal(sig, signal.SIG_DFL) 45 | del self.mapper[sig] 46 | if not self.mapper: 47 | os.close(self.fd_w) 48 | # give the recv side a chance to close 49 | self.hub.sleep(0) 50 | 51 | def subscribe(self, *signals): 52 | router = self.hub.router() 53 | for sig in signals: 54 | if sig not in self.mapper: 55 | self.capture(sig) 56 | self.mapper[sig].subscribe().pipe(router) 57 | return router.recver 58 | -------------------------------------------------------------------------------- /vanilla/tcp.py: -------------------------------------------------------------------------------- 1 | import socket 2 | import errno 3 | 4 | import vanilla.exception 5 | import vanilla.poll 6 | 7 | 8 | class __plugin__(object): 9 | def __init__(self, hub): 10 | self.hub = hub 11 | 12 | def listen(self, port=0, host='127.0.0.1'): 13 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 14 | sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 15 | sock.bind((host, port)) 16 | sock.listen(socket.SOMAXCONN) 17 | sock.setblocking(0) 18 | port = sock.getsockname()[1] 19 | server = self.hub.register(sock.fileno(), vanilla.poll.POLLIN) 20 | 21 | @server.pipe 22 | def server(upstream, downstream): 23 | for mask in upstream: 24 | while True: 25 | try: 26 | conn, host = sock.accept() 27 | downstream.send(self.hub.io.socket(conn)) 28 | except (socket.error, OSError), e: 29 | if e.errno == errno.EAGAIN: 30 | break 31 | raise 32 | self.hub.unregister(sock.fileno()) 33 | sock.close() 34 | 35 | server.port = port 36 | return server 37 | 38 | def connect(self, port, host='127.0.0.1'): 39 | conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 40 | # TODO: this shouldn't block on the connect 41 | conn.connect((host, port)) 42 | return self.hub.io.socket(conn) 43 | -------------------------------------------------------------------------------- /vanilla/thread.py: -------------------------------------------------------------------------------- 1 | import collections 2 | import threading 3 | import os 4 | 5 | from Queue import Queue 6 | 7 | import vanilla 8 | 9 | from vanilla import message 10 | 11 | from vanilla.exception import Closed 12 | 13 | 14 | class Pipe(object): 15 | class Sender(object): 16 | def __init__(self, q, w): 17 | self.q = q 18 | self.w = w 19 | 20 | def send(self, item, timeout=-1): 21 | self.q.append(item) 22 | os.write(self.w, chr(1)) 23 | 24 | def __new__(cls, hub): 25 | r, w = os.pipe() 26 | q = collections.deque() 27 | 28 | sender = Pipe.Sender(q, w) 29 | 30 | r = hub.io.fd_in(r) 31 | 32 | @r.pipe 33 | def recver(r, out): 34 | for s in r: 35 | for ch in s: 36 | ch = ord(ch) 37 | if not ch: 38 | break 39 | out.send(q.popleft()) 40 | r.close() 41 | out.close() 42 | 43 | return message.Pair(sender, recver) 44 | 45 | 46 | class Wrap(object): 47 | def __init__(self, pool, target): 48 | self.pool = pool 49 | self.target = target 50 | 51 | def __call__(self, *a, **kw): 52 | return self.pool.call(self.target, *a, **kw) 53 | 54 | def __getattr__(self, name): 55 | return Wrap(self.pool, getattr(self.target, name)) 56 | 57 | 58 | class Pool(object): 59 | def __init__(self, hub, size): 60 | self.hub = hub 61 | self.size = size 62 | 63 | self.parent = hub.thread.pipe().consume( 64 | lambda (sender, item): sender.send(item)) 65 | 66 | self.requests = Queue() 67 | self.closed = False 68 | self.threads = 0 69 | 70 | for i in xrange(size): 71 | t = threading.Thread(target=self.runner) 72 | t.daemon = True 73 | t.start() 74 | self.threads += 1 75 | 76 | def wrap(self, target): 77 | return Wrap(self, target) 78 | 79 | def runner(self): 80 | while True: 81 | item = self.requests.get() 82 | if type(item) == Closed: 83 | self.threads -= 1 84 | if self.threads <= 0: 85 | # TODO: fix up shutdown 86 | self.parent.close() 87 | return 88 | 89 | sender, f, a, kw = item 90 | self.parent.send((sender, f(*a, **kw))) 91 | self.requests.task_done() 92 | 93 | def call(self, f, *a, **kw): 94 | if self.closed: 95 | raise Closed 96 | sender, recver = self.hub.pipe() 97 | self.requests.put((sender, f, a, kw)) 98 | return recver 99 | 100 | def close(self): 101 | self.closed = True 102 | for i in xrange(self.size): 103 | # tell thread pool to stop when they have finished the last request 104 | self.requests.put(Closed()) 105 | 106 | 107 | class __plugin__(object): 108 | def __init__(self, hub): 109 | self.hub = hub 110 | 111 | def pipe(self): 112 | return Pipe(self.hub) 113 | 114 | def call(self, f, *a): 115 | def bootstrap(sender, f, a): 116 | sender.send(f(*a)) 117 | 118 | sender, recver = self.hub.thread.pipe() 119 | self.t = threading.Thread(target=bootstrap, args=(sender, f, a)) 120 | self.t.start() 121 | return recver 122 | 123 | def pool(self, size): 124 | return Pool(self.hub, size) 125 | 126 | def spawn(self, f, *a): 127 | def bootstrap(parent, f, a): 128 | h = vanilla.Hub() 129 | child = h.thread.pipe() 130 | h.parent = message.Pair(parent.sender, child.recver) 131 | h.parent.send(child.sender) 132 | f(h, *a) 133 | # TODO: handle shutdown 134 | 135 | parent = self.hub.thread.pipe() 136 | t = threading.Thread(target=bootstrap, args=(parent, f, a)) 137 | t.daemon = True 138 | t.start() 139 | return message.Pair(parent.recver.recv(), parent.recver) 140 | -------------------------------------------------------------------------------- /vanilla/udp.py: -------------------------------------------------------------------------------- 1 | import functools 2 | import socket 3 | import errno 4 | import os 5 | 6 | import vanilla.message 7 | import vanilla.poll 8 | 9 | 10 | class __plugin__(object): 11 | def __init__(self, hub): 12 | self.hub = hub 13 | 14 | def create(self): 15 | sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 16 | sock.setblocking(0) 17 | return Sock(self.hub, sock) 18 | 19 | def listen(self, port=0, host='127.0.0.1'): 20 | sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 21 | sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 22 | sock.bind((host, port)) 23 | sock.setblocking(0) 24 | server = Sock(self.hub, sock) 25 | server.port = sock.getsockname()[1] 26 | return server 27 | 28 | 29 | def Sock(hub, sock): 30 | sender = Sender(hub, sock) 31 | recver = Recver(hub, sock) 32 | return vanilla.message.Pair(sender, recver) 33 | 34 | 35 | def Sender(hub, sock): 36 | sender, recver = hub.pipe() 37 | recver.consume(lambda a: sock.sendto(*a)) 38 | return sender 39 | 40 | 41 | def close(hub, fileno): 42 | try: 43 | os.close(fileno) 44 | except OSError: 45 | pass 46 | hub.unregister(fileno) 47 | 48 | 49 | def Recver(hub, sock): 50 | sender, recver = hub.pipe() 51 | 52 | recver.onclose(functools.partial(close, hub, sock.fileno())) 53 | 54 | @hub.spawn 55 | def _(): 56 | ready = hub.register(sock.fileno(), vanilla.poll.POLLIN) 57 | for _ in ready: 58 | while True: 59 | try: 60 | got = sock.recvfrom(65507) 61 | except (socket.error, OSError), e: 62 | if e.errno == errno.EAGAIN: 63 | break 64 | sender.close() 65 | return 66 | sender.send(got) 67 | sender.close() 68 | 69 | return recver 70 | -------------------------------------------------------------------------------- /xx/notes.md: -------------------------------------------------------------------------------- 1 | ## Major Primitives to Document still 2 | 3 | - Channel 4 | - Broadcast 5 | - Value 6 | - State 7 | 8 | ### Major Actions 9 | 10 | - send 11 | - recv 12 | - connect 13 | - pipe 14 | - map 15 | - consume 16 | 17 | - TODO: 18 | - filter 19 | 20 | ### thoughts from Dataflow and Reactive Programming Systems: 21 | 22 | - Only want to activate a section of code if its outputs are available 23 | 24 | - Deemphasize pipes and instead emphasize code units 25 | 26 | - Bring back channels, they're an intuitive building block 27 | 28 | 29 | ### ways to send: 30 | 31 | - pipe: blocks on send, can only have one recver 32 | - one sender, one recver 33 | 34 | - dealer: one sender, many recvers 35 | 36 | - tee: when all are ready, give to all 37 | 38 | - broadcast: give to all that are ready 39 | - one sender, many recvers 40 | 41 | - value: when set always ready, can broadcast updates 42 | - one sender, many recvers 43 | 44 | 45 | ### not sure if this is a sender or a recver .. i think recver?? 46 | 47 | - buffer: always ready until buffer is full 48 | 49 | 50 | ### ways to recv: 51 | 52 | - pipe: as soon as this one is ready, recv 53 | - router: many senders one recver 54 | - gather: once all are ready, recv all 55 | 56 | 57 | ### selectable api: 58 | 59 | - are you ready now? (recv(timeout=0)?) 60 | - enqueue me, but don't pause 61 | - dequeue me (called if we give up on the select) 62 | 63 | 64 | ### consider: 65 | 66 | - what happens if there's an exception on the send side 67 | - what happens if there's an exception on the read side 68 | - what happens when the pipe wants to stop 69 | - what happens when something else wants the pipe to stop 70 | - what happens when the world stops 71 | 72 | ### rpc interface: 73 | 74 | - requires a two way pipe 75 | - how to turn 2x one way pipes into a two way pipe? 76 | - can this be layered on top of any type of message passing primitive? 77 | - how to layer on top of Descriptors? 78 | - observation: the server interface is the same as the current http server 79 | interface 80 | 81 | ``` 82 | push, pull vs request, response 83 | 84 | client: 85 | conn.send(data) 86 | 87 | server: 88 | conn.recv() 89 | 90 | vs 91 | 92 | client: 93 | response = conn.call(data) 94 | response.recv() 95 | 96 | server: 97 | request, response = conn.serve() 98 | response.send(data) 99 | ``` 100 | 101 | ### http: 102 | 103 | - client and server have a socket 104 | 105 | - client sends requests to server ----> 106 | - requests can be streamed, so different requesters need to be seralized 107 | - once a websocket upgrade has been request, new requesters can't be allowed 108 | 109 | - server receives requests <---- 110 | 111 | - for each request the server sends responses 112 | - these responses need to be seralized so they can be matched with requests 113 | - responses can be streamed 114 | 115 | - the client receives responses, it assumes it receives them in the order they 116 | were sent 117 | 118 | - client.request returns instream, outstreamer 119 | 120 | - client socket: 121 | - what happens if socket dies? 122 | - current request is broken 123 | - outstanding responses are broken 124 | - how to signal, and recover? 125 | 126 | - stop, only really makes sense to socket 127 | - prevent new requests 128 | - discard blocked requests 129 | - finish consuming current request 130 | - consume responses 131 | - done 132 | 133 | - recv instream -> drain to socket 134 | on exception, has request started? 135 | - if not, just carry on 136 | - midrequest ?, need to reset socket 137 | - wait for outstanding responses 138 | - reconnect 139 | 140 | - read socket -> pop outstreamer -> send till content; handle returns? 141 | on exception, consume remaining response, continue 142 | 143 | - ticker example 144 | 145 | - 1x conn to nasdaq.com 146 | - collector for e.g. aapl, amzn and outr 147 | - a collector is: 148 | - stream request to http socket 149 | - consume response on http socket 150 | - output parsed stream 151 | - this is just a oneshot function 152 | - if there's an exception, just fail and rerun next pulse 153 | 154 | - collector -> value / broadcast to connected clients 155 | - oneshot function again? 156 | 157 | 158 | - server socket 159 | - read socket, stream of -> instreamer, outstreamer 160 | - send to instreamer until content 161 | - recv outstreams -> drain to socket 162 | -------------------------------------------------------------------------------- /xx/pipe.py: -------------------------------------------------------------------------------- 1 | import vanilla 2 | 3 | from vanilla import * 4 | 5 | 6 | class Pipe(object): 7 | __slots__ = ['hub', 'pair'] 8 | 9 | def __init__(self, hub): 10 | self.hub = hub 11 | self.pair = None 12 | 13 | def send(self, item): 14 | if self.pair: 15 | pair = self.pair 16 | else: 17 | self.pair = getcurrent() 18 | pair = self.hub.pause() 19 | 20 | self.pair = None 21 | self.hub.switch_to(pair, item) 22 | 23 | def recv(self): 24 | if self.pair: 25 | return self.hub.switch_to(self.pair, getcurrent()) 26 | 27 | self.pair = getcurrent() 28 | return self.hub.pause() 29 | 30 | 31 | # what happens if there's an exception on the send side 32 | # what happens if there's an exception on the read side 33 | # what happens when the pipe wants to stop 34 | # what happens when something else wants the pipe to stop 35 | # what happens when the world stops 36 | 37 | def test_pipe(): 38 | h = vanilla.Hub() 39 | 40 | pipe = Pipe(h) 41 | @h.spawn 42 | def _(): 43 | for i in xrange(10): 44 | print "s", i 45 | pipe.send(i) 46 | 47 | print 48 | print 49 | print "-----" 50 | print pipe.recv() 51 | print 52 | print 53 | h.sleep(10) 54 | print "a" 55 | print pipe.recv() 56 | print "b" 57 | -------------------------------------------------------------------------------- /xx/pipe2.py: -------------------------------------------------------------------------------- 1 | import vanilla 2 | 3 | from vanilla import * 4 | 5 | import weakref 6 | import gc 7 | 8 | 9 | class Pair(object): 10 | __slots__ = ['hub', 'current', 'pair'] 11 | 12 | def __init__(self, hub): 13 | self.hub = hub 14 | self.current = None 15 | self.pair = None 16 | 17 | def yack(self, *a, **kw): 18 | print "YAKCK UAKC", a, kw, self.__class__ 19 | 20 | def pair_to(self, pair): 21 | self.pair = weakref.ref(pair, self.yack) 22 | 23 | 24 | class Sender(Pair): 25 | def send(self, item): 26 | if self.pair().current: 27 | current = self.pair().current 28 | 29 | else: 30 | self.current = getcurrent() 31 | current = self.hub.pause() 32 | self.current = None 33 | 34 | self.hub.switch_to(current, item) 35 | 36 | 37 | class Recver(Pair): 38 | def recv(self): 39 | if self.pair().current: 40 | return self.hub.switch_to(self.pair().current, getcurrent()) 41 | 42 | self.current = getcurrent() 43 | item = self.hub.pause() 44 | self.current = None 45 | return item 46 | 47 | 48 | class Pipe(object): 49 | __slots__ = ['hub', 'sender', 'recver'] 50 | 51 | def __init__(self, hub): 52 | self.hub = hub 53 | 54 | self.sender = Sender(hub) 55 | self.recver = Recver(hub) 56 | 57 | self.sender.pair_to(self.recver) 58 | self.recver.pair_to(self.sender) 59 | 60 | def send(self, item): 61 | self.sender.send(item) 62 | 63 | def recv(self): 64 | return self.recver.recv() 65 | 66 | 67 | def test_pipe(): 68 | h = vanilla.Hub() 69 | 70 | pipe = Pipe(h) 71 | @h.spawn 72 | def _(): 73 | for i in xrange(10): 74 | print "s", i 75 | pipe.send(i) 76 | 77 | print 78 | print 79 | print "-----" 80 | print pipe.recv() 81 | print 82 | h.sleep(10) 83 | print "a" 84 | print pipe.recv() 85 | print "b" 86 | 87 | del pipe.recver 88 | -------------------------------------------------------------------------------- /xx/react.py: -------------------------------------------------------------------------------- 1 | import vanilla 2 | 3 | from vanilla import * 4 | 5 | import weakref 6 | import gc 7 | 8 | 9 | class Pair(object): 10 | def __init__(self, hub): 11 | self.hub = hub 12 | self.current = None 13 | self.pair = None 14 | 15 | def yack(self, *a, **kw): 16 | print "YAKCK UAKC", a, kw, self.__class__ 17 | 18 | def pair_to(self, pair): 19 | self.pair = weakref.ref(pair, self.yack) 20 | 21 | 22 | class Sender(Pair): 23 | def send(self, item): 24 | if self.pair().current: 25 | current = self.pair().current 26 | 27 | else: 28 | self.current = getcurrent() 29 | current = self.hub.pause() 30 | self.current = None 31 | 32 | self.hub.switch_to(current, item) 33 | 34 | 35 | class Recver(Pair): 36 | def recv(self): 37 | # only allow one recv at a time 38 | assert self.current is None 39 | 40 | if self.pair().current: 41 | return self.hub.switch_to(self.pair().current, getcurrent()) 42 | 43 | self.current = getcurrent() 44 | item = self.hub.pause() 45 | self.current = None 46 | return item 47 | 48 | def __iter__(self): 49 | while True: 50 | yield self.recv() 51 | 52 | def pipe(self): 53 | sender, recver = pipe(self.hub) 54 | @self.hub.spawn 55 | def _(): 56 | for item in self: 57 | sender.send(item) 58 | return recver 59 | 60 | def map(self, f): 61 | sender, recver = pipe(self.hub) 62 | @self.hub.spawn 63 | def _(): 64 | for item in self: 65 | sender.send(f(item)) 66 | return recver 67 | 68 | 69 | def pipe(hub): 70 | sender = Sender(hub) 71 | recver = Recver(hub) 72 | 73 | sender.pair_to(recver) 74 | recver.pair_to(sender) 75 | 76 | return sender, recver 77 | 78 | 79 | def stream(hub): 80 | def dec(f): 81 | sender, recver = pipe(hub) 82 | hub.spawn(f, sender) 83 | return recver 84 | return dec 85 | 86 | 87 | def test_stream(): 88 | h = vanilla.Hub() 89 | 90 | print 91 | print 92 | 93 | @stream(h) 94 | def counter(out): 95 | for i in xrange(10): 96 | print "s", i 97 | out.send(i) 98 | 99 | 100 | """ 101 | @counter.map 102 | def double(n): 103 | return n * 2 104 | 105 | 106 | print double.recv() 107 | print double.recv() 108 | print double.recv() 109 | """ 110 | 111 | ch = counter.pipe() 112 | print ch.recv() 113 | print ch.recv() 114 | 115 | 116 | 117 | print 118 | print 119 | -------------------------------------------------------------------------------- /xx/router.py: -------------------------------------------------------------------------------- 1 | import vanilla 2 | 3 | from vanilla import * 4 | 5 | import weakref 6 | import gc 7 | 8 | import pytest 9 | 10 | 11 | class Pair(object): 12 | __slots__ = ['hub', 'current', 'pair'] 13 | 14 | def __init__(self, hub): 15 | self.hub = hub 16 | self.current = None 17 | self.pair = None 18 | 19 | def on_abandoned(self, *a, **kw): 20 | if self.current: 21 | self.hub.throw_to(self.current, Abandoned) 22 | 23 | def pair_to(self, pair): 24 | self.pair = weakref.ref(pair, self.on_abandoned) 25 | 26 | @property 27 | def other(self): 28 | if self.pair() is None: 29 | raise Abandoned 30 | return self.pair().current 31 | 32 | @property 33 | def ready(self): 34 | return self.other is not None 35 | 36 | def select(self, current=None): 37 | assert self.current is None 38 | self.current = current or getcurrent() 39 | 40 | def unselect(self): 41 | assert self.current == getcurrent() 42 | self.current = None 43 | 44 | def pause(self, timeout=-1): 45 | self.select() 46 | try: 47 | _, ret = self.hub.pause(timeout=timeout) 48 | finally: 49 | self.unselect() 50 | return ret 51 | 52 | 53 | class Sender(Pair): 54 | def send(self, item, timeout=-1): 55 | # only allow one send at a time 56 | assert self.current is None 57 | if not self.ready: 58 | self.pause(timeout=timeout) 59 | return self.hub.switch_to(self.other, self.pair(), item) 60 | 61 | 62 | class Recver(Pair): 63 | def recv(self, timeout=-1): 64 | # only allow one recv at a time 65 | assert self.current is None 66 | 67 | if self.ready: 68 | self.current = getcurrent() 69 | # switch directly, as we need to pause 70 | _, ret = self.other.switch(self.pair(), None) 71 | self.current = None 72 | return ret 73 | 74 | return self.pause(timeout=timeout) 75 | 76 | 77 | class Hub(vanilla.Hub): 78 | def sender(self): 79 | return Sender(self) 80 | 81 | def recver(self): 82 | return Recver(self) 83 | 84 | def pipe(self): 85 | sender = self.sender() 86 | recver = self.recver() 87 | sender.pair_to(recver) 88 | recver.pair_to(sender) 89 | return sender, recver 90 | 91 | def stream(self, f): 92 | sender, recver = self.pipe() 93 | self.spawn(f, sender) 94 | return recver 95 | 96 | def pulse(self, ms, item=True): 97 | @self.stream 98 | def _(sender): 99 | while True: 100 | self.sleep(ms) 101 | sender.send(item) 102 | return _ 103 | 104 | def select(self, pairs, timeout=-1): 105 | for pair in pairs: 106 | if pair.ready: 107 | return pair, isinstance(pair, Recver) and pair.recv() or None 108 | 109 | for pair in pairs: 110 | pair.select() 111 | 112 | try: 113 | fired, item = self.pause(timeout=timeout) 114 | finally: 115 | for pair in pairs: 116 | pair.unselect() 117 | 118 | return fired, item 119 | 120 | 121 | vanilla.Hub = Hub 122 | 123 | 124 | def buffer(hub, size): 125 | buff = collections.deque() 126 | 127 | # TODO: don't form a closure around sender and recver 128 | sender, _recver = hub.pipe() 129 | _sender, recver = hub.pipe() 130 | 131 | @hub.spawn 132 | def _(): 133 | while True: 134 | watch = [] 135 | if len(buff) < size: 136 | watch.append(_recver) 137 | if buff: 138 | watch.append(_sender) 139 | 140 | ch, item = hub.select(watch) 141 | 142 | if ch == _recver: 143 | buff.append(item) 144 | 145 | elif ch == _sender: 146 | item = buff.popleft() 147 | _sender.send(item) 148 | 149 | return sender, recver 150 | 151 | 152 | def test_stream(): 153 | h = vanilla.Hub() 154 | 155 | @h.stream 156 | def counter(sender): 157 | for i in xrange(10): 158 | sender.send(i) 159 | 160 | assert counter.recv() == 0 161 | h.sleep(10) 162 | assert counter.recv() == 1 163 | 164 | 165 | def test_abandoned_sender(): 166 | h = vanilla.Hub() 167 | 168 | check_sender, check_recver = h.pipe() 169 | 170 | # test abondoned after pause 171 | sender, recver = h.pipe() 172 | 173 | @h.spawn 174 | def _(): 175 | pytest.raises(vanilla.Abandoned, sender.send, 10) 176 | check_sender.send('done') 177 | 178 | # sleep so the spawn runs and the send pauses 179 | h.sleep(1) 180 | del recver 181 | gc.collect() 182 | assert check_recver.recv() == 'done' 183 | 184 | # test abondoned before pause 185 | sender, recver = h.pipe() 186 | 187 | @h.spawn 188 | def _(): 189 | pytest.raises(vanilla.Abandoned, sender.send, 10) 190 | check_sender.send('done') 191 | 192 | del recver 193 | gc.collect() 194 | assert check_recver.recv() == 'done' 195 | 196 | 197 | def test_abandoned_recver(): 198 | h = vanilla.Hub() 199 | 200 | check_sender, check_recver = h.pipe() 201 | 202 | # test abondoned after pause 203 | sender, recver = h.pipe() 204 | 205 | @h.spawn 206 | def _(): 207 | pytest.raises(vanilla.Abandoned, recver.recv) 208 | check_sender.send('done') 209 | 210 | # sleep so the spawn runs and the recv pauses 211 | h.sleep(1) 212 | del sender 213 | gc.collect() 214 | assert check_recver.recv() == 'done' 215 | 216 | # test abondoned before pause 217 | sender, recver = h.pipe() 218 | 219 | @h.spawn 220 | def _(): 221 | pytest.raises(vanilla.Abandoned, recver.recv) 222 | check_sender.send('done') 223 | 224 | del sender 225 | gc.collect() 226 | assert check_recver.recv() == 'done' 227 | 228 | 229 | def test_pulse(): 230 | h = vanilla.Hub() 231 | 232 | trigger = h.pulse(20) 233 | pytest.raises(vanilla.Timeout, trigger.recv, timeout=0) 234 | 235 | h.sleep(20) 236 | assert trigger.recv(timeout=0) 237 | pytest.raises(vanilla.Timeout, trigger.recv, timeout=0) 238 | 239 | h.sleep(20) 240 | assert trigger.recv(timeout=0) 241 | pytest.raises(vanilla.Timeout, trigger.recv, timeout=0) 242 | 243 | # TODO: test abandoned 244 | 245 | 246 | def test_select(): 247 | h = vanilla.Hub() 248 | 249 | s1, r1 = h.pipe() 250 | s2, r2 = h.pipe() 251 | check_s, check_r = h.pipe() 252 | 253 | @h.spawn 254 | def _(): 255 | check_s.send(r1.recv()) 256 | 257 | @h.spawn 258 | def _(): 259 | s2.send(10) 260 | check_s.send('done') 261 | 262 | ch, item = h.select([s1, r2]) 263 | assert ch == s1 264 | s1.send(20) 265 | 266 | ch, item = h.select([s1, r2]) 267 | assert ch == r2 268 | assert item == 10 269 | 270 | assert check_r.recv() == 20 271 | assert check_r.recv() == 'done' 272 | 273 | 274 | def test_select_timeout(): 275 | h = vanilla.Hub() 276 | 277 | s1, r1 = h.pipe() 278 | s2, r2 = h.pipe() 279 | check_s, check_r = h.pipe() 280 | 281 | pytest.raises(vanilla.Timeout, h.select, [s1, r2], timeout=0) 282 | 283 | @h.spawn 284 | def _(): 285 | h.sleep(20) 286 | check_s.send(r1.recv()) 287 | 288 | pytest.raises(vanilla.Timeout, h.select, [s1, r2], timeout=10) 289 | 290 | ch, item = h.select([s1, r2], timeout=20) 291 | assert ch == s1 292 | s1.send(20) 293 | assert check_r.recv() == 20 294 | 295 | @h.spawn 296 | def _(): 297 | h.sleep(20) 298 | s2.send(10) 299 | check_s.send('done') 300 | 301 | pytest.raises(vanilla.Timeout, h.select, [s1, r2], timeout=10) 302 | 303 | ch, item = h.select([s1, r2], timeout=20) 304 | assert ch == r2 305 | assert item == 10 306 | assert check_r.recv() == 'done' 307 | 308 | 309 | def test_timeout(): 310 | h = vanilla.Hub() 311 | 312 | sender, recver = h.pipe() 313 | check_sender, check_recver = h.pipe() 314 | 315 | pytest.raises(vanilla.Timeout, sender.send, 12, timeout=0) 316 | pytest.raises(vanilla.Timeout, recver.recv, timeout=0) 317 | pytest.raises(vanilla.Timeout, sender.send, 12, timeout=0) 318 | 319 | @h.spawn 320 | def _(): 321 | h.sleep(20) 322 | check_sender.send(recver.recv()) 323 | 324 | pytest.raises(vanilla.Timeout, sender.send, 12, timeout=10) 325 | sender.send(12, timeout=20) 326 | assert check_recver.recv() == 12 327 | 328 | @h.spawn 329 | def _(): 330 | h.sleep(20) 331 | sender.send(12) 332 | 333 | pytest.raises(vanilla.Timeout, recver.recv, timeout=10) 334 | assert recver.recv(timeout=20) == 12 335 | 336 | 337 | def test_buffer(): 338 | h = vanilla.Hub() 339 | 340 | sender, recver = buffer(h, 2) 341 | 342 | sender.send(1) 343 | sender.send(2) 344 | 345 | assert recver.recv() == 1 346 | assert recver.recv() == 2 347 | -------------------------------------------------------------------------------- /xx/stream.py: -------------------------------------------------------------------------------- 1 | import vanilla 2 | 3 | from vanilla import * 4 | 5 | import weakref 6 | import gc 7 | 8 | 9 | class Pair(object): 10 | __slots__ = ['hub', 'current', 'pair'] 11 | 12 | def __init__(self, hub): 13 | self.hub = hub 14 | self.current = None 15 | self.pair = None 16 | 17 | def yack(self, *a, **kw): 18 | print "YAKCK UAKC", a, kw, self.__class__ 19 | 20 | def pair_to(self, pair): 21 | self.pair = weakref.ref(pair, self.yack) 22 | 23 | 24 | class Sender(Pair): 25 | def send(self, item): 26 | if self.pair().current: 27 | current = self.pair().current 28 | 29 | else: 30 | self.current = getcurrent() 31 | current = self.hub.pause() 32 | self.current = None 33 | 34 | self.hub.switch_to(current, item) 35 | 36 | 37 | class Recver(Pair): 38 | def recv(self): 39 | if self.pair().current: 40 | return self.hub.switch_to(self.pair().current, getcurrent()) 41 | 42 | self.current = getcurrent() 43 | item = self.hub.pause() 44 | self.current = None 45 | return item 46 | 47 | 48 | def pipe(hub): 49 | sender = Sender(hub) 50 | recver = Recver(hub) 51 | 52 | sender.pair_to(recver) 53 | recver.pair_to(sender) 54 | 55 | return sender, recver 56 | 57 | 58 | def stream(hub): 59 | def dec(f): 60 | sender, recver = pipe(hub) 61 | hub.spawn(f, sender) 62 | return recver 63 | return dec 64 | 65 | 66 | def test_stream(): 67 | h = vanilla.Hub() 68 | 69 | @stream(h) 70 | def counter(out): 71 | for i in xrange(10): 72 | print "s", i 73 | out.send(i) 74 | 75 | print 76 | print 77 | print "-----" 78 | print counter.recv() 79 | print 80 | h.sleep(10) 81 | print "a" 82 | print counter.recv() 83 | print "b" 84 | 85 | del counter 86 | gc.collect() 87 | --------------------------------------------------------------------------------