├── .gitignore ├── .travis.yml ├── CHANGES.rst ├── CODE_OF_CONDUCT.md ├── MANIFEST.in ├── Makefile ├── NOTES ├── README.rst ├── docs ├── Makefile ├── make.bat └── source │ ├── .DS_Store │ ├── _static │ └── .empty │ ├── _templates │ └── indexsidebar.html │ ├── _themes │ └── bootstrap.zip │ ├── commands.rst │ ├── conf.py │ ├── examples.rst │ ├── images │ ├── diag.svg │ ├── medium-powerhose.png │ └── powerhose.png │ ├── index.rst │ ├── installation.rst │ ├── library.rst │ └── why.rst ├── examples ├── bench.py ├── crypto.ini ├── crypto.py ├── crypto_client.py ├── crypto_server.py ├── crypto_worker.py ├── echo.ini ├── echo.py ├── echo_client.py ├── echo_worker.cpp ├── echo_worker.py ├── web.ini ├── web_client.py └── web_worker.py ├── libhose ├── Worker.cpp ├── Worker.h ├── util.cpp └── util.h ├── pip-requirements.txt ├── powerhose ├── __init__.py ├── broker.py ├── client.py ├── exc.py ├── heartbeat.py ├── job.py ├── tests │ ├── __init__.py │ ├── jobs.py │ ├── test_cluster.py │ ├── test_heartbeat.py │ └── test_job.py ├── util.py └── worker.py ├── setup.py └── tox.ini /.gitignore: -------------------------------------------------------------------------------- 1 | *.o 2 | *.a 3 | *.pyc 4 | *.egg-info 5 | .Python 6 | bin 7 | lib 8 | include 9 | man 10 | .tox 11 | .makepp 12 | examples/echo_worker 13 | core 14 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "2.6" 4 | - "2.7" 5 | install: python setup.py develop 6 | script: python setup.py test 7 | notifications: 8 | email: 9 | - tarek@mozilla.com 10 | -------------------------------------------------------------------------------- /CHANGES.rst: -------------------------------------------------------------------------------- 1 | CHANGES 2 | ======= 3 | 4 | 0.8 5 | 6 | 0.7 - 2012-06-21 7 | ---------------- 8 | 9 | - auto-unregistring of dead/slow workers 10 | 11 | 12 | 0.6 - 2012-06-20 13 | ---------------- 14 | 15 | - introduced a graceful shutdown 16 | - added a worker registering so restarting workers are not 17 | impacting the system 18 | 19 | 0.5 - 2012-06-12 20 | ---------------- 21 | 22 | - drastically reduced the number of used FDs, mainly by 23 | reusing the same context when possible 24 | 25 | 26 | 0.4 - 2012-05-25 27 | ---------------- 28 | 29 | - the broker exits if there's already a valid broker running 30 | in the socket. 31 | - powerhose-broker gained 2 new options: --check and --purge-ghosts 32 | 33 | 34 | 0.3 - 2012-05-24 35 | ---------------- 36 | 37 | - implemented timeout_max_overflow in the client. 38 | - the stacks are dumped on worker timeouts 39 | - now using delayed callbacks for the heartbeat 40 | 41 | 0.2 - 2012-04-17 42 | ---------------- 43 | 44 | - make sure execution errors are properly transmited and raised. 45 | - fixed the pool of connectors - so every connector is correctly freed 46 | - now workers can get extra options from the command-line 47 | 48 | 0.1 - 2012-04-05 49 | ---------------- 50 | 51 | - initial release. 52 | 53 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Community Participation Guidelines 2 | 3 | This repository is governed by Mozilla's code of conduct and etiquette guidelines. 4 | For more details, please read the 5 | [Mozilla Community Participation Guidelines](https://www.mozilla.org/about/governance/policies/participation/). 6 | 7 | ## How to Report 8 | For more information on how to report violations of the Community Participation Guidelines, please read our '[How to Report](https://www.mozilla.org/about/governance/policies/participation/reporting/)' page. 9 | 10 | 16 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.rst 2 | include *.txt 3 | include MANIFEST.in 4 | recursive-include examples *.py 5 | recursive-include examples *.ini 6 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | LIBFILES = Worker.cpp util.cpp 2 | LIBOFILES = $(LIBFILES:%.cpp=%.o) 3 | DIR = $(CURDIR)/libhose 4 | INCLUDEDIR = -I$(DIR) -I/opt/local/include 5 | LIBDIR = -L$(DIR) -L/opt/local/lib 6 | OPTS = -g -Wall -Wextra -Wfatal-errors 7 | LIBS = -lpthread -lzmq -lprotobuf 8 | 9 | .PHONY: docs all build buildex proto 10 | 11 | 12 | all: build buildex 13 | 14 | build: 15 | cd $(DIR); rm -f libhose.a; rm -f *.o 16 | cd $(DIR); g++ $(OPTS) $(INCLUDEDIR) $(LIBDIR) -c $(LIBFILES) 17 | cd $(DIR); ar cq libhose.a $(LIBOFILES) 18 | 19 | buildex: 20 | cd examples; rm -f echo_worker 21 | cd examples; g++ -o echo_worker echo_worker.cpp -lhose $(OPTS) $(INCLUDEDIR) $(LIBDIR) $(LIBS) 22 | 23 | test: bin/nosetests 24 | bin/nosetests -s powerhose 25 | 26 | coverage: bin/coverage 27 | bin/nosetests --with-coverage --cover-html --cover-html-dir=html --cover-package=powerhose 28 | 29 | docs: bin/sphinx-build 30 | SPHINXBUILD=../bin/sphinx-build $(MAKE) -C docs html $^ 31 | -------------------------------------------------------------------------------- /NOTES: -------------------------------------------------------------------------------- 1 | 2 | if the worker dies 3 | - tell the master if it's not a -9 4 | - something external can decide to respawn it (daemontools) 5 | 6 | 7 | if the master dies. 8 | - the worker have an heartbeat, see that the master is dead, try to reconnect for 5s then exit 9 | 10 | 11 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ========= 2 | Powerhose 3 | ========= 4 | 5 | Powerhose turns your CPU-bound tasks into I/O-bound tasks so your Python applications 6 | are easier to scale. 7 | 8 | .. image:: https://secure.travis-ci.org/mozilla-services/powerhose.png?branch=master 9 | :alt: Build Status 10 | :target: https://secure.travis-ci.org/mozilla-services/powerhose 11 | 12 | 13 | Powerhose is an implementation of the 14 | `Request-Reply Broker `_ 15 | pattern in ZMQ. 16 | 17 | See http://powerhose.readthedocs.org for a full documentation. 18 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | ifndef SPHINXBUILD 7 | SPHINXBUILD = sphinx-build 8 | endif 9 | PAPER = 10 | BUILDDIR = build 11 | 12 | # Internal variables. 13 | PAPEROPT_a4 = -D latex_paper_size=a4 14 | PAPEROPT_letter = -D latex_paper_size=letter 15 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 16 | # the i18n builder cannot share the environment and doctrees with the others 17 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 18 | 19 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 20 | 21 | help: 22 | @echo "Please use \`make ' where is one of" 23 | @echo " html to make standalone HTML files" 24 | @echo " dirhtml to make HTML files named index.html in directories" 25 | @echo " singlehtml to make a single large HTML file" 26 | @echo " pickle to make pickle files" 27 | @echo " json to make JSON files" 28 | @echo " htmlhelp to make HTML files and a HTML help project" 29 | @echo " qthelp to make HTML files and a qthelp project" 30 | @echo " devhelp to make HTML files and a Devhelp project" 31 | @echo " epub to make an epub" 32 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 33 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 34 | @echo " text to make text files" 35 | @echo " man to make manual pages" 36 | @echo " texinfo to make Texinfo files" 37 | @echo " info to make Texinfo files and run them through makeinfo" 38 | @echo " gettext to make PO message catalogs" 39 | @echo " changes to make an overview of all changed/added/deprecated items" 40 | @echo " linkcheck to check all external links for integrity" 41 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 42 | 43 | clean: 44 | -rm -rf $(BUILDDIR)/* 45 | 46 | html: 47 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 48 | @echo 49 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 50 | 51 | dirhtml: 52 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 53 | @echo 54 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 55 | 56 | singlehtml: 57 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 58 | @echo 59 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 60 | 61 | pickle: 62 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 63 | @echo 64 | @echo "Build finished; now you can process the pickle files." 65 | 66 | json: 67 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 68 | @echo 69 | @echo "Build finished; now you can process the JSON files." 70 | 71 | htmlhelp: 72 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 73 | @echo 74 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 75 | ".hhp project file in $(BUILDDIR)/htmlhelp." 76 | 77 | qthelp: 78 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 79 | @echo 80 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 81 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 82 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Circus.qhcp" 83 | @echo "To view the help file:" 84 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Circus.qhc" 85 | 86 | devhelp: 87 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 88 | @echo 89 | @echo "Build finished." 90 | @echo "To view the help file:" 91 | @echo "# mkdir -p $$HOME/.local/share/devhelp/Circus" 92 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Circus" 93 | @echo "# devhelp" 94 | 95 | epub: 96 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 97 | @echo 98 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 99 | 100 | latex: 101 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 102 | @echo 103 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 104 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 105 | "(use \`make latexpdf' here to do that automatically)." 106 | 107 | latexpdf: 108 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 109 | @echo "Running LaTeX files through pdflatex..." 110 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 111 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 112 | 113 | text: 114 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 115 | @echo 116 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 117 | 118 | man: 119 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 120 | @echo 121 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 122 | 123 | texinfo: 124 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 125 | @echo 126 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 127 | @echo "Run \`make' in that directory to run these through makeinfo" \ 128 | "(use \`make info' here to do that automatically)." 129 | 130 | info: 131 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 132 | @echo "Running Texinfo files through makeinfo..." 133 | make -C $(BUILDDIR)/texinfo info 134 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 135 | 136 | gettext: 137 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 138 | @echo 139 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 140 | 141 | changes: 142 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 143 | @echo 144 | @echo "The overview file is in $(BUILDDIR)/changes." 145 | 146 | linkcheck: 147 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 148 | @echo 149 | @echo "Link check complete; look for any errors in the above output " \ 150 | "or in $(BUILDDIR)/linkcheck/output.txt." 151 | 152 | doctest: 153 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 154 | @echo "Testing of doctests in the sources finished, look at the " \ 155 | "results in $(BUILDDIR)/doctest/output.txt." 156 | 157 | # Need it to avoid error 158 | bin/sphinx-build: 159 | @echo "Doc updated" 160 | 161 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% source 10 | set I18NSPHINXOPTS=%SPHINXOPTS% source 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. linkcheck to check all external links for integrity 37 | echo. doctest to run all doctests embedded in the documentation if enabled 38 | goto end 39 | ) 40 | 41 | if "%1" == "clean" ( 42 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 43 | del /q /s %BUILDDIR%\* 44 | goto end 45 | ) 46 | 47 | if "%1" == "html" ( 48 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 49 | if errorlevel 1 exit /b 1 50 | echo. 51 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 52 | goto end 53 | ) 54 | 55 | if "%1" == "dirhtml" ( 56 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 57 | if errorlevel 1 exit /b 1 58 | echo. 59 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 60 | goto end 61 | ) 62 | 63 | if "%1" == "singlehtml" ( 64 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 65 | if errorlevel 1 exit /b 1 66 | echo. 67 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 68 | goto end 69 | ) 70 | 71 | if "%1" == "pickle" ( 72 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 73 | if errorlevel 1 exit /b 1 74 | echo. 75 | echo.Build finished; now you can process the pickle files. 76 | goto end 77 | ) 78 | 79 | if "%1" == "json" ( 80 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 81 | if errorlevel 1 exit /b 1 82 | echo. 83 | echo.Build finished; now you can process the JSON files. 84 | goto end 85 | ) 86 | 87 | if "%1" == "htmlhelp" ( 88 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 89 | if errorlevel 1 exit /b 1 90 | echo. 91 | echo.Build finished; now you can run HTML Help Workshop with the ^ 92 | .hhp project file in %BUILDDIR%/htmlhelp. 93 | goto end 94 | ) 95 | 96 | if "%1" == "qthelp" ( 97 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 98 | if errorlevel 1 exit /b 1 99 | echo. 100 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 101 | .qhcp project file in %BUILDDIR%/qthelp, like this: 102 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\Circus.qhcp 103 | echo.To view the help file: 104 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\Circus.ghc 105 | goto end 106 | ) 107 | 108 | if "%1" == "devhelp" ( 109 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 110 | if errorlevel 1 exit /b 1 111 | echo. 112 | echo.Build finished. 113 | goto end 114 | ) 115 | 116 | if "%1" == "epub" ( 117 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 118 | if errorlevel 1 exit /b 1 119 | echo. 120 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 121 | goto end 122 | ) 123 | 124 | if "%1" == "latex" ( 125 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 129 | goto end 130 | ) 131 | 132 | if "%1" == "text" ( 133 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 134 | if errorlevel 1 exit /b 1 135 | echo. 136 | echo.Build finished. The text files are in %BUILDDIR%/text. 137 | goto end 138 | ) 139 | 140 | if "%1" == "man" ( 141 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 142 | if errorlevel 1 exit /b 1 143 | echo. 144 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 145 | goto end 146 | ) 147 | 148 | if "%1" == "texinfo" ( 149 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 150 | if errorlevel 1 exit /b 1 151 | echo. 152 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 153 | goto end 154 | ) 155 | 156 | if "%1" == "gettext" ( 157 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 158 | if errorlevel 1 exit /b 1 159 | echo. 160 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 161 | goto end 162 | ) 163 | 164 | if "%1" == "changes" ( 165 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 166 | if errorlevel 1 exit /b 1 167 | echo. 168 | echo.The overview file is in %BUILDDIR%/changes. 169 | goto end 170 | ) 171 | 172 | if "%1" == "linkcheck" ( 173 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 174 | if errorlevel 1 exit /b 1 175 | echo. 176 | echo.Link check complete; look for any errors in the above output ^ 177 | or in %BUILDDIR%/linkcheck/output.txt. 178 | goto end 179 | ) 180 | 181 | if "%1" == "doctest" ( 182 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 183 | if errorlevel 1 exit /b 1 184 | echo. 185 | echo.Testing of doctests in the sources finished, look at the ^ 186 | results in %BUILDDIR%/doctest/output.txt. 187 | goto end 188 | ) 189 | 190 | :end 191 | -------------------------------------------------------------------------------- /docs/source/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/powerhose/3dfb0b4fa39ab2a0fb99af411d434ccb85b605ff/docs/source/.DS_Store -------------------------------------------------------------------------------- /docs/source/_static/.empty: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/powerhose/3dfb0b4fa39ab2a0fb99af411d434ccb85b605ff/docs/source/_static/.empty -------------------------------------------------------------------------------- /docs/source/_templates/indexsidebar.html: -------------------------------------------------------------------------------- 1 |
2 |

Feedback

3 | 6 |
7 | -------------------------------------------------------------------------------- /docs/source/_themes/bootstrap.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/powerhose/3dfb0b4fa39ab2a0fb99af411d434ccb85b605ff/docs/source/_themes/bootstrap.zip -------------------------------------------------------------------------------- /docs/source/commands.rst: -------------------------------------------------------------------------------- 1 | .. _commands: 2 | 3 | Command-line tools 4 | ================== 5 | 6 | Powerhose comes with three commands: 7 | 8 | - **powerhose-broker**: runs a broker. 9 | - **powerhose-worker**: runs a *python* worker. 10 | - **powerhose**: runs a broker *and* some workers. 11 | 12 | To run a Powerhose cluster, you need to run a broker and 13 | some workers. So you would use **powerhose** *or* 14 | **powerhose-broker** and some **powerhose-worker**. 15 | 16 | powerhose-broker 17 | ---------------- 18 | 19 | Runs a Powerhose broker:: 20 | 21 | $ powerhose-broker --help 22 | usage: powerhose-broker [-h] [--frontend FRONTEND] [--backend BACKEND] 23 | [--heartbeat HEARTBEAT] [--debug] [--check] 24 | [--purge-ghosts] [--logfile LOGFILE] 25 | 26 | Powerhose broker. 27 | 28 | optional arguments: 29 | -h, --help show this help message and exit 30 | --frontend FRONTEND ZMQ socket to receive jobs. 31 | --backend BACKEND ZMQ socket for workers. 32 | --heartbeat HEARTBEAT 33 | ZMQ socket for the heartbeat. 34 | --debug Debug mode 35 | --check Use this option to check if there's a running broker. 36 | Returns the PID if a broker is up. 37 | --purge-ghosts Use this option to purge ghost brokers. 38 | --logfile LOGFILE File to log in to . 39 | 40 | 41 | **--check** and **--purge-ghosts** are maintenance option that are not running 42 | a broker but just checking an existing broker:: 43 | 44 | $ powerhose-broker --check 45 | [2012-05-25 11:11:28,282][powerhose] A broker is running. PID: 11668 46 | 47 | $ bin/powerhose-broker --purge-ghosts 48 | [2012-05-25 11:12:09,744][powerhose] The active broker runs at PID: 11668 49 | [2012-05-25 11:12:09,744][powerhose] No ghosts where killed. 50 | 51 | Those options can be used to health check and monitor the broker, which 52 | is the stable node of the Powerhose architecture. 53 | 54 | 55 | powerhose-worker 56 | ---------------- 57 | 58 | Runs one worker. 59 | 60 | :: 61 | 62 | $ powerhose-worker --help 63 | usage: powerhose-worker [-h] [--backend BACKEND] [--debug] [--logfile LOGFILE] 64 | [--heartbeat HEARTBEAT] [--params PARAMS] 65 | [--timeout TIMEOUT] [--max-age MAX_AGE] 66 | [--max-age-delta MAX_AGE_DELTA] 67 | target 68 | 69 | Runs a worker. 70 | 71 | positional arguments: 72 | target Fully qualified name of the callable. 73 | 74 | optional arguments: 75 | -h, --help show this help message and exit 76 | --backend BACKEND ZMQ socket to the broker. 77 | --debug Debug mode 78 | --logfile LOGFILE File to log in to. 79 | --heartbeat HEARTBEAT 80 | ZMQ socket for the heartbeat. 81 | --params PARAMS The parameters to be used in the worker. 82 | --timeout TIMEOUT The maximum time allowed before the thread stacks is 83 | dump and the job result not sent back. 84 | --max-age MAX_AGE The maximum age for a worker in seconds. After that 85 | delay, the worker will simply quit. When set to -1, 86 | never quits. 87 | --max-age-delta MAX_AGE_DELTA 88 | The maximum value in seconds added to max_age 89 | 90 | 91 | The **--max-age** option is useful when you want your worker to exit after 92 | some time. The typical use case is when you have a program that keeps some 93 | connectors open on some external ressources, and those ressources change over 94 | time. 95 | 96 | 97 | powerhose 98 | --------- 99 | 100 | Runs one broker and several workers by using the two previous commands. 101 | 102 | :: 103 | 104 | $ powerhose --help 105 | usage: powerhose [-h] [--frontend FRONTEND] [--backend BACKEND] 106 | [--heartbeat HEARTBEAT] [--debug] 107 | [--numprocesses NUMPROCESSES] [--logfile LOGFILE] 108 | target 109 | 110 | Run a Powerhose cluster. 111 | 112 | positional arguments: 113 | target Fully qualified name of the callable. 114 | 115 | optional arguments: 116 | -h, --help show this help message and exit 117 | --frontend FRONTEND ZMQ socket to receive jobs. 118 | --backend BACKEND ZMQ socket for workers. 119 | --heartbeat HEARTBEAT 120 | ZMQ socket for the heartbeat. 121 | --debug Debug mode 122 | --numprocesses NUMPROCESSES 123 | Number of processes to run. 124 | --logfile LOGFILE File to log in to . 125 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Powerhose documentation build configuration file, created by 4 | # sphinx-quickstart on Fri Feb 24 15:30:44 2012. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | import sys, os 15 | 16 | 17 | class Mock(object): 18 | def __init__(self, *args, **kwargs): 19 | pass 20 | 21 | def __call__(self, *args, **kwargs): 22 | return Mock() 23 | 24 | @classmethod 25 | def __getattr__(self, name): 26 | if name in ('__file__', '__path__'): 27 | return '/dev/null' 28 | elif name[0] == name[0].upper(): 29 | return type(name, (), {}) 30 | else: 31 | return Mock() 32 | 33 | MOCK_MODULES = ['zmq', 'zmq.eventloop'] 34 | for mod_name in MOCK_MODULES: 35 | sys.modules[mod_name] = Mock() 36 | 37 | # If extensions (or modules to document with autodoc) are in another directory, 38 | # add these directories to sys.path here. If the directory is relative to the 39 | # documentation root, use os.path.abspath to make it absolute, like shown here. 40 | #sys.path.insert(0, os.path.abspath('.')) 41 | 42 | # -- General configuration ----------------------------------------------------- 43 | 44 | # If your documentation needs a minimal Sphinx version, state it here. 45 | #needs_sphinx = '1.0' 46 | 47 | # Add any Sphinx extension module names here, as strings. They can be extensions 48 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 49 | 50 | CURDIR = os.path.abspath(os.path.dirname(__file__)) 51 | sys.path.append(os.path.join(CURDIR, '..', '..')) 52 | sys.path.append(os.path.join(CURDIR, '..')) 53 | 54 | extensions = ['sphinx.ext.autodoc' ]#, 'sphinxcontrib.blockdiag'] 55 | 56 | # Add any paths that contain templates here, relative to this directory. 57 | templates_path = ['_templates'] 58 | 59 | # The suffix of source filenames. 60 | source_suffix = '.rst' 61 | 62 | # The encoding of source files. 63 | #source_encoding = 'utf-8-sig' 64 | 65 | # The master toctree document. 66 | master_doc = 'index' 67 | 68 | # General information about the project. 69 | project = u'Powerhose' 70 | copyright = u'2012, Mozilla Foundation - 2012' 71 | 72 | # The version info for the project you're documenting, acts as replacement for 73 | # |version| and |release|, also used in various other places throughout the 74 | # built documents. 75 | # 76 | # The short X.Y version. 77 | version = '0.4' 78 | # The full version, including alpha/beta/rc tags. 79 | release = '0.4' 80 | 81 | # The language for content autogenerated by Sphinx. Refer to documentation 82 | # for a list of supported languages. 83 | #language = None 84 | 85 | # There are two options for replacing |today|: either, you set today to some 86 | # non-false value, then it is used: 87 | #today = '' 88 | # Else, today_fmt is used as the format for a strftime call. 89 | #today_fmt = '%B %d, %Y' 90 | 91 | # List of patterns, relative to source directory, that match files and 92 | # directories to ignore when looking for source files. 93 | exclude_patterns = [] 94 | 95 | # The reST default role (used for this markup: `text`) to use for all documents. 96 | #default_role = None 97 | 98 | # If true, '()' will be appended to :func: etc. cross-reference text. 99 | #add_function_parentheses = True 100 | 101 | # If true, the current module name will be prepended to all description 102 | # unit titles (such as .. function::). 103 | #add_module_names = True 104 | 105 | # If true, sectionauthor and moduleauthor directives will be shown in the 106 | # output. They are ignored by default. 107 | #show_authors = False 108 | 109 | # The name of the Pygments (syntax highlighting) style to use. 110 | pygments_style = 'sphinx' 111 | 112 | # A list of ignored prefixes for module index sorting. 113 | #modindex_common_prefix = [] 114 | 115 | 116 | # -- Options for HTML output --------------------------------------------------- 117 | 118 | # The theme to use for HTML and HTML Help pages. See the documentation for 119 | # a list of builtin themes. 120 | sys.path.append(os.path.abspath('_themes')) 121 | html_theme_path = ['_themes'] 122 | html_short_title = "Powerhose" 123 | on_rtd = os.environ.get('READTHEDOCS', None) == 'True' 124 | #if on_rtd: 125 | html_theme = 'default' 126 | #else: 127 | # html_theme = 'bootstrap' 128 | #html_logo = "images/powerhose32.png" 129 | 130 | # Theme options are theme-specific and customize the look and feel of a theme 131 | # further. For a list of options available for each theme, see the 132 | # documentation. 133 | #html_theme_options = {} 134 | 135 | # Add any paths that contain custom themes here, relative to this directory. 136 | #html_theme_path = [] 137 | 138 | # The name for this set of Sphinx documents. If None, it defaults to 139 | # " v documentation". 140 | #html_title = None 141 | 142 | # A shorter title for the navigation bar. Default is the same as html_title. 143 | #html_short_title = None 144 | 145 | # The name of an image file (relative to this directory) to place at the top 146 | # of the sidebar. 147 | #html_logo = None 148 | 149 | # The name of an image file (within the static path) to use as favicon of the 150 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 151 | # pixels large. 152 | #html_favicon = None 153 | 154 | # Add any paths that contain custom static files (such as style sheets) here, 155 | # relative to this directory. They are copied after the builtin static files, 156 | # so a file named "default.css" will overwrite the builtin "default.css". 157 | html_static_path = ['_static'] 158 | 159 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 160 | # using the given strftime format. 161 | #html_last_updated_fmt = '%b %d, %Y' 162 | 163 | # If true, SmartyPants will be used to convert quotes and dashes to 164 | # typographically correct entities. 165 | #html_use_smartypants = True 166 | 167 | # Custom sidebar templates, maps document names to template names. 168 | #html_sidebars = {} 169 | 170 | CURDIR = os.path.dirname(__file__) 171 | sidebars = [] 172 | for f in os.listdir(CURDIR): 173 | name, ext = os.path.splitext(f) 174 | if ext != '.rst': 175 | continue 176 | sidebars.append((name, 'indexsidebar.html')) 177 | 178 | html_sidebars = dict(sidebars) 179 | 180 | # Additional templates that should be rendered to pages, maps page names to 181 | # template names. 182 | #html_additional_pages = {} 183 | 184 | # If false, no module index is generated. 185 | #html_domain_indices = True 186 | 187 | # If false, no index is generated. 188 | #html_use_index = True 189 | 190 | # If true, the index is split into individual pages for each letter. 191 | #html_split_index = False 192 | 193 | # If true, links to the reST sources are added to the pages. 194 | #html_show_sourcelink = True 195 | 196 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 197 | #html_show_sphinx = True 198 | 199 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 200 | #html_show_copyright = True 201 | 202 | # If true, an OpenSearch description file will be output, and all pages will 203 | # contain a tag referring to it. The value of this option must be the 204 | # base URL from which the finished HTML is served. 205 | #html_use_opensearch = '' 206 | 207 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 208 | #html_file_suffix = None 209 | 210 | # Output file base name for HTML help builder. 211 | htmlhelp_basename = 'Powerhosedoc' 212 | 213 | 214 | # -- Options for LaTeX output -------------------------------------------------- 215 | 216 | latex_elements = { 217 | # The paper size ('letterpaper' or 'a4paper'). 218 | #'papersize': 'letterpaper', 219 | 220 | # The font size ('10pt', '11pt' or '12pt'). 221 | #'pointsize': '10pt', 222 | 223 | # Additional stuff for the LaTeX preamble. 224 | #'preamble': '', 225 | } 226 | 227 | # Grouping the document tree into LaTeX files. List of tuples 228 | # (source start file, target name, title, author, documentclass [howto/manual]). 229 | latex_documents = [ 230 | ('index', 'Powerhose.tex', u'Powerhose Documentation', 231 | u'Mozilla Foundation', 'manual'), 232 | ] 233 | 234 | # The name of an image file (relative to this directory) to place at the top of 235 | # the title page. 236 | #latex_logo = None 237 | 238 | # For "manual" documents, if this is true, then toplevel headings are parts, 239 | # not chapters. 240 | #latex_use_parts = False 241 | 242 | # If true, show page references after internal links. 243 | #latex_show_pagerefs = False 244 | 245 | # If true, show URL addresses after external links. 246 | #latex_show_urls = False 247 | 248 | # Documents to append as an appendix to all manuals. 249 | #latex_appendices = [] 250 | 251 | # If false, no module index is generated. 252 | #latex_domain_indices = True 253 | 254 | 255 | # -- Options for manual page output -------------------------------------------- 256 | 257 | # One entry per manual page. List of tuples 258 | # (source start file, name, description, authors, manual section). 259 | man_pages = [ 260 | ('index', 'powerhose', u'Powerhose Documentation', 261 | [u'Mozilla Foundation',], 1) 262 | ] 263 | 264 | # If true, show URL addresses after external links. 265 | #man_show_urls = False 266 | 267 | 268 | # -- Options for Texinfo output ------------------------------------------------ 269 | 270 | # Grouping the document tree into Texinfo files. List of tuples 271 | # (source start file, target name, title, author, 272 | # dir menu entry, description, category) 273 | texinfo_documents = [ 274 | ('index', 'Powerhose', u'Powerhose Documentation', 275 | u'Mozilla Foundation', 'Powerhose', 'One line description of project.', 276 | 'Miscellaneous'), 277 | ] 278 | 279 | # Documents to append as an appendix to all manuals. 280 | #texinfo_appendices = [] 281 | 282 | # If false, no module index is generated. 283 | #texinfo_domain_indices = True 284 | 285 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 286 | #texinfo_show_urls = 'footnote' 287 | -------------------------------------------------------------------------------- /docs/source/examples.rst: -------------------------------------------------------------------------------- 1 | .. examples: 2 | 3 | Examples 4 | ======== 5 | 6 | **This section will be added later** 7 | -------------------------------------------------------------------------------- /docs/source/images/diag.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | blockdiag 10 | 11 | 12 | 13 | 14 | 15 | Client 16 | 17 | 1 18 | 19 | Broker 20 | 21 | 2 22 | 23 | Worker 1 24 | 25 | 26 | 27 | 28 | Worker N 29 | 30 | 3 31 | 32 | 33 | 34 | 35 | front 36 | 37 | 38 | 39 | 40 | back 41 | 42 | 43 | 44 | 45 | 46 | 47 | back 48 | -------------------------------------------------------------------------------- /docs/source/images/medium-powerhose.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/powerhose/3dfb0b4fa39ab2a0fb99af411d434ccb85b605ff/docs/source/images/medium-powerhose.png -------------------------------------------------------------------------------- /docs/source/images/powerhose.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/powerhose/3dfb0b4fa39ab2a0fb99af411d434ccb85b605ff/docs/source/images/powerhose.png -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | ========= 2 | Powerhose 3 | ========= 4 | 5 | .. note:: 6 | 7 | This is still a work in progress - no stable version has 8 | been released yet. 9 | 10 | 11 | .. image:: images/medium-powerhose.png 12 | :align: right 13 | 14 | **Powerhose turns your CPU-bound tasks into I/O-bound tasks so your Python 15 | applications are easier to scale.** 16 | 17 | Powerhose is an implementation of the 18 | `Request-Reply Broker `_ 19 | pattern in ZMQ. 20 | 21 | .. image:: images/diag.svg 22 | :width: 80 % 23 | 24 | .. Commented out for now blockdiag 25 | 26 | diagram { 27 | default_fontsize = 20; 28 | 29 | Client [color = "#FEF1E1", numbered=1]; 30 | Broker [color = "#6495ED", numbered=2]; 31 | Worker1 [label = "Worker 1", color = "#FF7F50"]; 32 | Worker2 [shape="dots"]; 33 | Worker3 [label = "Worker N", color = "#FF7F50", numbered=3]; 34 | Client <-> Broker [label="front"]; 35 | Broker <-> Worker2 [style="none"]; 36 | Broker <-> Worker1, Worker2, Worker3 [label="back"]; 37 | } 38 | 39 | The three main parts are: 40 | 41 | 1. a client that connects to a broker to send some jobs. 42 | 43 | 2. a broker that binds a socket to get some job (*"front"*) from clients, 44 | and another socket for workers (*"back"*) to connect. 45 | 46 | 3. workers that connect to the "back" socket of the broker, receive jobs 47 | and send back results. 48 | 49 | 50 | When client sends a job, the broker simply re-routes it to one of its 51 | workers, then gets back the result and send it back to the client. 52 | 53 | Workers also subscribe to a *"heartbeat"* socket that receives regular 54 | heart beats from the broker. When the beat is lost, workers simply die. 55 | 56 | Powerhose uses `Circus `_ to manage the 57 | life of the broker and the workers, and provide high level APIs for your 58 | program. 59 | 60 | Workers can be written in Python but also in any language that has a 61 | ZeroMQ library. 62 | 63 | If you have CPU-Bound tasks that could be performed in a specialized C++ 64 | program for example, Powerhose is a library you could use to ease your 65 | life. 66 | 67 | .. note:: 68 | 69 | Before using Powerhose, ask yourself if you really need it. 70 | 71 | The overhead of the data exchange can be quite important, so unless 72 | your job is a CPU-bound task that takes more than 20 ms to perform, 73 | there are high chances using Powerhose will not speed it up. 74 | 75 | There's a :file:`bench.py` module in the examples you can change 76 | to make some tests with your code. It will compare the speed with 77 | and without Powerhose in a multi-threaded environment so you can 78 | see if you get any benefit. 79 | 80 | 81 | If you are curious about why we wrote this library see :ref:`why`. 82 | 83 | 84 | Example 85 | ======= 86 | 87 | Here's a full example of usage -- we want to delegate some work 88 | to a specialized worker. 89 | 90 | Let's create a function that just echoes back what was sent to it, 91 | and save it in an :file:`example` module:: 92 | 93 | def echo(job): 94 | return job.data 95 | 96 | 97 | This function takes a :class:`Job` instance that contains the data sent by 98 | Powerhose. It returns its content immediately. 99 | 100 | Let's run our Powerhose cluster with the *powerhose* command, by simply 101 | pointing the :func:`echo` callable:: 102 | 103 | $ powerhose example.echo 104 | [circus] Starting master on pid 51177 105 | [circus] broker started 106 | [circus] workers started 107 | [circus] Arbiter now waiting for commands 108 | 109 | 110 | That's it ! By default one broker and 5 workers are launched, but you can 111 | run as many workers as you need, and even add more while the system is 112 | running. 113 | 114 | Now that the system is up and running, let's try it out in a Python 115 | shell:: 116 | 117 | >>> from powerhose.client import Client 118 | >>> client = Client() 119 | >>> client.execute('test') 120 | 'test' 121 | 122 | 123 | Congrats ! You have a Powerhose system up and running ! 124 | 125 | To learn about all existing commands and their options, see :ref:`commands`. 126 | 127 | 128 | Running Powerhose with Circus 129 | ============================= 130 | 131 | Of course, the goal is to keep the broker and its workers up and running 132 | on a system. You can use Daemontools, Supervisord or Circus. 133 | 134 | Circus is our preferred system. A Circus config can look like this:: 135 | 136 | [circus] 137 | check_delay = 5 138 | endpoint = tcp://127.0.0.1:5555 139 | 140 | [watcher:master] 141 | cmd = powerhose-broker 142 | args = --frontend ipc:///tmp/front --backend ipc:///tmp/backend --heartbeat ipc:///tmp/heartbeat 143 | warmup_delay = 0 144 | numprocesses = 1 145 | 146 | [watcher:workers] 147 | cmd = powerhose-worker 148 | args = --backend ipc:///tmp/backend --heartbeat ipc:///tmp/heartbeat echo_worker.echo 149 | warmup_delay = 0 150 | numprocesses = 5 151 | 152 | 153 | This file can then be launched via **circusd**. See the Circus documentation 154 | for details on this. 155 | 156 | 157 | Using Powerhose programmaticaly 158 | =============================== 159 | 160 | The simplest way to run a Powerhose system is to use the command line as 161 | previously shown, but you can also do everything programmatically via 162 | the :func:`get_cluster` function. 163 | 164 | Here's an example:: 165 | 166 | from powerhose import get_cluster 167 | from powerhose.client import Client 168 | 169 | 170 | cluster = get_cluster('example.echo', background=True) 171 | cluster.start() 172 | 173 | client = Client() 174 | 175 | for i in range(10): 176 | print client.execute(str(i)) 177 | 178 | cluster.stop() 179 | 180 | 181 | Here, the cluster is launched programmatically in the background and the 182 | client uses it as before. 183 | 184 | To learn more about Powerhose APIs, see :ref:`library`. 185 | 186 | 187 | 188 | More documentation 189 | ================== 190 | 191 | .. toctree:: 192 | :maxdepth: 2 193 | 194 | installation 195 | commands 196 | library 197 | examples 198 | why 199 | 200 | Contributions and Feedback 201 | ========================== 202 | 203 | You can reach us for any feedback, bug report, or to contribute, at 204 | https://github.com/mozilla-services/powerhose 205 | 206 | We can also be found in the **#services-dev** channel on irc.mozilla.org. 207 | -------------------------------------------------------------------------------- /docs/source/installation.rst: -------------------------------------------------------------------------------- 1 | .. _installation: 2 | 3 | Installation 4 | ============ 5 | 6 | Use pip:: 7 | 8 | $ pip install powerhose 9 | 10 | Or download the archive on PyPI and install it manually with:: 11 | 12 | $ python setup.py install 13 | 14 | If you want to try out Powerhose, see the :ref:`examples`. 15 | -------------------------------------------------------------------------------- /docs/source/library.rst: -------------------------------------------------------------------------------- 1 | .. _library: 2 | 3 | ======= 4 | Library 5 | ======= 6 | 7 | Powerhose is organized into a simple hierarchy of classes and a few functions: 8 | 9 | - :func:`get_cluster` -- a function that create a whole cluster. 10 | 11 | - :class:`Job` -- A class that holds a job to be performed. 12 | 13 | - :class:`Broker` -- A class that pass the jobs it receives to workers. 14 | 15 | - :class:`Worker` -- A class that connects to a broker and pass jobs it receives 16 | to a Python callable. 17 | 18 | - :class:`Heartbeat` and :class:`Stethoscope` -- implements a heartbeat. 19 | 20 | - :class:`Client` -- A class that connects to a broker and let you run jobs against 21 | it. 22 | 23 | 24 | 25 | get_cluster 26 | =========== 27 | 28 | The :func:`get_cluster` function creates a :class:`Broker` and several 29 | :class:`Worker` instance. It can be run in the background for conveniency. 30 | 31 | 32 | .. autofunction:: powerhose.get_cluster 33 | 34 | 35 | Example:: 36 | 37 | from powerhose import get_cluster 38 | from powerhose.client import Client 39 | 40 | 41 | cluster = get_cluster('example.echo', background=True) 42 | cluster.start() 43 | 44 | client = Client() 45 | 46 | for i in range(10): 47 | print client.execute(str(i)) 48 | 49 | cluster.stop() 50 | 51 | 52 | 53 | 54 | Job 55 | === 56 | 57 | 58 | .. autoclass:: powerhose.job.Job 59 | :members: add_header, serialize, load_from_string 60 | 61 | 62 | Example:: 63 | 64 | >>> from powerhose.job import Job 65 | >>> job = Job('4*2') 66 | >>> job.serialize() 67 | 'NONE:::4*2' 68 | >>> Job.load_from_string('NONE:::4*2') 69 | 70 | >>> Job.load_from_string('NONE:::4*2').data 71 | '4*2' 72 | 73 | 74 | Broker 75 | ====== 76 | 77 | .. autoclass:: powerhose.broker.Broker 78 | :members: start,stop 79 | 80 | 81 | Worker 82 | ====== 83 | 84 | .. autoclass:: powerhose.worker.Worker 85 | :members: start,stop 86 | 87 | 88 | Heartbeat 89 | ========= 90 | 91 | The :class:`Broker` class runs a :class:`Heartbeat` instance that regularly 92 | sends a *BEAT* message on a PUB channel. Each worker has a :class:`Stethoscope` 93 | instance that subscribes to the channel, to check if the :class:`Broker` is still 94 | around. 95 | 96 | .. autoclass:: powerhose.heartbeat.Heartbeat 97 | :members: stop,start 98 | 99 | .. autoclass:: powerhose.heartbeat.Stethoscope 100 | :members: stop,start 101 | 102 | 103 | Client 104 | ====== 105 | 106 | .. autoclass:: powerhose.client.Client 107 | :members: execute, ping 108 | 109 | 110 | Client Pool 111 | =========== 112 | 113 | .. autoclass:: powerhose.client.Pool 114 | :members: execute 115 | -------------------------------------------------------------------------------- /docs/source/why.rst: -------------------------------------------------------------------------------- 1 | .. _why: 2 | 3 | 4 | Why Powerhose ? 5 | =============== 6 | 7 | Python is a great language but as soon as you are doing CPU-bound tasks, 8 | you might bump into the GIL issue if you try to run in parallel multiple 9 | threads. 10 | 11 | The GIL is the *Global Interpreter Lock* used by the CPython & the PyPy 12 | implementations to protect some parts of the language implementation 13 | itself. 14 | 15 | The effect on CPU-bound tasks that are performed by several threads is 16 | that you won't be able to use all your machine CPU cores in parallel 17 | like in other languages. 18 | 19 | To solve this issue, the simplest thing to do is to use **multiprocessing**, 20 | a module that comes with the standard library and will let you 21 | spawn processes and interact with them using pickles. 22 | 23 | But that limits you to using the Python language on both sides. 24 | 25 | You could also use a C++ library binded into Python, but it turns out 26 | you're still locking the GIL here and there when you use a C++ 27 | bind through CPython. Unless you delegate *everything* to the C++ side, 28 | the contention can be smaller but is still there. 29 | 30 | So ideally, we'd want a library where you can delegate some specific 31 | tasks to specialized workers, whatever language they are written into. 32 | 33 | Of course this is feasible with the standard library, but requires 34 | extra work to set up a protocol between the master and the workers, 35 | and decide how to transport the data. 36 | 37 | There's also tools like RabbitMQ that can let you set up a queue 38 | where the master put some job to be performed, workers can pick up. 39 | 40 | But here, we're talking about running specific CPU-Bound jobs 41 | as fast as possible, synchronously, with no persistency at all. 42 | 43 | Our driving use case is Mozilla's Token Server - 44 | https://wiki.mozilla.org/Services/Sagrada/TokenServer 45 | 46 | On this server, we have an API you can call to trade a BrowserID 47 | assertion for a token you can use to authenticate to some of our 48 | services. 49 | 50 | Powerhose is our attempt to solve this issue, and is based 51 | on ZeroMQ. 52 | 53 | We chose ZeroMQ because: 54 | 55 | - it's insanely fast. 56 | - it greatly simplifies our code. 57 | - it can work over TCP, IPC (Inter Process Communication) or even in 58 | the same process. 59 | 60 | Powerhose allows us to: 61 | 62 | - deploy dynamically as many workers as we want, even on other 63 | servers when it makes sense. 64 | 65 | - write workers in C++ 66 | 67 | - greatly simplify the usage for our Python apps, since all it 68 | takes is a single :func:`execute` call to get the job done. 69 | 70 | Read more about this here: http://ziade.org/2012/02/06/scaling-crypto-work-in-python/ 71 | -------------------------------------------------------------------------------- /examples/bench.py: -------------------------------------------------------------------------------- 1 | import hmac 2 | import time 3 | import binascii 4 | import os 5 | import sys 6 | import subprocess 7 | import threading 8 | 9 | from powerhose import get_cluster 10 | from powerhose.client import Client 11 | 12 | 13 | _KEY = binascii.b2a_hex(os.urandom(4096))[:4096] 14 | 15 | 16 | def _sign(data): 17 | seed = hmac.new(_KEY, data).hexdigest() 18 | for i in range(400): 19 | seed = hmac.new(_KEY, seed).hexdigest() 20 | return 'OK' 21 | 22 | 23 | def sign(job): 24 | return _sign(job.data) 25 | 26 | 27 | _SIZE = 400 28 | _THREADS = 4 29 | _ONE = _SIZE / _THREADS 30 | _PROC = 10 31 | 32 | 33 | def timed(msg): 34 | def _timed(func): 35 | def __timed(*args, **kw): 36 | sys.stdout.write(msg + '...') 37 | sys.stdout.flush() 38 | start = time.time() 39 | try: 40 | return func(*args, **kw) 41 | finally: 42 | sys.stdout.write('%.4f s\n' % (time.time() - start)) 43 | sys.stdout.flush() 44 | return __timed 45 | return _timed 46 | 47 | 48 | @timed('%d calls, simple.' % _SIZE) 49 | def simple(): 50 | for i in range(_SIZE): 51 | _sign(str(i)) 52 | 53 | 54 | @timed('%d calls, %d threads' % (_SIZE, _THREADS)) 55 | def simple_3(): 56 | def _t(): 57 | for i in range(_ONE): 58 | _sign(str(i)) 59 | 60 | th = [threading.Thread(target=_t) for i in range(_THREADS)] 61 | for t in th: 62 | t.start() 63 | 64 | for t in th: 65 | t.join() 66 | 67 | 68 | @timed('%d calls via phose, %d threads %d phose workers' % (_SIZE, _THREADS, 69 | _PROC)) 70 | def _phose3(): 71 | 72 | def _t(): 73 | client = Client() 74 | for i in range(_ONE): 75 | try: 76 | client.execute(str(i)) 77 | except: 78 | print 'error' 79 | 80 | th = [threading.Thread(target=_t) for i in range(_THREADS)] 81 | for t in th: 82 | t.start() 83 | 84 | for t in th: 85 | t.join() 86 | 87 | 88 | @timed('%d calls via phose. %d workers' % (_SIZE, _PROC)) 89 | def _phose(client): 90 | for i in range(_SIZE): 91 | try: 92 | client.execute(str(i)) 93 | except: 94 | print 'error' 95 | 96 | 97 | def phose(): 98 | client = Client() 99 | p = run_cluster() 100 | time.sleep(5.) 101 | try: 102 | _phose(client) 103 | finally: 104 | p.terminate() 105 | 106 | 107 | def phose_3(): 108 | p = run_cluster() 109 | time.sleep(5.) 110 | try: 111 | _phose3() 112 | finally: 113 | p.terminate() 114 | 115 | 116 | def _run_cluster(): 117 | cluster = get_cluster('bench.sign', debug=False, 118 | numprocesses=_PROC, 119 | logfile='/tmp/phose') 120 | try: 121 | cluster.start() 122 | finally: 123 | cluster.stop() 124 | 125 | 126 | def run_cluster(): 127 | cmd = sys.executable + ' -c "import bench; bench._run_cluster()"' 128 | p = subprocess.Popen(cmd, shell=True) 129 | return p 130 | 131 | 132 | @timed('Single job') 133 | def single(): 134 | _sign('1') 135 | 136 | 137 | if __name__ == '__main__': 138 | single() 139 | #simple() 140 | simple_3() 141 | #phose() 142 | phose_3() 143 | -------------------------------------------------------------------------------- /examples/crypto.ini: -------------------------------------------------------------------------------- 1 | [circus] 2 | check_delay = 5 3 | endpoint = tcp://127.0.0.1:5555 4 | 5 | [watcher:master] 6 | cmd = ../bin/powerhose-broker 7 | warmup_delay = 0 8 | numprocesses = 1 9 | working_dir = . 10 | stdout_stream.class = StdoutStream 11 | 12 | [watcher:workers] 13 | cmd = ../bin/powerhose-worker 14 | args = crypto_worker.sign 15 | warmup_delay = 0 16 | numprocesses = 6 17 | working_dir = . 18 | stdout_stream.class = StdoutStream 19 | stderr_stream.class = StdoutStream 20 | 21 | [watcher:web] 22 | cmd = ../bin/gunicorn 23 | args = crypto_server:application -w 1 -k gevent 24 | warmup_delay = 0 25 | numprocesses = 1 26 | working_dir = . 27 | stdout_stream.class = StdoutStream 28 | stderr_stream.class = StdoutStream 29 | 30 | -------------------------------------------------------------------------------- /examples/crypto.py: -------------------------------------------------------------------------------- 1 | from powerhose.util import set_logger 2 | from powerhose import get_cluster 3 | 4 | 5 | set_logger(True) 6 | cluster = get_cluster('crypto_worker.sign') 7 | try: 8 | cluster.start() 9 | finally: 10 | cluster.stop() 11 | -------------------------------------------------------------------------------- /examples/crypto_client.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import binascii 4 | import random 5 | import time 6 | 7 | from powerhose.client import Client 8 | from powerhose.job import Job 9 | from powerhose.util import set_logger 10 | 11 | 12 | set_logger(True) 13 | algs = ('ECDSA256', 'Ed25519', 'RSA2048', 'RSA3248') 14 | client = Client() 15 | 16 | 17 | def run(): 18 | data = binascii.b2a_hex(os.urandom(256))[:256] 19 | job = Job(data + '--' + random.choice(algs)) 20 | return client.execute(job) 21 | 22 | 23 | print 'Running 100 crypto' 24 | start = time.time() 25 | 26 | 27 | for i in range(100): 28 | res = run() 29 | sys.stdout.write('.') 30 | sys.stdout.flush() 31 | 32 | print 'Done in %.2f' % (time.time() - start) 33 | -------------------------------------------------------------------------------- /examples/crypto_server.py: -------------------------------------------------------------------------------- 1 | """ Implements a web server that signs data passed into a POST 2 | """ 3 | from bottle import route, request, default_app 4 | from pycryptopp.publickey import rsa 5 | import time 6 | import threading 7 | import thread 8 | 9 | from gevent import monkey 10 | from gevent_zeromq import monkey_patch 11 | 12 | 13 | from powerhose.client import Pool 14 | 15 | 16 | monkey.patch_all() 17 | monkey_patch() 18 | 19 | 20 | class RSA3248(object): 21 | 22 | SIZEINBITS = 3248 23 | 24 | def __init__(self): 25 | self.signer = rsa.generate(sizeinbits=self.SIZEINBITS) 26 | 27 | def sign(self, msg): 28 | return self.signer.sign(msg) 29 | 30 | crypto = RSA3248() 31 | 32 | clients = {} 33 | _cl = Pool() 34 | 35 | def sign(job): 36 | crypto.sign(job.data) 37 | return 'OK' 38 | 39 | 40 | @route('/', method='POST') 41 | def index(): 42 | data = request.body.read() 43 | crypto.sign(data) 44 | return 'OK' 45 | 46 | 47 | @route('/phose', method='POST') 48 | def phose(): 49 | data = request.body.read() 50 | return _cl.execute(data) 51 | 52 | 53 | 54 | application = default_app() 55 | 56 | 57 | if __name__ == '__main__': 58 | def _t(): 59 | client = Client() 60 | for i in range(200): 61 | client.execute(str(i)) 62 | 63 | threads = [threading.Thread(target=_t) for i in range(5)] 64 | 65 | start = time.time() 66 | for t in threads: 67 | t.start() 68 | for t in threads: 69 | t.join() 70 | print time.time() - start 71 | -------------------------------------------------------------------------------- /examples/crypto_worker.py: -------------------------------------------------------------------------------- 1 | # taken from pycryptopp 2 | from pycryptopp.publickey import ecdsa, ed25519, rsa 3 | import random 4 | from powerhose.job import Job 5 | import binascii 6 | import os 7 | import sys 8 | 9 | 10 | def insecurerandstr(n): 11 | return ''.join(map(chr, map(random.randrange, [0] * n, [256] * n))) 12 | 13 | 14 | algs = {} 15 | 16 | 17 | class ECDSA256(object): 18 | def __init__(self): 19 | self.seed = insecurerandstr(12) 20 | self.signer = ecdsa.SigningKey(self.seed) 21 | 22 | def sign(self, msg): 23 | return self.signer.sign(msg) 24 | 25 | 26 | algs['ECDSA256'] = ECDSA256() 27 | 28 | 29 | class Ed25519(object): 30 | def __init__(self): 31 | self.seed = insecurerandstr(32) 32 | self.signer = ed25519.SigningKey(self.seed) 33 | 34 | def sign(self, msg): 35 | return self.signer.sign(msg) 36 | 37 | 38 | algs['Ed25519'] = Ed25519() 39 | 40 | 41 | class RSA2048(object): 42 | SIZEINBITS = 2048 43 | 44 | def __init__(self): 45 | self.signer = rsa.generate(sizeinbits=self.SIZEINBITS) 46 | 47 | def sign(self, msg): 48 | return self.signer.sign(msg) 49 | 50 | 51 | algs['RSA2048'] = RSA2048() 52 | 53 | 54 | class RSA3248(object): 55 | SIZEINBITS = 3248 56 | 57 | def __init__(self): 58 | self.signer = rsa.generate(sizeinbits=self.SIZEINBITS) 59 | 60 | def sign(self, msg): 61 | return self.signer.sign(msg) 62 | 63 | 64 | algs['RSA3248'] = RSA3248() 65 | 66 | 67 | def sign(job): 68 | msg, alg = job.data.split('--') 69 | ob = algs[alg] 70 | return ob.sign(msg) 71 | 72 | 73 | def run(): 74 | data = binascii.b2a_hex(os.urandom(256))[:256] 75 | job = Job(data + '--' + random.choice(algs.keys())) 76 | return sign(job) 77 | 78 | 79 | if __name__ == '__main__': 80 | while True: 81 | res = run() 82 | sys.stdout.write('.') 83 | sys.stdout.flush() 84 | -------------------------------------------------------------------------------- /examples/echo.ini: -------------------------------------------------------------------------------- 1 | [circus] 2 | check_delay = 5 3 | endpoint = tcp://127.0.0.1:5555 4 | ;flapping_window = 5 5 | ;flapping_attempts = 200 6 | check_flapping = False 7 | stats_endpoint = tcp://127.0.0.1:5557 8 | 9 | [watcher:master] 10 | cmd = /Users/tarek/Dev/github.com/powerhose/bin/powerhose-broker --debug --logfile /tmp/ok 11 | warmup_delay = 0 12 | numprocesses = 1 13 | working_dir = /Users/tarek/Dev/github.com/powerhose/examples 14 | ;stdout_stream.class = StdoutStream 15 | ;stderr_stream.class = StdoutStream 16 | priority = 2 17 | 18 | [watcher:workers] 19 | cmd = /Users/tarek/Dev/github.com/powerhose/bin/powerhose-worker echo_worker.echo --max-age 10 --max-age-delta 5 20 | warmup_delay = 0 21 | numprocesses = 50 22 | working_dir = /Users/tarek/Dev/github.com/powerhose/examples 23 | ;stdout_stream.class = StdoutStream 24 | ;stderr_stream.class = StdoutStream 25 | priority = 1 26 | -------------------------------------------------------------------------------- /examples/echo.py: -------------------------------------------------------------------------------- 1 | from powerhose import get_cluster 2 | from powerhose.client import Client 3 | 4 | 5 | cluster = get_cluster('echo_worker.echo', background=True) 6 | cluster.start() 7 | 8 | client = Client() 9 | 10 | for i in range(10): 11 | print client.execute(str(i)) 12 | 13 | cluster.stop() 14 | -------------------------------------------------------------------------------- /examples/echo_client.py: -------------------------------------------------------------------------------- 1 | import random 2 | import sys 3 | import time 4 | import threading 5 | 6 | from powerhose.client import Pool 7 | from powerhose.job import Job 8 | from powerhose.util import set_logger 9 | from powerhose.exc import NoWorkerError 10 | 11 | 12 | 13 | class Worker(threading.Thread): 14 | 15 | def __init__(self, pool): 16 | threading.Thread.__init__(self) 17 | self.pool = pool 18 | self.running = False 19 | 20 | def run(self): 21 | i = 0 22 | self.running = True 23 | 24 | while self.running: 25 | data = str(random.randint(1, 1000)) 26 | job = Job(data) 27 | sys.stdout.write(str(i) + '-> ') 28 | sys.stdout.flush() 29 | try: 30 | res = self.pool.execute(job) 31 | except NoWorkerError: 32 | sys.stdout.write('NO WORKER\n') 33 | else: 34 | assert res == data 35 | sys.stdout.write('OK\n') 36 | sys.stdout.flush() 37 | i += 1 38 | 39 | def stop(self): 40 | self.running = False 41 | self.join() 42 | 43 | 44 | if __name__ == '__main__': 45 | # a pool of 10 workers 46 | client = Pool() 47 | 48 | # 10 threads hammering the broker 49 | workers = [Worker(client) for i in range(20)] 50 | for worker in workers: 51 | worker.start() 52 | 53 | # just sit there for 100 seconds 54 | try: 55 | time.sleep(3600) 56 | finally: 57 | for worker in workers: 58 | worker.stop() 59 | 60 | 61 | -------------------------------------------------------------------------------- /examples/echo_worker.cpp: -------------------------------------------------------------------------------- 1 | #include "Worker.h" 2 | #include 3 | #include 4 | 5 | class SquareWorker: public Worker { 6 | 7 | protected: 8 | void execute(::std::vector< ::std::string>* vreq, ::std::vector< ::std::string>* vres); 9 | public: 10 | SquareWorker(const char* receiverChannel, const char* endPoint); 11 | }; 12 | 13 | 14 | SquareWorker::SquareWorker(const char* receiverChannel, const char* endPoint) : Worker(receiverChannel, endPoint) { 15 | // 16 | }; 17 | 18 | 19 | void SquareWorker::execute(::std::vector< ::std::string>* vreq, ::std::vector< ::std::string>* vres) { 20 | int value; 21 | ::std::string number = vreq->at(2); 22 | ::std::stringstream ss(number); 23 | ss >> value; 24 | value *= value; 25 | ::std::stringstream out; 26 | out << value; 27 | vres->push_back(out.str()); 28 | } 29 | 30 | 31 | int main(int argc, const char* const argv[]) { 32 | 33 | 34 | if(argc != 3) { 35 | ::std::cout << "Usage: echo_worker " << ::std::endl; 36 | return 1; 37 | } 38 | 39 | const char* receiver = argv[1]; //"ipc://worker-cpp.ipc"; 40 | const char* endpoint = argv[2]; // "ipc:///tmp/master-routing.ipc"; 41 | 42 | ::std::cout << receiver << ::std::endl; 43 | ::std::cout << endpoint << ::std::endl; 44 | ::std::cout << "Creating a worker" << ::std::endl; 45 | SquareWorker worker(receiver, endpoint); 46 | 47 | ::std::cout << "Let's run it" << ::std::endl; 48 | worker.run(); 49 | 50 | return 1; 51 | } 52 | 53 | 54 | -------------------------------------------------------------------------------- /examples/echo_worker.py: -------------------------------------------------------------------------------- 1 | 2 | def echo(job): 3 | """Just echo back the job's data""" 4 | return job.data 5 | -------------------------------------------------------------------------------- /examples/web.ini: -------------------------------------------------------------------------------- 1 | [circus] 2 | check_delay = 5 3 | endpoint = tcp://127.0.0.1:5555 4 | 5 | [watcher:master] 6 | cmd = ../bin/powerhose-broker 7 | warmup_delay = 0 8 | numprocesses = 1 9 | working_dir = . 10 | 11 | [watcher:workers] 12 | cmd = ../bin/powerhose-worker 13 | args = web_worker.hello 14 | warmup_delay = 0 15 | numprocesses = 5 16 | working_dir = . 17 | 18 | [watcher:webserver] 19 | cmd = ../bin/python 20 | args = web_client.py 21 | cmd= ../bin/gunicorn 22 | args = web_client:application -w 3 -k gevent 23 | numprocesses = 1 24 | working_dir = . 25 | -------------------------------------------------------------------------------- /examples/web_client.py: -------------------------------------------------------------------------------- 1 | from wsgiref.simple_server import make_server 2 | import json 3 | import time 4 | 5 | from powerhose.client import Client 6 | from powerhose.job import Job 7 | from powerhose.util import set_logger 8 | 9 | set_logger(True) 10 | 11 | client = Client() 12 | 13 | 14 | def application(environ, start_response): 15 | status = '200 OK' 16 | headers = [('Content-type', 'text/html')] 17 | start_response(status, headers) 18 | data = {} 19 | for key, value in environ.items(): 20 | if key.startswith('wsgi.') or key.startswith('gunicorn.'): 21 | continue 22 | data[key] = value 23 | 24 | job = Job(json.dumps(data)) 25 | 26 | start = time.time() 27 | try: 28 | return client.execute(job) 29 | finally: 30 | print('Time : %.4f\n' % (time.time() - start)) 31 | 32 | 33 | if __name__ == '__main__': 34 | httpd = make_server('', 8000, application) 35 | print "Listening on port 8000...." 36 | httpd.serve_forever() 37 | -------------------------------------------------------------------------------- /examples/web_worker.py: -------------------------------------------------------------------------------- 1 | import json 2 | import pprint 3 | import StringIO 4 | 5 | 6 | page = """\ 7 | 8 | 9 |

Hello

10 | 
%s
11 |
12 | 13 | 14 | """ 15 | 16 | 17 | def hello(job): 18 | environ = json.loads(job.data) 19 | printed = StringIO.StringIO() 20 | pprint.pprint(environ, stream=printed) 21 | printed.seek(0) 22 | return page % printed.read() 23 | -------------------------------------------------------------------------------- /libhose/Worker.cpp: -------------------------------------------------------------------------------- 1 | #include "Worker.h" 2 | #include "util.h" 3 | #include 4 | #include 5 | 6 | 7 | void callSocket(::std::string* request, ::std::string* response, ::zmq::socket_t* socket, int timeout) { 8 | ::std::cout << "" << " Sending " << *request << ::std::endl; 9 | 10 | // setting up the poller for this exchange 11 | 12 | zmq_pollitem_t poll_items[1]; 13 | zmq_pollitem_t item1; 14 | item1.socket = *socket; 15 | item1.events = ZMQ_POLLIN; 16 | poll_items[0] = item1; 17 | 18 | // sending the message to the master 19 | ::zmq::message_t msg; 20 | str2msg(request, &msg); 21 | try { 22 | socket->send(msg, ZMQ_NOBLOCK); 23 | } 24 | catch (...) { 25 | ::std::cout << "" << " could not send a msg" << ::std::endl; 26 | throw RegisterError(); 27 | } 28 | 29 | ::std::cout << "" << " waiting for an answer" << ::std::endl; 30 | 31 | // waiting for an answer now 32 | ::zmq::poll(poll_items, 1, timeout); 33 | 34 | ::std::cout << "" << " poll is over" << ::std::endl; 35 | // no answer in time 36 | if (poll_items[0].revents == 0) { 37 | ::std::cout << "did not get anything" << ::std::endl; 38 | throw RegisterError(); 39 | } 40 | else { 41 | // getting the result 42 | ::zmq::message_t res; 43 | socket->recv(&res); 44 | 45 | ::std::string result; 46 | msg2str(&res, &result); 47 | 48 | ::std::cout <<"" << " we got " << result << ::std::endl; 49 | ::std::cout << "" << " we want " << *response << ::std::endl; 50 | 51 | if (result != *response) { 52 | throw RegisterError(); 53 | } 54 | else ::std::cout << "" << " Registered!" << ::std::endl; 55 | } 56 | } 57 | 58 | /* 59 | heartbeat -- pings the master 60 | */ 61 | void *heartbeat(void *ptr) { 62 | Worker *worker = (Worker*) ptr; 63 | ::std::string pong = "PONG"; 64 | ::std::vector< ::std::string> vreq; 65 | vreq.push_back("PING"); 66 | vreq.push_back(worker->receiverChannel); 67 | ::std::string req; 68 | serialize(&vreq, &req); 69 | int failures = 0; 70 | int max_failures = 10; 71 | 72 | while (worker->heartbeatRunning && failures < max_failures) { 73 | try { 74 | callSocket(&req, &pong, worker->endpoint, worker->timeout); 75 | ::std::cout << "ping did work" << ::std::endl; 76 | } 77 | catch (...) { 78 | ::std::cout << "ping did not work!" << ::std::endl; 79 | failures += 1; 80 | } 81 | // we need to sleep here 82 | sleep(worker->heartbeatDelay); 83 | } 84 | // if we quit because of failures, we need to toggle the worker flag 85 | if (failures >= max_failures) { 86 | worker->heartbeatFailed = true; 87 | } 88 | ::std::cout << "bye!" << ::std::endl; 89 | return NULL; 90 | } 91 | 92 | 93 | /* 94 | * Workers Class 95 | * 96 | */ 97 | 98 | Worker::Worker(const char* receiverChannel, const char* endPoint) { 99 | this->heartbeatDelay = 10; 100 | this->timeout = 1000000; // zmq timeout is in microseconds 101 | this->ctx = new ::zmq::context_t(1); 102 | this->receiver = new ::zmq::socket_t(*this->ctx, ZMQ_REP); 103 | this->receiver->bind(receiverChannel); 104 | this->receiverChannel = receiverChannel; 105 | this->endpoint = new ::zmq::socket_t(*this->ctx, ZMQ_REQ); 106 | this->endpoint->connect(endPoint); 107 | this->heartbeatFailed = false; 108 | zmq_pollitem_t item1; 109 | item1.socket = *this->receiver; 110 | item1.events = ZMQ_POLLIN; 111 | this->poll_items[0] = item1; 112 | } 113 | 114 | Worker::~Worker() { 115 | this->receiver->close(); 116 | delete this->receiver; 117 | this->endpoint->close(); 118 | delete this->endpoint; 119 | delete this->ctx; 120 | } 121 | 122 | void Worker::execute(::std::vector< ::std::string>* vreq, ::std::vector< ::std::string>* vres) { 123 | vres->push_back("NOTIMPLEMENTED"); 124 | } 125 | 126 | void Worker::reg() { 127 | // register to the master 128 | ::std::cout << "registering" << ::std::endl; 129 | ::std::vector< ::std::string> vreq; 130 | vreq.push_back("PING"); 131 | vreq.push_back(this->receiverChannel); 132 | ::std::string req; 133 | serialize(&vreq, &req); 134 | ::std::string resp = "PONG"; 135 | callSocket(&req, &resp, this->endpoint, this->timeout); 136 | } 137 | 138 | void Worker::run() { 139 | // register 140 | this->reg(); 141 | 142 | // wait a bit 143 | sleep(1); 144 | 145 | // start the heartbeat 146 | pthread_t th; 147 | ::std::cout << "creating the HB" << ::std::endl; 148 | this->heartbeatRunning = true; 149 | pthread_create(&th, NULL, heartbeat, (void*)this); 150 | 151 | ::std::cout << "Now waiting for some job" << ::std::endl; 152 | // now loop and wait for some work to do 153 | zmq_pollitem_t poll_items[1]; 154 | zmq_pollitem_t item1; 155 | item1.socket = *this->receiver; 156 | item1.events = ZMQ_POLLIN; 157 | poll_items[0] = item1; 158 | 159 | ::std::vector< ::std::string> vresp; 160 | ::std::vector< ::std::string> vreq; 161 | 162 | this->running = true; 163 | 164 | while (this->running && !this->heartbeatFailed) { 165 | // waiting for an answer now 166 | ::zmq::poll(poll_items, 1, this->timeout); 167 | 168 | for (short j = 0; j < poll_items[0].revents; j++) { 169 | 170 | vresp.clear(); 171 | vreq.clear(); 172 | 173 | recv(this->receiver, &vreq); 174 | 175 | if (vreq.at(0) != "JOB") { 176 | // error, raise something 177 | } 178 | 179 | vresp.push_back("JOBRES"); 180 | 181 | try { 182 | this->execute(&vreq, &vresp); 183 | } 184 | catch (...) { 185 | vresp.push_back("ERROR"); 186 | } 187 | // sending back the response 188 | send(this->receiver, &vresp); 189 | } 190 | } 191 | 192 | // stop the heartbeat 193 | ::std::cout << "stopping the heartbeat" << ::std::endl; 194 | this->heartbeatRunning = false; 195 | pthread_join(th, NULL); 196 | ::std::cout << "The worker is done" << ::std::endl; 197 | } 198 | 199 | -------------------------------------------------------------------------------- /libhose/Worker.h: -------------------------------------------------------------------------------- 1 | #ifndef WORKER_H 2 | #define WORKER_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | 9 | class RegisterError: public ::std::exception { 10 | virtual const char* what() const throw() { 11 | return "Registration error"; 12 | } 13 | }; 14 | 15 | class Worker { 16 | 17 | private: 18 | bool running; 19 | ::zmq::context_t* ctx; 20 | ::zmq::socket_t* receiver; 21 | zmq_pollitem_t poll_items[1]; 22 | void reg(); 23 | 24 | protected: 25 | virtual void execute(::std::vector< ::std::string>* vreq, 26 | ::std::vector< ::std::string>* vres); 27 | 28 | public: 29 | Worker(const char* receiverChannel, const char* endPoint); 30 | virtual ~Worker(); 31 | void run(); 32 | 33 | int timeout; 34 | 35 | const char* receiverChannel; 36 | ::zmq::socket_t* endpoint; 37 | bool heartbeatRunning; 38 | bool heartbeatFailed; 39 | bool heartbeatDelay; 40 | }; 41 | 42 | #endif 43 | -------------------------------------------------------------------------------- /libhose/util.cpp: -------------------------------------------------------------------------------- 1 | #include "util.h" 2 | #include 3 | 4 | 5 | void str2msg(::std::string* data, ::zmq::message_t* msg) { 6 | const char* sres = data->c_str(); 7 | msg->rebuild((void *)(sres), data->size(), NULL, NULL); 8 | } 9 | 10 | void msg2str(::zmq::message_t* msg, ::std::string* res) { 11 | size_t size = msg->size(); 12 | char* data = new char[msg->size() + 1]; 13 | memcpy(data, msg->data(), size); 14 | data[size] = 0; 15 | res->assign(data); 16 | } 17 | 18 | void serialize(::std::vector< ::std::string>* data, ::std::string* res) { 19 | res->clear(); 20 | for (unsigned int i = 0; i < data->size(); i++) { 21 | res->append(data->at(i)); 22 | if (i < data->size() - 1) { 23 | res->append(":::"); 24 | } 25 | } 26 | } 27 | 28 | void unserialize(::std::string* data, ::std::vector< ::std::string>* res) { 29 | size_t found = 0; 30 | int current = 0; 31 | int size; 32 | ::std::string sep = ":::"; 33 | 34 | while (found != ::std::string::npos) { 35 | found = data->find_first_of(sep, current + 1); 36 | if (found > 0) { 37 | size = found - current; 38 | res->push_back(data->substr(current, size)); 39 | current = found + sep.size(); 40 | } 41 | } 42 | } 43 | 44 | void send(::zmq::socket_t* socket, ::std::vector< ::std::string>* data) { 45 | ::zmq::message_t msg; 46 | ::std::string res; 47 | serialize(data, &res); 48 | ::std::cout << "sending " << res << ::std::endl; 49 | str2msg(&res, &msg); 50 | socket->send(msg); 51 | } 52 | 53 | void recv(::zmq::socket_t* socket, ::std::vector< ::std::string>* data) { 54 | ::zmq::message_t msg; 55 | ::std::string smsg; 56 | socket->recv(&msg); 57 | msg2str(&msg, &smsg); 58 | ::std::cout << "received " << smsg << ::std::endl; 59 | data->clear(); 60 | unserialize(&smsg, data); 61 | } 62 | -------------------------------------------------------------------------------- /libhose/util.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | 6 | void str2msg(::std::string* data, ::zmq::message_t* msg); 7 | void msg2str(::zmq::message_t* msg, ::std::string* res); 8 | void serialize(::std::vector< ::std::string>* data, ::std::string* res); 9 | void unserialize(::std::string* data, ::std::vector< ::std::string>* res); 10 | void send(::zmq::socket_t* socket, ::std::vector< ::std::string>* data); 11 | void recv(::zmq::socket_t* socket, ::std::vector< ::std::string>* data); 12 | -------------------------------------------------------------------------------- /pip-requirements.txt: -------------------------------------------------------------------------------- 1 | circus 2 | pyzmq==2.1.11 3 | -------------------------------------------------------------------------------- /powerhose/__init__.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 3 | # You can obtain one at http://mozilla.org/MPL/2.0/. 4 | import os 5 | import sys 6 | import argparse 7 | import time 8 | 9 | from powerhose.util import (DEFAULT_BACKEND, DEFAULT_HEARTBEAT, # NOQA 10 | DEFAULT_FRONTEND, encode_params, get_params, 11 | DEFAULT_REG) 12 | from powerhose.client import DEFAULT_TIMEOUT_MOVF 13 | from powerhose.worker import DEFAULT_MAX_AGE, DEFAULT_MAX_AGE_DELTA 14 | 15 | 16 | __all__ = ('get_cluster', 'get_params') 17 | 18 | 19 | def get_cluster(target, numprocesses=5, frontend=DEFAULT_FRONTEND, 20 | backend=DEFAULT_BACKEND, heartbeat=DEFAULT_HEARTBEAT, 21 | register=DEFAULT_REG, 22 | working_dir='.', logfile='stdout', 23 | debug=False, background=False, worker_params=None, 24 | timeout=DEFAULT_TIMEOUT_MOVF, max_age=DEFAULT_MAX_AGE, 25 | max_age_delta=DEFAULT_MAX_AGE_DELTA): 26 | """Runs a Powerhose cluster. 27 | 28 | Options: 29 | 30 | - **callable**: The Python callable that will be called when the broker 31 | receive a job. 32 | - **numprocesses**: The number of workers. Defaults to 5. 33 | - **frontend**: the ZMQ socket to receive jobs. 34 | - **backend**: the ZMQ socket to communicate with workers. 35 | - **register** : the ZMQ socket to register workers 36 | - **heartbeat**: the ZMQ socket to receive heartbeat requests 37 | - **working_dir**: The working directory. Defaults to *"."* 38 | - **logfile**: The file to log into. Defaults to stdout. 39 | - **debug**: If True, the logs are at the DEBUG level. Defaults to False 40 | - **background**: If True, the cluster is run in the background. 41 | Defaults to False. 42 | - **worker_params**: a dict of params to pass to the worker. Default is 43 | None 44 | - **timeout** the maximum time allowed before the thread stacks is dumped 45 | and the job result not sent back. 46 | - **max_age**: maximum age for a worker in seconds. After that delay, 47 | the worker will simply quit. When set to -1, never quits. 48 | Defaults to -1. 49 | - **max_age_delta**: maximum value in seconds added to max age. 50 | The Worker will quit after *max_age + random(0, max_age_delta)* 51 | This is done to avoid having all workers quit at the same instant. 52 | """ 53 | from circus import get_arbiter 54 | 55 | python = sys.executable 56 | if debug: 57 | debug = ' --debug' 58 | else: 59 | debug = '' 60 | if worker_params: 61 | params = encode_params(worker_params) 62 | 63 | broker_cmd = [python, '-m', 'powerhose.broker', '--logfile', logfile, 64 | debug, '--frontend', frontend, '--backend', backend, 65 | '--heartbeat', heartbeat] 66 | 67 | worker_cmd = [python, '-m', 'powerhose.worker', target, '--logfile', 68 | logfile, debug, '--backend', backend, '--heartbeat', 69 | heartbeat, '--timeout', str(timeout), '--max-age', 70 | str(max_age), '--max-age-delta', str(max_age_delta)] 71 | 72 | if worker_params: 73 | worker_cmd += ['--params', params] 74 | 75 | if logfile == 'stdout': 76 | stream = {'class': 'StdoutStream'} 77 | else: 78 | stream = {'class': 'FileStream', 79 | 'filename': logfile} 80 | 81 | watchers = [{'name': 'broker', 82 | 'cmd': ' '.join(broker_cmd), 83 | 'working_dir': working_dir, 84 | 'executable': python, 85 | 'stderr_stream': stream, 86 | 'stdout_stream': stream 87 | }, 88 | {'name': 'workers', 89 | 'cmd': ' '.join(worker_cmd), 90 | 'numprocesses': numprocesses, 91 | 'working_dir': working_dir, 92 | 'executable': python, 93 | 'stderr_stream': stream, 94 | 'stdout_stream': stream 95 | } 96 | ] 97 | 98 | # XXX add more options 99 | arbiter = get_arbiter(watchers, background=background) 100 | 101 | # give a chance to all processes to start 102 | # XXX this should be in Circus 103 | if background: 104 | start = time.clock() 105 | while time.clock() - start < 5: 106 | statuses = [status == 'active' for status in 107 | arbiter.statuses().values()] 108 | if all(statuses): 109 | break 110 | 111 | return arbiter 112 | 113 | 114 | def main(args=sys.argv): 115 | from powerhose.util import set_logger, resolve_name 116 | 117 | parser = argparse.ArgumentParser(description='Run a Powerhose cluster.') 118 | 119 | parser.add_argument('--frontend', dest='frontend', 120 | default=DEFAULT_FRONTEND, 121 | help="ZMQ socket to receive jobs.") 122 | 123 | parser.add_argument('--backend', dest='backend', 124 | default=DEFAULT_BACKEND, 125 | help="ZMQ socket for workers.") 126 | 127 | parser.add_argument('--heartbeat', dest='heartbeat', 128 | default=DEFAULT_HEARTBEAT, 129 | help="ZMQ socket for the heartbeat.") 130 | 131 | parser.add_argument('target', help="Fully qualified name of the callable.") 132 | 133 | parser.add_argument('--debug', action='store_true', default=False, 134 | help="Debug mode") 135 | 136 | parser.add_argument('--numprocesses', dest='numprocesses', default=5, 137 | help="Number of processes to run.") 138 | 139 | parser.add_argument('--logfile', dest='logfile', default='stdout', 140 | help="File to log in to .") 141 | 142 | args = parser.parse_args() 143 | 144 | set_logger(args.debug, 'powerhose', args.logfile) 145 | set_logger(args.debug, 'circus', args.logfile) 146 | sys.path.insert(0, os.getcwd()) # XXX 147 | resolve_name(args.target) # check the callable 148 | 149 | cluster = get_cluster(args.target, args.numprocesses, 150 | frontend=args.frontend, backend=args.backend, 151 | heartbeat=args.heartbeat, logfile=args.logfile, 152 | debug=args.debug) 153 | try: 154 | cluster.start() 155 | except KeyboardInterrupt: 156 | pass 157 | finally: 158 | cluster.stop() 159 | 160 | 161 | if __name__ == '__main__': 162 | main() 163 | -------------------------------------------------------------------------------- /powerhose/broker.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 3 | # You can obtain one at http://mozilla.org/MPL/2.0/. 4 | """ Jobs runner. 5 | """ 6 | import random 7 | import errno 8 | import sys 9 | import traceback 10 | import argparse 11 | import os 12 | import time 13 | import psutil 14 | 15 | from zmq.eventloop import ioloop, zmqstream 16 | import zmq 17 | 18 | from powerhose.util import (set_logger, register_ipc_file, DEFAULT_FRONTEND, 19 | DEFAULT_BACKEND, DEFAULT_HEARTBEAT, DEFAULT_REG, 20 | logger, verify_broker, kill_ghost_brokers) 21 | from powerhose.heartbeat import Heartbeat 22 | from powerhose.exc import DuplicateBrokerError 23 | from powerhose.client import DEFAULT_TIMEOUT_MOVF 24 | 25 | 26 | DEFAULT_IOTHREADS = 1 27 | 28 | 29 | class Broker(object): 30 | """Class that route jobs to workers. 31 | 32 | Options: 33 | 34 | - **frontend**: the ZMQ socket to receive jobs. 35 | - **backend**: the ZMQ socket to communicate with workers. 36 | - **heartbeat**: the ZMQ socket to receive heartbeat requests. 37 | - **register** : the ZMQ socket to register workers 38 | """ 39 | def __init__(self, frontend=DEFAULT_FRONTEND, backend=DEFAULT_BACKEND, 40 | heartbeat=DEFAULT_HEARTBEAT, register=DEFAULT_REG, 41 | io_threads=DEFAULT_IOTHREADS, 42 | worker_timeout=DEFAULT_TIMEOUT_MOVF): 43 | # before doing anything, we verify if a broker is already up and 44 | # running 45 | logger.debug('Verifying if there is a running broker') 46 | pid = verify_broker(frontend) 47 | if pid is not None: # oops. can't do this ! 48 | logger.debug('Ooops, we have a running broker on that socket') 49 | raise DuplicateBrokerError(pid) 50 | 51 | logger.debug('Initializing the broker.') 52 | 53 | for endpoint in (frontend, backend, heartbeat): 54 | if endpoint.startswith('ipc'): 55 | register_ipc_file(endpoint) 56 | 57 | self.context = zmq.Context(io_threads=io_threads) 58 | 59 | # setting up the three sockets 60 | self._frontend = self.context.socket(zmq.ROUTER) 61 | self._frontend.identity = 'broker-' + frontend 62 | self._frontend.bind(frontend) 63 | self._backend = self.context.socket(zmq.ROUTER) 64 | self._backend.bind(backend) 65 | self._registration = self.context.socket(zmq.PULL) 66 | self._registration.bind(register) 67 | 68 | # setting up the streams 69 | self.loop = ioloop.IOLoop() 70 | self._frontstream = zmqstream.ZMQStream(self._frontend, self.loop) 71 | self._frontstream.on_recv(self._handle_recv_front) 72 | self._backstream = zmqstream.ZMQStream(self._backend, self.loop) 73 | self._backstream.on_recv(self._handle_recv_back) 74 | self._regstream = zmqstream.ZMQStream(self._registration, self.loop) 75 | self._regstream.on_recv(self._handle_reg) 76 | 77 | # heartbeat 78 | self.pong = Heartbeat(heartbeat, io_loop=self.loop, ctx=self.context) 79 | 80 | # status 81 | self.started = False 82 | self.poll_timeout = None 83 | 84 | # workers registration and timers 85 | self._workers = [] 86 | self._worker_times = {} 87 | self.worker_timeout = worker_timeout 88 | 89 | def _remove_worker(self, worker_id): 90 | logger.debug('%r removed' % worker_id) 91 | self._workers.remove(worker_id) 92 | if worker_id in self._worker_times: 93 | del self._worker_times[worker_id] 94 | 95 | def _handle_reg(self, msg): 96 | if msg[0] == 'REGISTER': 97 | if msg[1] not in self._workers: 98 | logger.debug('%r registered' % msg[1]) 99 | self._workers.append(msg[1]) 100 | elif msg[0] == 'UNREGISTER': 101 | if msg[1] in self._workers: 102 | self._remove_worker(msg[1]) 103 | 104 | def _check_worker(self, worker_id): 105 | # box-specific, will do better later XXX 106 | exists = psutil.pid_exists(int(worker_id)) 107 | if not exists: 108 | logger.debug('The worker %r is gone' % worker_id) 109 | return False 110 | 111 | if worker_id in self._worker_times: 112 | 113 | start, stop = self._worker_times[worker_id] 114 | if stop is not None: 115 | duration = start - stop 116 | if duration > self.worker_timeout: 117 | logger.debug('The worker %r is slow (%.2f)' % (worker_id, 118 | duration)) 119 | return False 120 | return True 121 | 122 | def _handle_recv_front(self, msg, tentative=0): 123 | # front => back 124 | # if the last part of the message is 'PING', we just PONG back 125 | # this is used as a health check 126 | if msg[-1] == 'PING': 127 | self._frontstream.send_multipart(msg[:-1] + [str(os.getpid())]) 128 | return 129 | 130 | #logger.debug('front -> back [choosing a worker]') 131 | if tentative == 3: 132 | logger.debug('No workers') 133 | self._frontstream.send_multipart(msg[:-1] + 134 | ['%d:ERROR:No worker' % os.getpid()]) 135 | return 136 | 137 | # we want to decide who's going to do the work 138 | found_worker = False 139 | 140 | while not found_worker and len(self._workers) > 0: 141 | worker_id = random.choice(self._workers) 142 | if not self._check_worker(worker_id): 143 | self._remove_worker(worker_id) 144 | else: 145 | found_worker = True 146 | 147 | if not found_worker: 148 | logger.debug('No worker, will try later') 149 | later = time.time() + 0.5 + (tentative * 0.2) 150 | self.loop.add_timeout(later, lambda: self._handle_recv_front(msg, 151 | tentative + 1)) 152 | return 153 | 154 | # start the timer 155 | self._worker_times[worker_id] = time.time(), None 156 | 157 | # now we can send to the right guy 158 | msg.insert(0, worker_id) 159 | #logger.debug('front -> back [%s]' % worker_id) 160 | 161 | try: 162 | self._backstream.send_multipart(msg) 163 | except Exception, e: 164 | # we don't want to die on error. we just log it 165 | exc_type, exc_value, exc_traceback = sys.exc_info() 166 | exc = traceback.format_tb(exc_traceback) 167 | exc.insert(0, str(e)) 168 | logger.error('\n'.join(exc)) 169 | 170 | def _handle_recv_back(self, msg): 171 | # back => front 172 | #logger.debug('front <- back [%s]' % msg[0]) 173 | 174 | # let's remove the worker id and track the time it took 175 | worker_id = msg[0] 176 | msg = msg[1:] 177 | now = time.time() 178 | 179 | if worker_id in self._worker_times: 180 | start, stop = self._worker_times[worker_id] 181 | self._worker_times[worker_id] = start, now 182 | else: 183 | self._worker_times[worker_id] = now, now 184 | 185 | try: 186 | self._frontstream.send_multipart(msg) 187 | except Exception, e: 188 | # we don't want to die on error. we just log it 189 | exc_type, exc_value, exc_traceback = sys.exc_info() 190 | exc = traceback.format_tb(exc_traceback) 191 | exc.insert(0, str(e)) 192 | logger.error('\n'.join(exc)) 193 | 194 | def start(self): 195 | """Starts the broker. 196 | """ 197 | logger.debug('Starting the loop') 198 | if self.started: 199 | return 200 | 201 | # running the heartbeat 202 | self.pong.start() 203 | 204 | self.started = True 205 | while self.started: 206 | try: 207 | self.loop.start() 208 | except zmq.ZMQError as e: 209 | logger.debug(str(e)) 210 | 211 | if e.errno == errno.EINTR: 212 | continue 213 | elif e.errno == zmq.ETERM: 214 | break 215 | else: 216 | logger.debug("got an unexpected error %s (%s)", str(e), 217 | e.errno) 218 | raise 219 | else: 220 | break 221 | 222 | def stop(self): 223 | """Stops the broker. 224 | """ 225 | if not self.started: 226 | return 227 | 228 | self._backstream.flush() 229 | logger.debug('Stopping the heartbeat') 230 | self.pong.stop() 231 | logger.debug('Stopping the loop') 232 | self.loop.stop() 233 | self.started = False 234 | self.context.destroy(0) 235 | 236 | 237 | def main(args=sys.argv): 238 | parser = argparse.ArgumentParser(description='Powerhose broker.') 239 | 240 | parser.add_argument('--frontend', dest='frontend', 241 | default=DEFAULT_FRONTEND, 242 | help="ZMQ socket to receive jobs.") 243 | 244 | parser.add_argument('--backend', dest='backend', 245 | default=DEFAULT_BACKEND, 246 | help="ZMQ socket for workers.") 247 | 248 | parser.add_argument('--heartbeat', dest='heartbeat', 249 | default=DEFAULT_HEARTBEAT, 250 | help="ZMQ socket for the heartbeat.") 251 | 252 | parser.add_argument('--register', dest='register', 253 | default=DEFAULT_REG, 254 | help="ZMQ socket for the registration.") 255 | 256 | parser.add_argument('--io-threads', type=int, 257 | default=DEFAULT_IOTHREADS, 258 | help="Number of I/O threads") 259 | 260 | parser.add_argument('--debug', action='store_true', default=False, 261 | help="Debug mode") 262 | 263 | parser.add_argument('--check', action='store_true', default=False, 264 | help=("Use this option to check if there's a running " 265 | " broker. Returns the PID if a broker is up.")) 266 | 267 | parser.add_argument('--purge-ghosts', action='store_true', default=False, 268 | help="Use this option to purge ghost brokers.") 269 | 270 | parser.add_argument('--logfile', dest='logfile', default='stdout', 271 | help="File to log in to .") 272 | 273 | args = parser.parse_args() 274 | set_logger(args.debug, logfile=args.logfile) 275 | 276 | if args.purge_ghosts: 277 | broker_pids, ghosts = kill_ghost_brokers(args.frontend) 278 | if broker_pids is None: 279 | logger.info('No running broker.') 280 | else: 281 | logger.info('The active broker runs at PID: %s' % broker_pids) 282 | 283 | if len(ghosts) == 0: 284 | logger.info('No ghosts where killed.') 285 | else: 286 | logger.info('Ghost(s) killed: %s' \ 287 | % ', '.join([str(g) for g in ghosts])) 288 | return 0 289 | 290 | if args.check: 291 | pid = verify_broker(args.frontend) 292 | if pid is None: 293 | logger.info('There seem to be no broker on this endpoint') 294 | else: 295 | logger.info('A broker is running. PID: %s' % pid) 296 | return 0 297 | 298 | logger.info('Starting the broker') 299 | try: 300 | broker = Broker(frontend=args.frontend, backend=args.backend, 301 | heartbeat=args.heartbeat, register=args.register, 302 | io_threads=args.io_threads) 303 | except DuplicateBrokerError, e: 304 | logger.info('There is already a broker running on PID %s' % e) 305 | logger.info('Exiting') 306 | return 1 307 | 308 | logger.info('Listening to incoming jobs at %r' % args.frontend) 309 | logger.info('Workers may register at %r' % args.backend) 310 | logger.info('The heartbeat socket is at %r' % args.heartbeat) 311 | try: 312 | broker.start() 313 | except KeyboardInterrupt: 314 | pass 315 | finally: 316 | broker.stop() 317 | 318 | return 0 319 | 320 | 321 | if __name__ == '__main__': 322 | sys.exit(main()) 323 | -------------------------------------------------------------------------------- /powerhose/client.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 3 | # You can obtain one at http://mozilla.org/MPL/2.0/. 4 | import threading 5 | from Queue import Queue 6 | from collections import defaultdict 7 | import errno 8 | import contextlib 9 | 10 | import zmq 11 | 12 | from powerhose.exc import TimeoutError, ExecutionError, NoWorkerError 13 | from powerhose.job import Job 14 | from powerhose.util import (send, recv, DEFAULT_FRONTEND, logger, 15 | extract_result, timed) 16 | 17 | 18 | DEFAULT_TIMEOUT = 5. 19 | DEFAULT_TIMEOUT_MOVF = 7.5 20 | DEFAULT_TIMEOUT_OVF = 1 21 | 22 | 23 | class Client(object): 24 | """Class to call a Powerhose cluster. 25 | 26 | Options: 27 | 28 | - **frontend**: ZMQ socket to call. 29 | - **timeout**: maximum allowed time for a job to run. 30 | Defaults to 1s. 31 | - **timeout_max_overflow**: maximum timeout overflow allowed. 32 | Defaults to 1.5s 33 | - **timeout_overflows**: number of times in a row the timeout value 34 | can be overflowed per worker. The client keeps a counter of 35 | executions that were longer than the regular timeout but shorter 36 | than **timeout_max_overflow**. When the number goes over 37 | **timeout_overflows**, the usual TimeoutError is raised. 38 | When a worker returns on time, the counter is reset. 39 | """ 40 | def __init__(self, frontend=DEFAULT_FRONTEND, timeout=DEFAULT_TIMEOUT, 41 | timeout_max_overflow=DEFAULT_TIMEOUT_MOVF, 42 | timeout_overflows=DEFAULT_TIMEOUT_OVF, 43 | debug=False, ctx=None): 44 | self.kill_ctx = ctx is None 45 | self.ctx = ctx or zmq.Context() 46 | self.frontend = frontend 47 | self.master = self.ctx.socket(zmq.REQ) 48 | self.master.connect(frontend) 49 | logger.debug('Client connected to %s' % frontend) 50 | self.poller = zmq.Poller() 51 | self.poller.register(self.master, zmq.POLLIN) 52 | self.timeout = timeout * 1000 53 | self.lock = threading.Lock() 54 | self.timeout_max_overflow = timeout_max_overflow * 1000 55 | self.timeout_overflows = timeout_overflows 56 | self.timeout_counters = defaultdict(int) 57 | self.debug = debug 58 | 59 | def execute(self, job, timeout=None): 60 | """Runs the job 61 | 62 | Options: 63 | 64 | - **job**: Job to be performed. Can be a :class:`Job` 65 | instance or a string. If it's a string a :class:`Job` instance 66 | will be automatically created out of it. 67 | - **timeout**: maximum allowed time for a job to run. 68 | If not provided, uses the one defined in the constructor. 69 | 70 | If the job fails after the timeout, raises a :class:`TimeoutError`. 71 | 72 | This method is thread-safe and uses a lock. If you need to execute a 73 | lot of jobs simultaneously on a broker, use the :class:`Pool` class. 74 | 75 | """ 76 | if timeout is None: 77 | timeout = self.timeout_max_overflow 78 | 79 | try: 80 | duration, res = timed(self.debug)(self._execute)(job, timeout) 81 | worker_pid, res, data = res 82 | 83 | # if we overflowed we want to increment the counter 84 | # if not we reset it 85 | if duration * 1000 > self.timeout: 86 | self.timeout_counters[worker_pid] += 1 87 | 88 | # XXX well, we have the result but we want to timeout 89 | # nevertheless because that's been too much overflow 90 | if self.timeout_counters[worker_pid] > self.timeout_overflows: 91 | raise TimeoutError(timeout / 1000) 92 | else: 93 | self.timeout_counters[worker_pid] = 0 94 | 95 | if not res: 96 | if data == 'No worker': 97 | raise NoWorkerError() 98 | raise ExecutionError(data) 99 | except Exception: 100 | # logged, connector replaced. 101 | logger.exception('Failed to execute the job.') 102 | raise 103 | 104 | return data 105 | 106 | def ping(self, timeout=1.): 107 | """Can be used to simply ping the broker to make sure 108 | it's responsive. 109 | 110 | 111 | Returns the broker PID""" 112 | with self.lock: 113 | send(self.master, 'PING') 114 | while True: 115 | try: 116 | socks = dict(self.poller.poll(timeout * 1000)) 117 | break 118 | except zmq.ZMQError as e: 119 | if e.errno != errno.EINTR: 120 | return None 121 | 122 | if socks.get(self.master) == zmq.POLLIN: 123 | res = recv(self.master) 124 | try: 125 | res = int(res) 126 | except TypeError: 127 | pass 128 | return res 129 | 130 | return None 131 | 132 | def close(self): 133 | #self.master.close() 134 | self.master.setsockopt(zmq.LINGER, 0) 135 | self.master.close() 136 | 137 | if self.kill_ctx: 138 | self.ctx.destroy(0) 139 | 140 | def _execute(self, job, timeout=None): 141 | if isinstance(job, str): 142 | job = Job(job) 143 | 144 | if timeout is None: 145 | timeout = self.timeout_max_overflow 146 | 147 | with self.lock: 148 | send(self.master, job.serialize()) 149 | 150 | while True: 151 | try: 152 | socks = dict(self.poller.poll(timeout)) 153 | break 154 | except zmq.ZMQError as e: 155 | if e.errno != errno.EINTR: 156 | raise 157 | 158 | if socks.get(self.master) == zmq.POLLIN: 159 | return extract_result(recv(self.master)) 160 | 161 | raise TimeoutError(timeout / 1000) 162 | 163 | 164 | class Pool(object): 165 | """The pool class manage several :class:`CLient` instances 166 | and publish the same interface, 167 | 168 | Options: 169 | 170 | - **size**: size of the pool. Defaults to 10. 171 | - **frontend**: ZMQ socket to call. 172 | - **timeout**: maximum allowed time for a job to run. 173 | Defaults to 5s. 174 | - **timeout_max_overflow**: maximum timeout overflow allowed 175 | - **timeout_overflows**: number of times in a row the timeout value 176 | can be overflowed per worker. The client keeps a counter of 177 | executions that were longer than the regular timeout but shorter 178 | than **timeout_max_overflow**. When the number goes over 179 | **timeout_overflows**, the usual TimeoutError is raised. 180 | When a worker returns on time, the counter is reset. 181 | """ 182 | def __init__(self, size=10, frontend=DEFAULT_FRONTEND, 183 | timeout=DEFAULT_TIMEOUT, 184 | timeout_max_overflow=DEFAULT_TIMEOUT_MOVF, 185 | timeout_overflows=DEFAULT_TIMEOUT_OVF, 186 | debug=False, ctx=None): 187 | self._connectors = Queue() 188 | self.frontend = frontend 189 | self.timeout = timeout 190 | self.timeout_overflows = timeout_overflows 191 | self.timeout_max_overflow = timeout_max_overflow 192 | self.debug = debug 193 | self.ctx = ctx or zmq.Context() 194 | 195 | for i in range(size): 196 | self._connectors.put(self._create_client()) 197 | 198 | def _create_client(self): 199 | return Client(self.frontend, self.timeout, 200 | self.timeout_max_overflow, self.timeout_overflows, 201 | debug=self.debug, ctx=self.ctx) 202 | 203 | @contextlib.contextmanager 204 | def _connector(self, timeout): 205 | connector = self._connectors.get(timeout=timeout) 206 | try: 207 | yield connector 208 | except Exception: 209 | # connector replaced 210 | try: 211 | connector.close() 212 | finally: 213 | self._connectors.put(self._create_client()) 214 | raise 215 | else: 216 | self._connectors.put(connector) 217 | 218 | def execute(self, job, timeout=None): 219 | with self._connector(timeout) as connector: 220 | return connector.execute(job, timeout) 221 | 222 | def close(self): 223 | self.ctx.destroy(0) 224 | 225 | def ping(self, timeout=.1): 226 | with self._connector(self.timeout) as connector: 227 | return connector.ping(timeout) 228 | -------------------------------------------------------------------------------- /powerhose/exc.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 3 | # You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | 6 | class TimeoutError(Exception): 7 | pass 8 | 9 | 10 | class ExecutionError(Exception): 11 | pass 12 | 13 | 14 | class DuplicateBrokerError(Exception): 15 | pass 16 | 17 | 18 | class NoWorkerError(Exception): 19 | pass 20 | -------------------------------------------------------------------------------- /powerhose/heartbeat.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 3 | # You can obtain one at http://mozilla.org/MPL/2.0/. 4 | import threading 5 | import errno 6 | import time 7 | 8 | import zmq 9 | from zmq.eventloop import ioloop, zmqstream 10 | 11 | from powerhose.util import logger, DEFAULT_HEARTBEAT 12 | 13 | 14 | class Stethoscope(threading.Thread): 15 | """Class that implements a ZMQ heartbeat client. 16 | 17 | Options: 18 | 19 | - **endpoint** : The ZMQ socket to call. 20 | - **warmup_delay** : The delay before starting to Ping. Defaults to 5s. 21 | - **delay**: The delay between two pings. Defaults to 3s. 22 | - **retries**: The number of attempts to ping. Defaults to 3. 23 | - **onbeatlost**: a callable that will be called when a ping failed. 24 | If the callable returns **True**, the ping quits. Defaults to None. 25 | - **onbeat**: a callable that will be called when a ping succeeds. 26 | Defaults to None. 27 | """ 28 | def __init__(self, endpoint=DEFAULT_HEARTBEAT, warmup_delay=.5, delay=10., 29 | retries=3, 30 | onbeatlost=None, onbeat=None, io_loop=None, ctx=None): 31 | threading.Thread.__init__(self) 32 | self.loop = io_loop or ioloop.IOLoop.instance() 33 | self._stop_loop = io_loop is None 34 | self.daemon = True 35 | self.context = ctx or zmq.Context() 36 | self.endpoint = endpoint 37 | self.running = False 38 | self.delay = delay 39 | self.retries = retries 40 | self.onbeatlost = onbeatlost 41 | self.onbeat = onbeat 42 | self.warmup_delay = warmup_delay 43 | self._endpoint = None 44 | self._stream = None 45 | self._timer = None 46 | self.tries = 0 47 | 48 | def _initialize(self): 49 | logger.debug('Subscribing to ' + self.endpoint) 50 | self._endpoint = self.context.socket(zmq.SUB) 51 | self._endpoint.setsockopt(zmq.SUBSCRIBE, '') 52 | self._endpoint.linger = 0 53 | #self._endpoint.identity = str(os.getpid()) 54 | self._endpoint.connect(self.endpoint) 55 | self._stream = zmqstream.ZMQStream(self._endpoint, self.loop) 56 | self._stream.on_recv(self._handle_recv) 57 | self._timer = ioloop.PeriodicCallback(self._delayed, 58 | self.delay * 1000, io_loop=self.loop) 59 | 60 | def _delayed(self): 61 | self.tries += 1 62 | if self.tries >= self.retries: 63 | logger.debug('Nothing came back') 64 | if self.onbeatlost is None or self.onbeatlost(): 65 | self.stop() # bye ! 66 | 67 | def _handle_recv(self, msg): 68 | self.tries = 0 69 | if self.onbeat is not None: 70 | self.onbeat() 71 | logger.debug(msg[0]) 72 | 73 | def run(self): 74 | """Starts the loop""" 75 | logger.debug('Starting the loop') 76 | if self.running: 77 | return 78 | 79 | self._initialize() 80 | time.sleep(self.warmup_delay) 81 | self._timer.start() 82 | self.running = True 83 | while self.running: 84 | try: 85 | self.loop.start() 86 | except zmq.ZMQError as e: 87 | logger.debug(str(e)) 88 | 89 | if e.errno == errno.EINTR: 90 | continue 91 | elif e.errno == zmq.ETERM: 92 | break 93 | else: 94 | logger.debug("got an unexpected error %s (%s)", str(e), 95 | e.errno) 96 | raise 97 | else: 98 | break 99 | 100 | def stop(self): 101 | """Stops the Pinger""" 102 | logger.debug('Stopping the Pinger') 103 | self.running = False 104 | try: 105 | self._stream.flush() 106 | except zmq.ZMQError: 107 | pass 108 | if self._stop_loop: 109 | self.loop.stop() 110 | if self.isAlive(): 111 | try: 112 | self.join() 113 | except RuntimeError: 114 | pass 115 | 116 | 117 | class Heartbeat(object): 118 | """Class that implements a ZMQ heartbeat server. 119 | 120 | This class sends in a ZMQ socket regular beats. 121 | 122 | Options: 123 | 124 | - **endpoint** : The ZMQ socket to call. 125 | - **interval** : Interval between two beat. 126 | """ 127 | def __init__(self, endpoint=DEFAULT_HEARTBEAT, interval=10., 128 | io_loop=None, ctx=None): 129 | self.loop = io_loop or ioloop.IOLoop.instance() 130 | self.daemon = True 131 | self.kill_context = ctx is None 132 | self.context = ctx or zmq.Context() 133 | self.endpoint = endpoint 134 | self.running = False 135 | self.interval = interval 136 | logger.debug('Publishing to ' + self.endpoint) 137 | self._endpoint = self.context.socket(zmq.PUB) 138 | self._endpoint.linger = 0 139 | #self._endpoint.identity = b'HB' 140 | self._endpoint.hwm = 0 141 | self._endpoint.bind(self.endpoint) 142 | self._cb = ioloop.PeriodicCallback(self._ping, interval * 1000, 143 | io_loop=self.loop) 144 | 145 | def start(self): 146 | """Starts the Pong service""" 147 | self.running = True 148 | self._cb.start() 149 | 150 | def _ping(self): 151 | logger.debug('*beat*') 152 | self._endpoint.send('BEAT') 153 | 154 | def stop(self): 155 | """Stops the Pong service""" 156 | self.running = False 157 | self._cb.stop() 158 | if self.kill_context: 159 | self.context.destroy(0) 160 | -------------------------------------------------------------------------------- /powerhose/job.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 3 | # You can obtain one at http://mozilla.org/MPL/2.0/. 4 | """ Job class. 5 | """ 6 | 7 | 8 | class Job(object): 9 | """A Job is just a container that's passed into the wire. 10 | 11 | A job is composed of headers and raw data, and offers serialization. 12 | 13 | Options: 14 | 15 | - **data**: the raw string data (default: '') 16 | - **headers**: a mapping of headers (default: None) 17 | """ 18 | def __init__(self, data='', headers=None): 19 | self.data = data 20 | self.headers = {} 21 | 22 | if headers is not None: 23 | for name, value in headers.items(): 24 | self.add_header(name, value) 25 | 26 | def add_header(self, name, value): 27 | """Adds a header. 28 | 29 | Options: 30 | 31 | - **name**: header name 32 | - **value**: value 33 | 34 | Both values should be strings. If the header already exists 35 | it's overwritten. 36 | """ 37 | name = name.replace(':', '\:') 38 | value = value.replace(':', '\:') 39 | self.headers[name] = value 40 | 41 | def serialize(self): 42 | """Serializes the job. 43 | 44 | The output can be sent over a wire. A serialized job 45 | can be read with a cursor with no specific preprocessing. 46 | """ 47 | if len(self.headers) == 0: 48 | headers = ['NONE'] 49 | else: 50 | headers = ['%s:%s' % (name, value) for name, value in 51 | self.headers.items()] 52 | 53 | headers = '::'.join(headers) 54 | return headers + ':::' + self.data 55 | 56 | @classmethod 57 | def load_from_string(cls, data): 58 | """Loads a job from a serialized string and return a Job instance. 59 | 60 | Options: 61 | 62 | - **data** : serialized string. 63 | """ 64 | if ':::' not in data: 65 | raise ValueError(data) 66 | headers, data = data.split(':::', 1) 67 | res = {} 68 | for header in headers.split('::'): 69 | if header == 'NONE': 70 | break 71 | header_data = header.strip().split(':') 72 | if len(header_data) != 2: 73 | raise ValueError(header_data) 74 | res[header_data[0]] = header_data[1] 75 | 76 | return cls(data, res) 77 | -------------------------------------------------------------------------------- /powerhose/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/powerhose/3dfb0b4fa39ab2a0fb99af411d434ccb85b605ff/powerhose/tests/__init__.py -------------------------------------------------------------------------------- /powerhose/tests/jobs.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 3 | # You can obtain one at http://mozilla.org/MPL/2.0/. 4 | import time 5 | import sys 6 | from powerhose.util import logger, set_logger 7 | 8 | set_logger(True, logfile='stdout') 9 | 10 | 11 | def _p(msg): 12 | sys.stdout.write(msg + '\n') 13 | logger.debug(msg) 14 | sys.stdout.flush() 15 | 16 | 17 | def fail(job): 18 | _p('Starting powerhose.tests.jobs.fail') 19 | try: 20 | raise ValueError(job.data) 21 | finally: 22 | _p('Ending powerhose.tests.jobs.fail') 23 | 24 | 25 | def timeout(job): 26 | _p('Starting powerhose.tests.jobs.timeout') 27 | time.sleep(2.) 28 | try: 29 | return job.data 30 | finally: 31 | _p('Ending powerhose.tests.jobs.timeout') 32 | 33 | 34 | def timeout_overflow(job): 35 | _p('Starting powerhose.tests.jobs.timeout_overflow') 36 | time.sleep(float(job.data)) 37 | try: 38 | return 'xx' 39 | finally: 40 | _p('Ending powerhose.tests.jobs.timeout_overflow') 41 | 42 | 43 | def success(job): 44 | _p('Starting powerhose.tests.jobs.success') 45 | try: 46 | return job.data 47 | finally: 48 | _p('Ending powerhose.tests.jobs.success') 49 | -------------------------------------------------------------------------------- /powerhose/tests/test_cluster.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 3 | # You can obtain one at http://mozilla.org/MPL/2.0/. 4 | import unittest 5 | import logging 6 | import os 7 | import tempfile 8 | import time 9 | 10 | import psutil 11 | 12 | from powerhose import get_cluster 13 | from powerhose.exc import ExecutionError, TimeoutError 14 | from powerhose import client 15 | 16 | 17 | logger = logging.getLogger('powerhose') 18 | 19 | 20 | class TestCluster(unittest.TestCase): 21 | 22 | def setUp(self): 23 | self.clusters = [] 24 | self.files = [] 25 | self.old_timeout = client.DEFAULT_TIMEOUT 26 | self.old_movf = client.DEFAULT_TIMEOUT_MOVF 27 | self.old_ovf = client.DEFAULT_TIMEOUT_OVF 28 | client.DEFAULT_TIMEOUT = .5 29 | client.DEFAULT_TIMEOUT_MOVF = 1. 30 | client.DEFAULT_TIMEOUT_OVF = 1 31 | self.overflow = str(client.DEFAULT_TIMEOUT + .5) 32 | self.moverflow = str(client.DEFAULT_TIMEOUT_MOVF + .5) 33 | 34 | def tearDown(self): 35 | logger.debug('stopping cluster') 36 | for cl in self.clusters: 37 | cl.stop() 38 | for fl in self.files: 39 | os.remove(fl) 40 | logger.debug('cluster stopped') 41 | client.DEFAULT_TIMEOUT = self.old_timeout 42 | client.DEFAULT_TIMEOUT_MOVF = self.old_movf 43 | client.DEFAULT_TIMEOUT_OVF = self.old_ovf 44 | 45 | def _get_file(self): 46 | fd, path = tempfile.mkstemp() 47 | os.close(fd) 48 | self.files.append(path) 49 | return path 50 | 51 | def _get_cluster(self, callable, **kw): 52 | logger.debug('getting cluster') 53 | front = 'ipc:///tmp/f-%s' % callable 54 | back = 'ipc:///tmp/b-%s' % callable 55 | hb = 'ipc:///tmp/h-%s' % callable 56 | reg = 'ipc:///tmp/r-%s' % callable 57 | 58 | cl = get_cluster(callable, frontend=front, backend=back, heartbeat=hb, 59 | register=reg, 60 | numprocesses=1, background=True, debug=True, 61 | timeout=client.DEFAULT_TIMEOUT_MOVF, **kw) 62 | cl.start() 63 | time.sleep(1.) # stabilization 64 | self.clusters.append(cl) 65 | logger.debug('cluster ready') 66 | return client.Pool(size=3, frontend=front, debug=True, 67 | timeout=client.DEFAULT_TIMEOUT, 68 | timeout_max_overflow=client.DEFAULT_TIMEOUT_MOVF, 69 | timeout_overflows=client.DEFAULT_TIMEOUT_OVF) 70 | 71 | def test_error(self): 72 | client = self._get_cluster('powerhose.tests.jobs.fail') 73 | self.assertRaises(ExecutionError, client.execute, 'xx') 74 | 75 | def test_timeout(self): 76 | client = self._get_cluster('powerhose.tests.jobs.timeout_overflow') 77 | self.assertRaises(TimeoutError, client.execute, self.moverflow) 78 | 79 | def test_success(self): 80 | client = self._get_cluster('powerhose.tests.jobs.success') 81 | self.assertEqual(client.execute('xx'), 'xx') 82 | 83 | def test_timeout_ovf(self): 84 | # this should work 1 time then fail 85 | file = self._get_file() 86 | client = self._get_cluster('powerhose.tests.jobs.timeout_overflow', 87 | logfile=file) 88 | 89 | # trying a PING 90 | time.sleep(.5) 91 | self.assertTrue(client.ping() is not None) 92 | 93 | try: 94 | self.assertEqual(client.execute(self.overflow), 'xx') 95 | except Exception: 96 | with open(file) as f: 97 | raise Exception(f.read()) 98 | 99 | try: 100 | self.assertRaises(TimeoutError, client.execute, self.overflow) 101 | except Exception: 102 | with open(file) as f: 103 | raise Exception(f.read()) 104 | 105 | # calling it back with the right execution time resets the counter 106 | self.assertEqual(client.execute('.1'), 'xx') 107 | self.assertEqual(client.execute(self.overflow), 'xx') 108 | self.assertRaises(TimeoutError, client.execute, self.overflow) 109 | 110 | def test_timeout_dump(self): 111 | file = self._get_file() 112 | client = self._get_cluster('powerhose.tests.jobs.timeout_overflow', 113 | logfile=file) 114 | 115 | self.assertRaises(TimeoutError, client.execute, self.moverflow) 116 | time.sleep(1.) # give the worker a chance to dump the stack 117 | 118 | with open(file) as f: 119 | res = [line.strip() for line in f.readlines() if line.strip()] 120 | 121 | # the worker should be blocked on the sleep 122 | self.assertTrue('time.sleep(float(job.data))' in res) 123 | 124 | def test_worker_max_age(self): 125 | # a worker with a max age of 1.5 126 | client = self._get_cluster('powerhose.tests.jobs.success', 127 | max_age=1.5, max_age_delta=0) 128 | time.sleep(.5) 129 | 130 | self.assertEqual(client.execute('xx'), 'xx') 131 | 132 | cl = self.clusters[-1] 133 | 134 | # get the pid of the current worker 135 | pid = cl.watchers[1].processes.keys()[0] 136 | 137 | # wait 3 seconds 138 | time.sleep(2.) 139 | 140 | # should be different 141 | self.assertNotEqual(pid, cl.watchers[1].processes.keys()[0]) 142 | 143 | def test_worker_max_age2(self): 144 | 145 | # we want to run a job, and have the max age reached while the job 146 | # is being executed, to verify that the job returns before the 147 | # worker is killed. 148 | client.DEFAULT_TIMEOUT = 5. 149 | client.DEFAULT_TIMEOUT_MOVF = 7. 150 | file = self._get_file() 151 | _client = self._get_cluster('powerhose.tests.jobs.timeout_overflow', 152 | max_age=5., max_age_delta=0, 153 | logfile=file) 154 | 155 | time.sleep(.2) 156 | cl = self.clusters[-1] 157 | 158 | # get the pid of the current worker 159 | pid = cl.watchers[1].processes.keys()[0] 160 | 161 | # work for 3 seconds 162 | try: 163 | self.assertEqual(_client.execute('6.0'), 'xx') 164 | except Exception: 165 | with open(file) as f: 166 | print(f.read()) 167 | raise 168 | 169 | # wait until the process is dead 170 | now = time.time() 171 | while psutil.pid_exists(int(pid)) and time.time() - now < 10.: 172 | time.sleep(.1) 173 | 174 | # now give a chance to the new one to stabilize 175 | time.sleep(1.) 176 | 177 | # should be different 178 | try: 179 | self.assertNotEqual(pid, cl.watchers[1].processes.keys()[0]) 180 | except Exception: 181 | with open(file) as f: 182 | print(f.read()) 183 | raise 184 | -------------------------------------------------------------------------------- /powerhose/tests/test_heartbeat.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 3 | # You can obtain one at http://mozilla.org/MPL/2.0/. 4 | import unittest 5 | import time 6 | from powerhose.heartbeat import Stethoscope, Heartbeat 7 | 8 | 9 | class TestHeartbeat(unittest.TestCase): 10 | 11 | def test_working(self): 12 | beats = [] 13 | lost = [] 14 | 15 | def onbeat(): 16 | beats.append('.') 17 | 18 | def onbeatlost(): 19 | lost.append('.') 20 | 21 | hb = Heartbeat('ipc:///tmp/stetho.ipc', interval=0.1) 22 | hb.start() 23 | time.sleep(.2) 24 | 25 | stetho = Stethoscope('ipc:///tmp/stetho.ipc', onbeat=onbeat, 26 | onbeatlost=onbeatlost, delay=1., retries=5.) 27 | stetho.start() 28 | 29 | time.sleep(5.) 30 | 31 | stetho.stop() 32 | hb.stop() 33 | self.assertEqual(len(lost), 0, len(lost)) 34 | self.assertTrue(len(beats) > 10, len(beats)) 35 | 36 | def test_lost(self): 37 | beats = [] 38 | lost = [] 39 | 40 | def _onbeat(): 41 | beats.append('.') 42 | 43 | def _onbeatlost(): 44 | lost.append('.') 45 | 46 | hb = Heartbeat('ipc:///tmp/stetho.ipc', interval=0.1) 47 | hb.start() 48 | time.sleep(.2) 49 | 50 | stetho = Stethoscope('ipc:///tmp/stetho.ipc', onbeat=_onbeat, 51 | onbeatlost=_onbeatlost, delay=0.1) 52 | stetho.start() 53 | 54 | time.sleep(2.) 55 | hb.stop() # the hber stops 56 | 57 | # the stethoer continues for a while 58 | time.sleep(2.) 59 | 60 | stetho.stop() 61 | 62 | self.assertTrue(len(beats) > 0) 63 | self.assertTrue(len(lost) > 3) 64 | -------------------------------------------------------------------------------- /powerhose/tests/test_job.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 3 | # You can obtain one at http://mozilla.org/MPL/2.0/. 4 | import unittest 5 | from powerhose.job import Job 6 | 7 | 8 | class TestJob(unittest.TestCase): 9 | 10 | def test_job(self): 11 | job = Job('somedata', {'one': '1'}) 12 | data = job.serialize() 13 | job2 = Job.load_from_string(data) 14 | self.assertTrue(job.data, job2.data) 15 | self.assertTrue(job.headers.items(), job2.headers.items()) 16 | -------------------------------------------------------------------------------- /powerhose/util.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 3 | # You can obtain one at http://mozilla.org/MPL/2.0/. 4 | import os 5 | import atexit 6 | import time 7 | import zmq 8 | import logging 9 | import logging.handlers 10 | import sys 11 | import gc 12 | import traceback 13 | import threading 14 | 15 | from powerhose.exc import TimeoutError 16 | 17 | 18 | DEFAULT_FRONTEND = "ipc:///tmp/powerhose-front.ipc" 19 | DEFAULT_BACKEND = "ipc:///tmp/powerhose-back.ipc" 20 | DEFAULT_HEARTBEAT = "ipc:///tmp/powerhose-beat.ipc" 21 | DEFAULT_REG = "ipc:///tmp/powerhose-reg.ipc" 22 | 23 | logger = logging.getLogger('powerhose') 24 | _IPC_FILES = [] 25 | 26 | PARAMS = {} 27 | 28 | 29 | @atexit.register 30 | def _cleanup_ipc_files(): 31 | for file in _IPC_FILES: 32 | file = file.split('ipc://')[-1] 33 | if os.path.exists(file): 34 | os.remove(file) 35 | 36 | 37 | def register_ipc_file(file): 38 | _IPC_FILES.append(file) 39 | 40 | 41 | def send(socket, msg, more=False, max_retries=3, retry_sleep=0.1): 42 | retries = 0 43 | while retries < max_retries: 44 | try: 45 | logger.debug('send') 46 | 47 | if more: 48 | socket.send(msg, zmq.SNDMORE | zmq.NOBLOCK) 49 | else: 50 | socket.send(msg, zmq.NOBLOCK) 51 | return 52 | except zmq.ZMQError, e: 53 | logger.debug('Failed on send()') 54 | logger.debug(str(e)) 55 | if e.errno in (zmq.EFSM, zmq.EAGAIN): 56 | retries += 1 57 | time.sleep(retry_sleep) 58 | else: 59 | raise 60 | 61 | logger.debug('Sending failed') 62 | logger.debug(msg) 63 | raise TimeoutError() 64 | 65 | 66 | def recv(socket, max_retries=3, retry_sleep=0.1): 67 | retries = 0 68 | while retries < max_retries: 69 | try: 70 | logger.debug('receive') 71 | return socket.recv(zmq.NOBLOCK) 72 | except zmq.ZMQError, e: 73 | logger.debug('Failed on recv()') 74 | logger.debug(str(e)) 75 | if e.errno in (zmq.EFSM, zmq.EAGAIN): 76 | retries += 1 77 | time.sleep(retry_sleep) 78 | else: 79 | raise 80 | 81 | logger.debug('Receiving failed') 82 | raise TimeoutError() 83 | 84 | 85 | def set_logger(debug=False, name='powerhose', logfile='stdout'): 86 | # setting up the logger 87 | logger_ = logging.getLogger(name) 88 | logger_.setLevel(logging.DEBUG) 89 | 90 | if logfile == 'stdout': 91 | ch = logging.StreamHandler() 92 | else: 93 | ch = logging.handlers.RotatingFileHandler(logfile, mode='a+') 94 | 95 | if debug: 96 | ch.setLevel(logging.DEBUG) 97 | else: 98 | ch.setLevel(logging.INFO) 99 | 100 | formatter = logging.Formatter('[%(asctime)s][%(name)s] %(message)s') 101 | ch.setFormatter(formatter) 102 | logger_.addHandler(ch) 103 | 104 | 105 | # taken from distutils2 106 | def resolve_name(name): 107 | """Resolve a name like ``module.object`` to an object and return it. 108 | 109 | This functions supports packages and attributes without depth limitation: 110 | ``package.package.module.class.class.function.attr`` is valid input. 111 | However, looking up builtins is not directly supported: use 112 | ``__builtin__.name``. 113 | 114 | Raises ImportError if importing the module fails or if one requested 115 | attribute is not found. 116 | """ 117 | if '.' not in name: 118 | # shortcut 119 | __import__(name) 120 | return sys.modules[name] 121 | 122 | # FIXME clean up this code! 123 | parts = name.split('.') 124 | cursor = len(parts) 125 | module_name = parts[:cursor] 126 | ret = '' 127 | 128 | while cursor > 0: 129 | try: 130 | ret = __import__('.'.join(module_name)) 131 | break 132 | except ImportError: 133 | cursor -= 1 134 | module_name = parts[:cursor] 135 | 136 | if ret == '': 137 | raise ImportError(parts[0]) 138 | 139 | for part in parts[1:]: 140 | try: 141 | ret = getattr(ret, part) 142 | except AttributeError, exc: 143 | raise ImportError(exc) 144 | 145 | return ret 146 | 147 | 148 | if sys.platform == "win32": 149 | timer = time.clock 150 | else: 151 | timer = time.time 152 | 153 | 154 | def timed(debug=False): 155 | def _timed(func): 156 | def __timed(*args, **kw): 157 | start = timer() 158 | try: 159 | res = func(*args, **kw) 160 | finally: 161 | duration = timer() - start 162 | if debug: 163 | logger.debug('%.4f' % duration) 164 | return duration, res 165 | return __timed 166 | return _timed 167 | 168 | 169 | def decode_params(params): 170 | """Decode a string into a dict. This is mainly useful when passing a dict 171 | trough the command line. 172 | 173 | The params passed in "params" should be in the form of key:value, separated 174 | by a pipe, the output is a python dict. 175 | """ 176 | output_dict = {} 177 | for items in params.split('|'): 178 | key, value = items.split(':') 179 | output_dict[key] = value 180 | return output_dict 181 | 182 | 183 | def encode_params(intput_dict): 184 | """Convert the dict given in input into a string of key:value separated 185 | with pipes, like spam:yeah|eggs:blah 186 | """ 187 | return '|'.join([':'.join(i) for i in intput_dict.items()]) 188 | 189 | 190 | def get_params(): 191 | return PARAMS 192 | 193 | 194 | def extract_result(data): 195 | data = data.split(':', 2) 196 | if len(data) != 3: 197 | raise ValueError("Wrong data: %s" % data) 198 | pid, result, data = data 199 | return long(pid), result == 'OK', data 200 | 201 | 202 | def dump_stacks(): 203 | dump = [] 204 | 205 | # threads 206 | threads = dict([(th.ident, th.name) 207 | for th in threading.enumerate()]) 208 | 209 | for thread, frame in sys._current_frames().items(): 210 | dump.append('Thread 0x%x (%s)\n' % (thread, threads[thread])) 211 | dump.append(''.join(traceback.format_stack(frame))) 212 | dump.append('\n') 213 | 214 | # greenlets 215 | try: 216 | from greenlet import greenlet 217 | except ImportError: 218 | return dump 219 | 220 | # if greenlet is present, let's dump each greenlet stack 221 | for ob in gc.get_objects(): 222 | if not isinstance(ob, greenlet): 223 | continue 224 | if not ob: 225 | continue # not running anymore or not started 226 | dump.append('Greenlet\n') 227 | dump.append(''.join(traceback.format_stack(ob.gr_frame))) 228 | dump.append('\n') 229 | 230 | return dump 231 | 232 | 233 | def verify_broker(broker_endpoint=DEFAULT_FRONTEND, timeout=1.): 234 | """ Return True if there's a working broker bound at broker_endpoint 235 | """ 236 | from powerhose.client import Client 237 | client = Client(broker_endpoint) 238 | try: 239 | return client.ping(timeout) 240 | finally: 241 | client.close() 242 | 243 | 244 | def kill_ghost_brokers(broker_endpoint=DEFAULT_FRONTEND, timeout=1.): 245 | """Kills ghost brokers. 246 | 247 | Return a pid, pids tuple. The first pid is the running broker 248 | and the second is a list of pids that where killed. 249 | """ 250 | # checking if there's a working broker 251 | working_broker = verify_broker(broker_endpoint, timeout) 252 | if working_broker is not None: 253 | working_broker = int(working_broker) 254 | 255 | # listing running brokers and killing the ghost ones 256 | killed = [] 257 | import psutil 258 | for pid in psutil.get_pid_list(): 259 | if pid in (os.getpid(), os.getppid()): 260 | continue 261 | 262 | p = psutil.Process(pid) 263 | try: 264 | cmd = ' '.join(p.cmdline) 265 | except psutil.error.AccessDenied: 266 | continue 267 | 268 | cmd = cmd.replace('-', '.') 269 | 270 | if 'powerhose.broker' not in cmd or pid == working_broker: 271 | continue 272 | 273 | killed.append(pid) 274 | p.terminate() 275 | 276 | return working_broker, killed 277 | -------------------------------------------------------------------------------- /powerhose/worker.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 3 | # You can obtain one at http://mozilla.org/MPL/2.0/. 4 | import os 5 | import errno 6 | import time 7 | import sys 8 | import traceback 9 | import argparse 10 | import logging 11 | import threading 12 | import Queue 13 | import contextlib 14 | import random 15 | 16 | import zmq 17 | 18 | from powerhose import util 19 | from powerhose.util import (logger, set_logger, DEFAULT_BACKEND, 20 | DEFAULT_HEARTBEAT, DEFAULT_REG) 21 | from powerhose.job import Job 22 | from powerhose.util import resolve_name, decode_params, timed, dump_stacks 23 | from powerhose.heartbeat import Stethoscope 24 | from powerhose.client import DEFAULT_TIMEOUT_MOVF 25 | 26 | from zmq.eventloop import ioloop, zmqstream 27 | 28 | 29 | DEFAULT_MAX_AGE = -1 30 | DEFAULT_MAX_AGE_DELTA = 0 31 | 32 | 33 | class ExecutionTimer(threading.Thread): 34 | 35 | def __init__(self, timeout=DEFAULT_TIMEOUT_MOVF, interval=.1): 36 | logger.debug('Initializing the execution timer. timeout is %.2f' \ 37 | % timeout) 38 | threading.Thread.__init__(self) 39 | self.armed = self.running = False 40 | self.timeout = timeout 41 | self.daemon = True 42 | 43 | # creating a queue for I/O with the worker 44 | self.queue = Queue.Queue() 45 | self.interval = interval 46 | self.timed_out = self.working = False 47 | self.last_dump = None 48 | 49 | @contextlib.contextmanager 50 | def run_job(self): 51 | self.job_starts() 52 | try: 53 | yield 54 | finally: 55 | self.job_ends() 56 | 57 | def job_starts(self): 58 | if self.working: 59 | raise ValueError("The worker is already busy -- call job_ends") 60 | self.working = True 61 | self.timed_out = False 62 | self.queue.put('STARTING') 63 | 64 | def job_ends(self): 65 | if not self.working: 66 | raise ValueError("The worker is not busy -- call job_starts") 67 | self.queue.put('DONE') 68 | self.working = self.armed = False 69 | 70 | def run(self): 71 | self.running = True 72 | 73 | while self.running: 74 | # arming, so waiting for ever 75 | self.queue.get() 76 | self.armed = True 77 | 78 | # now waiting for the second call, which means 79 | # the worker has done the work. 80 | # 81 | # This time we time out 82 | try: 83 | self.queue.get(timeout=self.timeout) 84 | except Queue.Empty: 85 | # too late, we want to log the stack 86 | self.last_dump = dump_stacks() 87 | self.timed_out = True 88 | finally: 89 | self.armed = False 90 | 91 | def stop(self): 92 | self.running = False 93 | if not self.armed: 94 | self.queue.put('STARTING') 95 | self.queue.put('DONE') 96 | if self.isAlive(): 97 | self.join() 98 | 99 | 100 | class Worker(object): 101 | """Class that links a callable to a broker. 102 | 103 | Options: 104 | 105 | - **target**: The Python callable that will be called when the broker 106 | send a job. 107 | - **backend**: The ZMQ socket to connect to the broker. 108 | - **heartbeat**: The ZMQ socket to perform PINGs on the broker to make 109 | sure it's still alive. 110 | - **register** : the ZMQ socket to register workers 111 | - **ping_delay**: the delay in seconds betweem two pings. 112 | - **ping_retries**: the number of attempts to ping the broker before 113 | quitting. 114 | - **params** a dict containing the params to set for this worker. 115 | - **timeout** the maximum time allowed before the thread stacks is dump 116 | and the job result not sent back. 117 | - **max_age**: maximum age for a worker in seconds. After that delay, 118 | the worker will simply quit. When set to -1, never quits. 119 | Defaults to -1. 120 | - **max_age_delta**: maximum value in seconds added to max age. 121 | The Worker will quit after *max_age + random(0, max_age_delta)* 122 | This is done to avoid having all workers quit at the same instant. 123 | Defaults to 0. The value must be an integer. 124 | """ 125 | def __init__(self, target, backend=DEFAULT_BACKEND, 126 | heartbeat=DEFAULT_HEARTBEAT, register=DEFAULT_REG, 127 | ping_delay=10., ping_retries=3, 128 | params=None, timeout=DEFAULT_TIMEOUT_MOVF, 129 | max_age=DEFAULT_MAX_AGE, max_age_delta=DEFAULT_MAX_AGE_DELTA): 130 | logger.debug('Initializing the worker.') 131 | self.ctx = zmq.Context() 132 | self.backend = backend 133 | self._reg = self.ctx.socket(zmq.PUSH) 134 | self._reg.connect(register) 135 | self._backend = self.ctx.socket(zmq.REP) 136 | self._backend.identity = str(os.getpid()) 137 | self._backend.connect(self.backend) 138 | self.target = target 139 | self.running = False 140 | self.loop = ioloop.IOLoop() 141 | self._backstream = zmqstream.ZMQStream(self._backend, self.loop) 142 | self._backstream.on_recv(self._handle_recv_back) 143 | self.ping = Stethoscope(heartbeat, onbeatlost=self.lost, 144 | delay=ping_delay, retries=ping_retries, 145 | ctx=self.ctx) 146 | self.debug = logger.isEnabledFor(logging.DEBUG) 147 | self.params = params 148 | self.pid = os.getpid() 149 | self.timeout = timeout 150 | self.timer = ExecutionTimer(timeout=timeout) 151 | self.max_age = max_age 152 | self.max_age_delta = max_age_delta 153 | self.delayed_exit = None 154 | self.lock = threading.RLock() 155 | 156 | def _handle_recv_back(self, msg): 157 | # do the job and send the result 158 | if self.debug: 159 | logger.debug('Job received') 160 | target = timed()(self.target) 161 | else: 162 | target = self.target 163 | 164 | duration = -1 165 | 166 | # results are sent with a PID:OK: or a PID:ERROR prefix 167 | try: 168 | with self.timer.run_job(): 169 | res = target(Job.load_from_string(msg[0])) 170 | 171 | # did we timout ? 172 | if self.timer.timed_out: 173 | # let's dump the last 174 | for line in self.timer.last_dump: 175 | logger.error(line) 176 | 177 | if self.debug: 178 | duration, res = res 179 | res = '%d:OK:%s' % (self.pid, res) 180 | except Exception, e: 181 | exc_type, exc_value, exc_traceback = sys.exc_info() 182 | exc = traceback.format_tb(exc_traceback) 183 | exc.insert(0, str(e)) 184 | res = '%d:ERROR:%s' % (self.pid, '\n'.join(exc)) 185 | logger.error(res) 186 | 187 | if self.timer.timed_out: 188 | # let's not send back anything, we know the client 189 | # is gone anyway 190 | return 191 | 192 | if self.debug: 193 | logger.debug('Duration - %.6f' % duration) 194 | 195 | try: 196 | self._backstream.send(res) 197 | except Exception: 198 | logging.error("Could not send back the result", exc_info=True) 199 | 200 | def lost(self): 201 | logger.info('Master lost ! Quitting..') 202 | self.running = False 203 | self.loop.stop() 204 | return True 205 | 206 | def stop(self): 207 | """Stops the worker. 208 | """ 209 | if not self.running: 210 | return 211 | 212 | # telling the broker we are stopping 213 | try: 214 | self._reg.send_multipart(['UNREGISTER', str(os.getpid())]) 215 | except zmq.ZMQError: 216 | logger.debug('Could not unregister') 217 | 218 | # give it a chance to finish a job 219 | logger.debug('Starting the graceful period') 220 | self.graceful_delay = ioloop.DelayedCallback(self._stop, 221 | self.timeout * 1000, 222 | io_loop=self.loop) 223 | self.graceful_delay.start() 224 | 225 | def _stop(self): 226 | logger.debug('Stopping the worker') 227 | self.running = False 228 | try: 229 | self._backstream.flush() 230 | except zmq.core.error.ZMQError: 231 | pass 232 | self.loop.stop() 233 | self.ping.stop() 234 | self.timer.stop() 235 | time.sleep(.1) 236 | self.ctx.destroy(0) 237 | logger.debug('Worker is stopped') 238 | 239 | def start(self): 240 | """Starts the worker 241 | """ 242 | util.PARAMS = self.params 243 | logger.debug('Starting the worker loop') 244 | 245 | # running the pinger 246 | self.ping.start() 247 | self.timer.start() 248 | self.running = True 249 | 250 | # telling the broker we are ready 251 | self._reg.send_multipart(['REGISTER', str(os.getpid())]) 252 | 253 | # arming the exit callback 254 | if self.max_age != -1: 255 | if self.max_age_delta > 0: 256 | delta = random.randint(0, self.max_age_delta) 257 | else: 258 | delta = 0 259 | 260 | cb_time = self.max_age + delta 261 | self.delayed_exit = ioloop.DelayedCallback(self.stop, 262 | cb_time * 1000, 263 | io_loop=self.loop) 264 | self.delayed_exit.start() 265 | 266 | while self.running: 267 | try: 268 | self.loop.start() 269 | except zmq.ZMQError as e: 270 | logger.debug(str(e)) 271 | 272 | if e.errno == errno.EINTR: 273 | continue 274 | elif e.errno == zmq.ETERM: 275 | break 276 | else: 277 | logger.debug("got an unexpected error %s (%s)", str(e), 278 | e.errno) 279 | raise 280 | else: 281 | break 282 | 283 | logger.debug('Worker loop over') 284 | 285 | 286 | def main(args=sys.argv): 287 | 288 | parser = argparse.ArgumentParser(description='Run some watchers.') 289 | 290 | parser.add_argument('--backend', dest='backend', 291 | default=DEFAULT_BACKEND, 292 | help="ZMQ socket to the broker.") 293 | 294 | parser.add_argument('--register', dest='register', 295 | default=DEFAULT_REG, 296 | help="ZMQ socket for the registration.") 297 | 298 | parser.add_argument('target', help="Fully qualified name of the callable.") 299 | 300 | parser.add_argument('--debug', action='store_true', default=False, 301 | help="Debug mode") 302 | 303 | parser.add_argument('--logfile', dest='logfile', default='stdout', 304 | help="File to log in to.") 305 | 306 | parser.add_argument('--heartbeat', dest='heartbeat', 307 | default=DEFAULT_HEARTBEAT, 308 | help="ZMQ socket for the heartbeat.") 309 | 310 | parser.add_argument('--params', dest='params', default=None, 311 | help='The parameters to be used in the worker.') 312 | 313 | parser.add_argument('--timeout', dest='timeout', type=float, 314 | default=DEFAULT_TIMEOUT_MOVF, 315 | help=('The maximum time allowed before the thread ' 316 | 'stacks is dump and the job result not sent ' 317 | 'back.')) 318 | 319 | parser.add_argument('--max-age', dest='max_age', type=float, 320 | default=DEFAULT_MAX_AGE, 321 | help=('The maximum age for a worker in seconds. ' 322 | 'After that delay, the worker will simply quit. ' 323 | 'When set to -1, never quits.')) 324 | 325 | parser.add_argument('--max-age-delta', dest='max_age_delta', type=int, 326 | default=DEFAULT_MAX_AGE_DELTA, 327 | help='The maximum value in seconds added to max_age') 328 | 329 | args = parser.parse_args() 330 | set_logger(args.debug, logfile=args.logfile) 331 | sys.path.insert(0, os.getcwd()) # XXX 332 | target = resolve_name(args.target) 333 | if args.params is None: 334 | params = {} 335 | else: 336 | params = decode_params(args.params) 337 | 338 | logger.info('Worker registers at %s' % args.backend) 339 | logger.info('The heartbeat socket is at %r' % args.heartbeat) 340 | worker = Worker(target, backend=args.backend, heartbeat=args.heartbeat, 341 | register=args.register, 342 | params=params, timeout=args.timeout, max_age=args.max_age, 343 | max_age_delta=args.max_age_delta) 344 | 345 | try: 346 | worker.start() 347 | except KeyboardInterrupt: 348 | return 1 349 | finally: 350 | worker.stop() 351 | 352 | return 0 353 | 354 | 355 | if __name__ == '__main__': 356 | main() 357 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 3 | # You can obtain one at http://mozilla.org/MPL/2.0/. 4 | import os 5 | from setuptools import setup, find_packages 6 | 7 | here = os.path.abspath(os.path.dirname(__file__)) 8 | 9 | with open(os.path.join(here, 'README.rst')) as f: 10 | README = f.read() 11 | 12 | with open(os.path.join(here, 'CHANGES.rst')) as f: 13 | CHANGES = f.read() 14 | 15 | 16 | requires = ['pyzmq', 'circus'] 17 | 18 | 19 | setup(name='powerhose', 20 | version='0.8', 21 | packages=find_packages(), 22 | include_package_data=True, 23 | description='Implementation of the Request-Reply Broker pattern in ZMQ', 24 | long_description=README + '\n\n' + CHANGES, 25 | zip_safe=False, 26 | license='MPLv2.0', 27 | classifiers=[ 28 | "Programming Language :: Python", 29 | "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)" 30 | ], 31 | install_requires=requires, 32 | author='Mozilla Services', 33 | author_email='services-dev@mozilla.org', 34 | url='https://github.com/mozilla-services/powerhose', 35 | tests_require=['nose'], 36 | test_suite = 'nose.collector', 37 | entry_points=""" 38 | [console_scripts] 39 | powerhose-broker = powerhose.broker:main 40 | powerhose-worker = powerhose.worker:main 41 | powerhose = powerhose:main 42 | """) 43 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | # Tox (http://tox.testrun.org/) is a tool for running tests 2 | # in multiple virtualenvs. This configuration file will run the 3 | # test suite on all supported python versions. To use it, "pip install tox" 4 | # and then run "tox" from this directory. 5 | 6 | [tox] 7 | envlist = py26, py27 8 | 9 | [testenv] 10 | commands = python setup.py test 11 | deps = nose 12 | --------------------------------------------------------------------------------