├── .bumpversion.cfg ├── .coveragerc ├── .editorconfig ├── .gitignore ├── .travis.yml ├── AUTHORS ├── Changelog ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.rst ├── appveyor.yml ├── cyanide ├── __init__.py ├── __main__.py ├── app.py ├── bin │ ├── __init__.py │ ├── cyanide.py │ └── vagrant.py ├── compat.py ├── data.py ├── fbi.py ├── suite.py ├── suites │ ├── __init__.py │ └── default.py ├── tasks.py ├── templates.py ├── tests │ ├── __init__.py │ └── case.py └── vagrant │ ├── Vagrantfile │ ├── __init__.py │ └── provision │ ├── celeryd-init.config │ └── provision.sh ├── docs ├── Makefile ├── _static │ └── .keep ├── _templates │ └── .keep ├── changelog.rst ├── conf.py ├── copyright.rst ├── glossary.rst ├── images │ ├── favicon.ico │ └── logo.png ├── includes │ ├── installation.txt │ └── introduction.txt ├── index.rst ├── make.bat ├── reference │ ├── cyanide.app.rst │ ├── cyanide.bin.cyanide.rst │ ├── cyanide.bin.vagrant.rst │ ├── cyanide.compat.rst │ ├── cyanide.data.rst │ ├── cyanide.fbi.rst │ ├── cyanide.rst │ ├── cyanide.suite.rst │ ├── cyanide.suites.default.rst │ ├── cyanide.tasks.rst │ ├── cyanide.templates.rst │ ├── cyanide.vagrant.rst │ └── index.rst └── templates │ └── readme.txt ├── extra └── appveyor │ ├── install.ps1 │ └── run_with_compiler.cmd ├── requirements ├── default.txt ├── docs.txt ├── pkgutils.txt ├── test-ci.txt └── test.txt ├── setup.cfg ├── setup.py └── tox.ini /.bumpversion.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 1.3.0 3 | commit = True 4 | tag = True 5 | parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?P[a-z]+)? 6 | serialize = 7 | {major}.{minor}.{patch}{releaselevel} 8 | {major}.{minor}.{patch} 9 | 10 | [bumpversion:file:cyanide/__init__.py] 11 | 12 | [bumpversion:file:docs/includes/introduction.txt] 13 | 14 | [bumpversion:file:README.rst] 15 | 16 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = 1 3 | cover_pylib = 0 4 | include = *cyanide/* 5 | omit = cyanide.tests.* 6 | 7 | [report] 8 | omit = 9 | */python?.?/* 10 | */site-packages/* 11 | */pypy/* 12 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | indent_style = space 7 | indent_size = 4 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | charset = utf-8 11 | end_of_line = lf 12 | 13 | [Makefile] 14 | indent_style = tab 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.pyc 3 | *$py.class 4 | *~ 5 | .*.sw[pon] 6 | dist/ 7 | *.egg-info 8 | *.egg 9 | *.egg/ 10 | build/ 11 | .build/ 12 | _build/ 13 | pip-log.txt 14 | .directory 15 | erl_crash.dump 16 | *.db 17 | Documentation/ 18 | .tox/ 19 | .ropeproject/ 20 | .project 21 | .pydevproject 22 | .idea/ 23 | .coverage 24 | celery/tests/cover/ 25 | .ve* 26 | cover/ 27 | .vagrant/ 28 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | sudo: false 3 | cache: false 4 | python: 5 | - '3.5' 6 | env: 7 | global: 8 | PYTHONUNBUFFERED=yes 9 | matrix: 10 | - TOXENV=2.7 11 | - TOXENV=3.4 12 | - TOXENV=pypy 13 | - TOXENV=3.5 14 | - TOXENV=pypy3 15 | - TOXENV=flake8 16 | - TOXENV=flakeplus 17 | - TOXENV=apicheck 18 | install: travis_retry pip install -U tox 19 | script: tox -v -- -v 20 | after_success: 21 | - .tox/$TRAVIS_PYTHON_VERSION/bin/coverage xml 22 | - .tox/$TRAVIS_PYTHON_VERSION/bin/codecov -e TOXENV 23 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Ask Solem 2 | -------------------------------------------------------------------------------- /Changelog: -------------------------------------------------------------------------------- 1 | .. _changelog: 2 | 3 | ================ 4 | Change history 5 | ================ 6 | 7 | .. _version-1.3.0: 8 | 9 | 1.3.0 10 | ===== 11 | :release-date: 2016-10-19 12:36:00 P.M PDT 12 | 13 | - Adds cyanide.suite.ManagerMixin 14 | 15 | .. _version-1.2.0: 16 | 17 | 1.2.0 18 | ===== 19 | :release-date: 2016-06-30 11:27:00 P.M PDT 20 | 21 | - ``celery vagrant``: Updates to new RabbitMQ APT key URL. 22 | 23 | - New tasks: ``errback`` and ``old_errback`` for testing new canvas error 24 | callbacks. 25 | 26 | - Integrates tests from old Celery functional test suite. 27 | 28 | .. _version-1.1.0: 29 | 30 | 1.1.0 31 | ===== 32 | :release-date: 2016-05-12 03:07:00 P.M PDT 33 | :release-by: Ask Solem 34 | 35 | - Moved tasks from ``cyanide.app`` into :mod:`cyanide.tasks`. 36 | 37 | - Adds Suite.wait_for with progress bar. 38 | 39 | - More useful harness output. 40 | 41 | .. _version-1.0.1 42 | 43 | 1.0.1 44 | ===== 45 | :release-date: 2106-04-26 06:42:00 P.M PDT 46 | :release-by: Ask Solem 47 | 48 | - Now compatible with Celery 3.1 49 | 50 | .. _version-1.0.0 51 | 52 | 1.0.0 53 | ===== 54 | :release-date: 2106-04-26 06:17:00 P.M PST 55 | :release-by: Ask Solem 56 | 57 | - Initial release assembled from ``celery/funtests/stress/`` 58 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013-2016, Ask Solem and individual contributors. 2 | All rights reserved. 3 | 4 | Cyanide is licensed under The BSD License (3 Clause, also known as 5 | the new BSD license). The license is an OSI approved Open Source 6 | license and is GPL-compatible(1). 7 | 8 | The license text can also be found here: 9 | http://www.opensource.org/licenses/BSD-3-Clause 10 | 11 | License 12 | ======= 13 | 14 | Redistribution and use in source and binary forms, with or without 15 | modification, are permitted provided that the following conditions are met: 16 | * Redistributions of source code must retain the above copyright 17 | notice, this list of conditions and the following disclaimer. 18 | * Redistributions in binary form must reproduce the above copyright 19 | notice, this list of conditions and the following disclaimer in the 20 | documentation and/or other materials provided with the distribution. 21 | * Neither the name of Ask Solem nor the 22 | names of its contributors may be used to endorse or promote products 23 | derived from this software without specific prior written permission. 24 | 25 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 26 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 27 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 | PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL Ask Solem OR CONTRIBUTORS 29 | BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 | POSSIBILITY OF SUCH DAMAGE. 36 | 37 | Documentation License 38 | ===================== 39 | 40 | The documentation portion of Cyanide (the rendered contents of the 41 | "docs" directory of a software distribution or checkout) is supplied 42 | under the "Creative Commons Attribution-ShareAlike 4.0 43 | International" (CC BY-SA 4.0) License as described by 44 | http://creativecommons.org/licenses/by-sa/4.0/ 45 | 46 | Footnotes 47 | ========= 48 | (1) A GPL-compatible license makes it possible to 49 | combine Cyanide with other software that is released 50 | under the GPL, it does not mean that we're distributing 51 | Cyanide under the GPL license. The BSD license, unlike the GPL, 52 | let you distribute a modified version without making your 53 | changes open source. 54 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include Changelog 2 | include LICENSE 3 | include README.rst 4 | include MANIFEST.in 5 | include setup.cfg 6 | include setup.py 7 | recursive-include docs * 8 | recursive-include extra *.cmd *.ps1 9 | recursive-include examples * 10 | recursive-include requirements *.txt *.rst 11 | recursive-include cyanide/vagrant Vagrantfile *.config *.sh 12 | 13 | recursive-exclude * __pycache__ 14 | recursive-exclude * *.py[co] 15 | recursive-exclude * .*.sw[a-z] 16 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PROJ=cyanide 2 | PGPIDENT="Celery Security Team" 3 | PYTHON=python 4 | GIT=git 5 | TOX=tox 6 | NOSETESTS=nosetests 7 | ICONV=iconv 8 | FLAKE8=flake8 9 | FLAKEPLUS=flakeplus 10 | SPHINX2RST=sphinx2rst 11 | 12 | SPHINX_DIR=docs/ 13 | SPHINX_BUILDDIR="${SPHINX_DIR}/_build" 14 | README=README.rst 15 | README_SRC="docs/templates/readme.txt" 16 | CONTRIBUTING=CONTRIBUTING.rst 17 | CONTRIBUTING_SRC="docs/contributing.rst" 18 | SPHINX_HTMLDIR="${SPHINX_BUILDDIR}/html" 19 | DOCUMENTATION=Documentation 20 | FLAKEPLUSTARGET=2.7 21 | 22 | all: help 23 | 24 | help: 25 | @echo "docs - Build documentation." 26 | @echo "test-all - Run tests for all supported python versions." 27 | @echo "distcheck ---------- - Check distribution for problems." 28 | @echo " test - Run unittests using current python." 29 | @echo " lint ------------ - Check codebase for problems." 30 | @echo " apicheck - Check API reference coverage." 31 | @echo " configcheck - Check configuration reference coverage." 32 | @echo " readmecheck - Check README.rst encoding." 33 | @echo " contribcheck - Check CONTRIBUTING.rst encoding" 34 | @echo " flakes -------- - Check code for syntax and style errors." 35 | @echo " flakecheck - Run flake8 on the source code." 36 | @echo " flakepluscheck - Run flakeplus on the source code." 37 | @echo "readme - Regenerate README.rst file." 38 | @echo "contrib - Regenerate CONTRIBUTING.rst file" 39 | @echo "clean-dist --------- - Clean all distribution build artifacts." 40 | @echo " clean-git-force - Remove all uncomitted files." 41 | @echo " clean ------------ - Non-destructive clean" 42 | @echo " clean-pyc - Remove .pyc/__pycache__ files" 43 | @echo " clean-docs - Remove documentation build artifacts." 44 | @echo " clean-build - Remove setup artifacts." 45 | @echo "bump - Bump patch version number." 46 | @echo "bump-minor - Bump minor version number." 47 | @echo "bump-major - Bump major version number." 48 | @echo "release - Make PyPI release." 49 | 50 | clean: clean-docs clean-pyc clean-build 51 | 52 | clean-dist: clean clean-git-force 53 | 54 | bump: 55 | bumpversion patch 56 | 57 | bump-minor: 58 | bumpversion minor 59 | 60 | bump-major: 61 | bumpversion major 62 | 63 | release: 64 | python setup.py register sdist bdist_wheel upload --sign --identity="$(PGPIDENT)" 65 | 66 | Documentation: 67 | (cd "$(SPHINX_DIR)"; $(MAKE) html) 68 | mv "$(SPHINX_HTMLDIR)" $(DOCUMENTATION) 69 | 70 | docs: Documentation 71 | 72 | clean-docs: 73 | -rm -rf "$(SPHINX_BUILDDIR)" 74 | 75 | lint: flakecheck apicheck configcheck readmecheck 76 | 77 | apicheck: 78 | (cd "$(SPHINX_DIR)"; $(MAKE) apicheck) 79 | 80 | configcheck: 81 | (cd "$(SPHINX_DIR)"; $(MAKE) configcheck) 82 | 83 | flakecheck: 84 | $(FLAKE8) --ignore=X999 "$(PROJ)" 85 | 86 | flakediag: 87 | -$(MAKE) flakecheck 88 | 89 | flakepluscheck: 90 | $(FLAKEPLUS) --$(FLAKEPLUSTARGET) "$(PROJ)" 91 | 92 | flakeplusdiag: 93 | -$(MAKE) flakepluscheck 94 | 95 | flakes: flakediag flakeplusdiag 96 | 97 | clean-readme: 98 | -rm -f $(README) 99 | 100 | readmecheck: 101 | $(ICONV) -f ascii -t ascii $(README) >/dev/null 102 | 103 | $(README): 104 | $(SPHINX2RST) "$(README_SRC)" --ascii > $@ 105 | 106 | readme: clean-readme $(README) readmecheck 107 | 108 | clean-contrib: 109 | -rm -f "$(CONTRIBUTING)" 110 | 111 | $(CONTRIBUTING): 112 | $(SPHINX2RST) "$(CONTRIBUTING_SRC)" > $@ 113 | 114 | contrib: clean-contrib $(CONTRIBUTING) 115 | 116 | clean-pyc: 117 | -find . -type f -a \( -name "*.pyc" -o -name "*$$py.class" \) | xargs rm 118 | -find . -type d -name "__pycache__" | xargs rm -r 119 | 120 | removepyc: clean-pyc 121 | 122 | clean-build: 123 | rm -rf build/ dist/ .eggs/ *.egg-info/ .tox/ .coverage cover/ 124 | 125 | clean-git: 126 | $(GIT) clean -xdn 127 | 128 | clean-git-force: 129 | $(GIT) clean -xdf 130 | 131 | test-all: clean-pyc 132 | $(TOX) 133 | 134 | test: 135 | $(PYTHON) setup.py test 136 | 137 | cov: 138 | $(NOSETESTS) -xv --with-coverage --cover-html --cover-branch 139 | 140 | build: 141 | $(PYTHON) setup.py sdist bdist_wheel 142 | 143 | distcheck: lint test clean 144 | 145 | dist: readme contrib clean-dist build 146 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ===================================================================== 2 | Cyanide - Celery stress testing and integration test support. 3 | ===================================================================== 4 | 5 | |build-status| |coverage| |license| |wheel| |pyversion| |pyimp| 6 | 7 | :Version: 1.3.0 8 | :Web: https://cyanide.readthedocs.io/ 9 | :Download: http://pypi.python.org/pypi/cyanide/ 10 | :Source: http://github.com/celery/cyanide/ 11 | :Keywords: celery, stress, integration, functional, testing 12 | 13 | .. contents:: 14 | :local: 15 | 16 | Introduction 17 | ============ 18 | 19 | This stress test suite will attempt to break the Celery worker in different 20 | ways, and can also be used to write new stress test suites for projects 21 | depending on Celery infrastructure. 22 | 23 | The worker must currently be started separately, and it's encouraged 24 | to repeat the suite using workers started with different configuration values. 25 | 26 | Ideas include: 27 | 28 | #. Default, single process: 29 | :: 30 | 31 | $ celery -A cyanide worker -c 1 32 | 33 | #. Default, multiple processes: 34 | :: 35 | 36 | $ celery -A cyanide worker -c 8 37 | 38 | #. Frequent ``maxtasksperchild`` recycling, single child process: 39 | :: 40 | 41 | $ celery -A cyanide worker -c 1 --maxtasksperchild=1 42 | 43 | #. Frequent autoscale scale down & ``maxtasksperchild``, single child process: 44 | :: 45 | 46 | $ AUTOSCALE_KEEPALIVE=0.01 celery -A cyanide worker \ 47 | > --autoscale=1,0 --maxtasksperchild=1 48 | 49 | #. Frequent ``maxtasksperchild``, multiple child processes: 50 | :: 51 | 52 | $ celery -A cyanide worker -c 8 --maxtasksperchild=1 53 | 54 | #. Processes terminated by time limits: 55 | :: 56 | 57 | $ celery -A cyanide worker --time-limit=1 58 | 59 | #. Frequent ``maxtasksperchild``, single child process with late ack: 60 | :: 61 | 62 | $ celery -A cyanide worker -c1 --maxtasksperchild=1 -Z acks_late 63 | 64 | #. Worker using the ``eventlet`` pool: 65 | 66 | Start the worker, here having a thousand green-threads: 67 | :: 68 | 69 | $ celery -A cyanide worker -c1000 -P eventlet 70 | 71 | You must activate the `green` test group when starting the test suite: 72 | :: 73 | 74 | $ celery cyanide -g green 75 | 76 | #. Worker using the ``gevent`` pool: 77 | 78 | Start the worker, here having a thousand green-threads: 79 | :: 80 | 81 | $ celery -A cyanide worker -c1000 -P gevent 82 | 83 | You must activate the `green` test group when starting the test suite: 84 | :: 85 | 86 | $ celery cyanide -g green 87 | 88 | Tips 89 | ==== 90 | 91 | It's a good idea to include the ``--purge `` 92 | argument to clear out tasks from previous runs. 93 | 94 | Note that the stress client will probably hang if the test fails, so this 95 | test suite is currently not suited for automatic runs. 96 | 97 | Configuration Templates 98 | ======================= 99 | 100 | You can select a configuration template using the `-Z` command-line argument 101 | to any ``celery -A cyanide`` command or the ``celery cyanide`` 102 | command used to execute the test suite. 103 | 104 | The templates available are: 105 | 106 | * ``default`` 107 | 108 | Using AMQP as a broker, RPC as a result backend, 109 | and using JSON serialization for task and result messages. 110 | 111 | Both broker and result store is expected to run at localhost. 112 | 113 | * ``vagrant1`` 114 | 115 | Use the VM started by ``celery vagrant up`` as the broker 116 | and result backend (RabbitMQ). 117 | 118 | * ``vagrant1_redis`` 119 | 120 | Use the VM started by ``celery vagrant up`` as the broker 121 | and result backend (Redis). 122 | 123 | * ``redis`` 124 | 125 | Using Redis as a broker and result backend. 126 | 127 | * ``redistore`` 128 | 129 | Using Redis as a result backend only. 130 | 131 | * ``acks_late`` 132 | 133 | Enables late ack globally. 134 | 135 | * ``pickle`` 136 | 137 | Using pickle as the serializer for tasks and results 138 | (also allowing the worker to receive and process pickled messages) 139 | 140 | * ``confirms`` 141 | 142 | Enables RabbitMQ publisher confirmations. 143 | 144 | * ``events`` 145 | 146 | Configure workers to send task events. 147 | 148 | * ``proto1`` 149 | 150 | Use version 1 of the task message protocol (pre 4.0) 151 | 152 | You can see the resulting configuration from any template by running 153 | the command: 154 | :: 155 | 156 | $ celery -A cyanide report -Z redis 157 | 158 | Examples 159 | -------- 160 | 161 | Example running the stress test using the ``redis`` configuration template: 162 | :: 163 | 164 | $ cyanide -Z redis 165 | 166 | Example running the worker using the ``redis`` configuration template: 167 | :: 168 | 169 | $ celery -A cyanide worker -Z redis 170 | 171 | You can also mix several templates by providing a comma-separated list: 172 | :: 173 | 174 | $ celery -A cyanide worker -Z redis,acks_late 175 | 176 | In this example (``redis,acks_late``) the ``redis`` template will be used 177 | as main configuration, and then the additional keys from the ``acks_late`` template 178 | will be merged as changes. 179 | 180 | Test Suite Options 181 | ================== 182 | 183 | After one or more worker instances are running, you can start executing the 184 | tests. 185 | 186 | By default the complete test suite will be executed: 187 | :: 188 | 189 | $ celery cyanide 190 | 191 | You can also specify what test cases to run by providing one or more names 192 | as arguments: 193 | :: 194 | 195 | $ celery cyanide revoketermfast revoketermslow 196 | 197 | A full list of test case names can be retrieved with the 198 | ``-l `` switch: 199 | :: 200 | 201 | $ celery cyanide -l 202 | .> 1) chain, 203 | .> 2) chaincomplex, 204 | .> 3) parentids_chain, 205 | .> 4) parentids_group, 206 | .> 5) manyshort, 207 | .> 6) unicodetask, 208 | .> 7) always_timeout, 209 | .> 8) termbysig, 210 | .> 9) timelimits, 211 | .> 10) timelimits_soft, 212 | .> 11) alwayskilled, 213 | .> 12) alwaysexits, 214 | .> 13) bigtasksbigvalue, 215 | .> 14) bigtasks, 216 | .> 15) smalltasks, 217 | .> 16) revoketermfast, 218 | .> 17) revoketermslow 219 | 220 | You can also start from an offset within this list, e.g. to skip the first two 221 | tests use ``--offset=2 ``: 222 | :: 223 | 224 | $ celery cyanide --offset=2 225 | 226 | See ``celery cyanide --help`` for a list of all available 227 | command-line options. 228 | 229 | Vagrant 230 | ======= 231 | 232 | Starting 233 | -------- 234 | 235 | Cyanide ships with a complete virtual machine solution to run your tests. 236 | The image ships with Celery, Cyanide, RabbitMQ and Redis and can be deployed 237 | simply by running the ``celery vagrant`` command: 238 | :: 239 | 240 | $ celery vagrant up 241 | 242 | 243 | The IP address of the new virtual machine will be 192.168.33.123, 244 | and you can easily tell both the worker and cyanide test suite to use 245 | it by specifying the ``vagrant1`` (RabbitMQ) or ``vagrant1_redis`` templates: 246 | :: 247 | 248 | $ celery -A worker -Z vagrant1 249 | $ celery cyanide -Z vagrant1 250 | 251 | SSH 252 | --- 253 | 254 | To open an SSH session with the virtual machine after starting 255 | with ``celery vagrant up`` do: 256 | :: 257 | 258 | $ ssh $(celery vagrant sshargs) 259 | 260 | Stopping 261 | -------- 262 | 263 | To shutdown the virtual machine run the command: 264 | :: 265 | 266 | $ celery vagrant halt 267 | 268 | To destroy the instance run the command: 269 | :: 270 | 271 | $ celery vagrant destroy 272 | 273 | 274 | .. note:: 275 | 276 | To completely wipe your instance you need to remove the 277 | ``.vagrant`` directory. 278 | 279 | The location of this directory can be retrieved by executing 280 | the following: 281 | :: 282 | 283 | $ celery vagrant statedir 284 | /opt/devel/cyanide/cyanide/vagrant/.vagrant 285 | 286 | You can combine this with ``rm`` to force removal of this 287 | directory: 288 | :: 289 | 290 | $ rm -rf $(celery vagrant statedir) 291 | 292 | Environment Variables 293 | ===================== 294 | 295 | ``CYANIDE_TRANS`` 296 | ----------------- 297 | 298 | If the ``CYANIDE_TRANS`` environment variable is set 299 | the stress test suite will use transient task messages instead of persisting 300 | messages to disk. 301 | 302 | To avoid declaration collision the ``cstress.trans`` queue name will be used 303 | when this option is enabled. 304 | 305 | ``CYANIDE_BROKER`` 306 | ------------------ 307 | 308 | You can set the ``CYANIDE_BROKER`` environment variable 309 | to change the default broker used: 310 | :: 311 | 312 | $ CYANIDE_BROKER='amqp://' celery -A cyanide worker # ... 313 | $ CYANIDE_BROKER='amqp://' celery cyanide 314 | 315 | ``CYANIDE_BACKEND`` 316 | ------------------- 317 | 318 | You can set the ``CYANIDE_BACKEND`` environment variable to change 319 | the result backend used: 320 | :: 321 | 322 | $ CYANIDE_BACKEND='amqp://' celery -A cyanide worker # ... 323 | $ CYANIDE_BACKEND='amqp://' celery cyanide 324 | 325 | ``CYANIDE_QUEUE`` 326 | ----------------- 327 | 328 | A queue named ``c.stress`` is created and used by default for all task 329 | communication. 330 | 331 | You can change the name of this queue using the ``CYANIDE_QUEUE`` 332 | environment variable: 333 | :: 334 | 335 | $ CYANIDE_QUEUE='cyanide' celery -A cyanide worker # ... 336 | $ CYANIDE_QUEUE='cyanide' celery cyanide 337 | 338 | ``CYANIDE_PREFETCH`` 339 | -------------------- 340 | 341 | The ``CYANIDE_PREFETCH`` environment variable sets the default prefetch 342 | multiplier (default value is 10). 343 | 344 | ``AWS_REGION`` 345 | -------------- 346 | 347 | The ``AWS_REGION`` environment variable changes the Amazon AWS region 348 | to something other than the default ``us-east-1``, to be used with the 349 | ``sqs`` template. 350 | 351 | 352 | Custom Suites 353 | ============= 354 | 355 | You can define custom suites (look at source code of 356 | ``cyanide.suites.default`` for inspiration), and tell cyanide to use that 357 | suite by specifying the ``celery cyanide -S`` option: 358 | :: 359 | 360 | $ celery cyanide -S proj.funtests:MySuite 361 | 362 | .. _installation: 363 | 364 | Installation 365 | ============ 366 | 367 | You can install cyanide either via the Python Package Index (PyPI) 368 | or from source. 369 | 370 | To install using `pip`: 371 | :: 372 | 373 | $ pip install -U cyanide 374 | 375 | .. _installing-from-source: 376 | 377 | Downloading and installing from source 378 | -------------------------------------- 379 | 380 | Download the latest version of cyanide from 381 | http://pypi.python.org/pypi/cyanide 382 | 383 | You can install it by doing the following: 384 | :: 385 | 386 | $ tar xvfz cyanide-0.0.0.tar.gz 387 | $ cd cyanide-0.0.0 388 | $ python setup.py build 389 | # python setup.py install 390 | 391 | The last command must be executed as a privileged user if 392 | you are not currently using a virtualenv. 393 | 394 | .. _installing-from-git: 395 | 396 | Using the development version 397 | ----------------------------- 398 | 399 | With pip 400 | ~~~~~~~~ 401 | 402 | You can install the latest snapshot of cyanide using the following 403 | pip command: 404 | :: 405 | 406 | $ pip install https://github.com/celery/cyanide/zipball/master#egg=cyanide 407 | 408 | .. |build-status| image:: https://secure.travis-ci.org/celery/cyanide.png?branch=master 409 | :alt: Build status 410 | :target: https://travis-ci.org/celery/cyanide 411 | 412 | .. |coverage| image:: https://codecov.io/github/celery/cyanide/coverage.svg?branch=master 413 | :target: https://codecov.io/github/celery/cyanide?branch=master 414 | 415 | .. |license| image:: https://img.shields.io/pypi/l/cyanide.svg 416 | :alt: BSD License 417 | :target: https://opensource.org/licenses/BSD-3-Clause 418 | 419 | .. |wheel| image:: https://img.shields.io/pypi/wheel/cyanide.svg 420 | :alt: Cyanide can be installed via wheel 421 | :target: http://pypi.python.org/pypi/cyanide/ 422 | 423 | .. |pyversion| image:: https://img.shields.io/pypi/pyversions/cyanide.svg 424 | :alt: Supported Python versions. 425 | :target: http://pypi.python.org/pypi/cyanide/ 426 | 427 | .. |pyimp| image:: https://img.shields.io/pypi/implementation/cyanide.svg 428 | :alt: Support Python implementations. 429 | :target: http://pypi.python.org/pypi/cyanide/ 430 | 431 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | environment: 2 | 3 | global: 4 | # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the 5 | # /E:ON and /V:ON options are not enabled in the batch script intepreter 6 | # See: http://stackoverflow.com/a/13751649/163740 7 | WITH_COMPILER: "cmd /E:ON /V:ON /C .\\extra\\appveyor\\run_with_compiler.cmd" 8 | 9 | matrix: 10 | 11 | # Pre-installed Python versions, which Appveyor may upgrade to 12 | # a later point release. 13 | # See: http://www.appveyor.com/docs/installed-software#python 14 | 15 | - PYTHON: "C:\\Python27" 16 | PYTHON_VERSION: "2.7.x" 17 | PYTHON_ARCH: "32" 18 | 19 | - PYTHON: "C:\\Python34" 20 | PYTHON_VERSION: "3.4.x" 21 | PYTHON_ARCH: "32" 22 | 23 | - PYTHON: "C:\\Python27-x64" 24 | PYTHON_VERSION: "2.7.x" 25 | PYTHON_ARCH: "64" 26 | WINDOWS_SDK_VERSION: "v7.0" 27 | 28 | - PYTHON: "C:\\Python34-x64" 29 | PYTHON_VERSION: "3.4.x" 30 | PYTHON_ARCH: "64" 31 | WINDOWS_SDK_VERSION: "v7.1" 32 | 33 | 34 | init: 35 | - "ECHO %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH%" 36 | 37 | install: 38 | - "powershell extra\\appveyor\\install.ps1" 39 | - "%PYTHON%/Scripts/pip.exe install -U setuptools" 40 | 41 | build: off 42 | 43 | test_script: 44 | - "%WITH_COMPILER% %PYTHON%/python setup.py test" 45 | 46 | after_test: 47 | - "%WITH_COMPILER% %PYTHON%/python setup.py bdist_wheel" 48 | 49 | artifacts: 50 | - path: dist\* 51 | 52 | #on_success: 53 | # - TODO: upload the content of dist/*.whl to a public wheelhouse 54 | -------------------------------------------------------------------------------- /cyanide/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Celery stress testing and integration test support.""" 3 | # :copyright: (c) 2013-2016, Ask Solem. 4 | # All rights reserved. 5 | # :license: BSD (3 Clause), see LICENSE for more details. 6 | 7 | from __future__ import absolute_import, unicode_literals 8 | 9 | import re 10 | 11 | from collections import namedtuple 12 | 13 | # data must be imported first to install json serializer 14 | from . import data # noqa 15 | from .app import app as celery_app # noqa 16 | 17 | __version__ = '1.3.0' 18 | __author__ = 'Ask Solem' 19 | __contact__ = 'ask@celeryproject.org' 20 | __homepage__ = 'https://github.com/celery/cyanide' 21 | __docformat__ = 'restructuredtext' 22 | 23 | # -eof meta- 24 | 25 | version_info_t = namedtuple('version_info_t', ( 26 | 'major', 'minor', 'micro', 'releaselevel', 'serial' 27 | )) 28 | 29 | # bumpversion can only search for {current_version} 30 | # so we have to parse the version here. 31 | _temp = re.match( 32 | r'(\d+)\.(\d+).(\d+)(.+)?', __version__).groups() 33 | VERSION = version_info = version_info_t( 34 | int(_temp[0]), int(_temp[1]), int(_temp[2]), _temp[3] or '', '') 35 | del(_temp) 36 | del(re) 37 | 38 | __all__ = [] 39 | -------------------------------------------------------------------------------- /cyanide/__main__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from .bin.cyanide import main 4 | 5 | main() 6 | -------------------------------------------------------------------------------- /cyanide/app.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, print_function, unicode_literals 2 | 3 | import celery 4 | 5 | from celery import signals 6 | from celery.bin.base import Option 7 | 8 | from .templates import use_template, template_names 9 | 10 | IS_CELERY_4 = celery.VERSION[0] >= 4 11 | 12 | 13 | class App(celery.Celery): 14 | cyanide_suite = 'cyanide.suites.default:Default' 15 | template_selected = False 16 | 17 | def __init__(self, *args, **kwargs): 18 | self.template = kwargs.pop('template', None) 19 | super(App, self).__init__(*args, **kwargs) 20 | self.user_options['preload'].add( 21 | Option( 22 | '-Z', '--template', default='default', type=str, 23 | help='Configuration template to use: {0}'.format( 24 | template_names(), 25 | ), 26 | ) 27 | ) 28 | signals.user_preload_options.connect(self.on_preload_parsed) 29 | if IS_CELERY_4: 30 | self.on_configure.connect(self._maybe_use_default_template) 31 | 32 | def on_preload_parsed(self, options=None, **kwargs): 33 | self.use_template(options['template']) 34 | 35 | def use_template(self, name='default'): 36 | if self.template_selected: 37 | raise RuntimeError('App already configured') 38 | self.template_selected = True 39 | use_template(self, name) 40 | 41 | def _maybe_use_default_template(self, **kwargs): 42 | if not self.template_selected: 43 | self.use_template('default') 44 | 45 | if not IS_CELERY_4: 46 | after_configure = None 47 | 48 | def _get_config(self): 49 | ret = super(App, self)._get_config() 50 | if self.after_configure: 51 | self.after_configure(ret) 52 | return ret 53 | 54 | def on_configure(self): 55 | self._maybe_use_default_template() 56 | 57 | app = App('cyanide', set_as_current=False) 58 | -------------------------------------------------------------------------------- /cyanide/bin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/celery/cyanide/9b7d78dd4d804ed74c50f0ab9492f19ac06dc9ca/cyanide/bin/__init__.py -------------------------------------------------------------------------------- /cyanide/bin/cyanide.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, print_function, unicode_literals 2 | 3 | from celery.bin.base import Command, Option 4 | from celery.utils.imports import symbol_by_name 5 | 6 | from cyanide.app import app as cyanide_app 7 | 8 | 9 | class cyanide(Command): 10 | 11 | def __init__(self, app=None, *args, **kwargs): 12 | if app is None or app.main == 'default': 13 | app = cyanide_app 14 | app.set_current() 15 | app.set_default() 16 | super(cyanide, self).__init__(app, *args, **kwargs) 17 | 18 | def run(self, *names, **options): 19 | try: 20 | return self.run_suite(names, **options) 21 | except KeyboardInterrupt: 22 | print('###interrupted by user: exiting...', file=self.stdout) 23 | 24 | def run_suite(self, names, suite, 25 | block_timeout=None, no_color=False, **options): 26 | return symbol_by_name(suite)( 27 | self.app, 28 | block_timeout=block_timeout, 29 | no_color=no_color, 30 | ).run(names, **options) 31 | 32 | def get_options(self): 33 | return ( 34 | Option('-i', '--iterations', type='int', default=50, 35 | help='Number of iterations for each test'), 36 | Option('-n', '--numtests', type='int', default=None, 37 | help='Number of tests to execute'), 38 | Option('-o', '--offset', type='int', default=0, 39 | help='Start at custom offset'), 40 | Option('--block-timeout', type='int', default=30 * 60), 41 | Option('-l', '--list', action='store_true', dest='list_all', 42 | default=False, help='List all tests'), 43 | Option('-r', '--repeat', type='float', default=0, 44 | help='Number of times to repeat the test suite'), 45 | Option('-g', '--group', default='all', 46 | help='Specify test group (all|green|redis)'), 47 | Option('--diag', default=False, action='store_true', 48 | help='Enable diagnostics (slow)'), 49 | Option('-J', '--no-join', default=False, action='store_true', 50 | help='Do not wait for task results'), 51 | Option('-S', '--suite', 52 | default=self.app.cyanide_suite, 53 | help='Specify test suite to execute (path to class)'), 54 | ) 55 | 56 | 57 | def main(argv=None): 58 | return cyanide().execute_from_commandline(argv=argv) 59 | 60 | 61 | if __name__ == '__main__': 62 | main() 63 | -------------------------------------------------------------------------------- /cyanide/bin/vagrant.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, print_function, unicode_literals 2 | 3 | import json 4 | import os 5 | 6 | from collections import OrderedDict 7 | 8 | from celery.bin.base import Command, Option 9 | 10 | from cyanide.vagrant import Vagrant, path 11 | 12 | 13 | def csv_list_option(option, opt, value, parser): 14 | setattr(parser.values, option.dest, value.split(',')) 15 | 16 | 17 | class vagrant(Command): 18 | Vagrant = Vagrant 19 | 20 | def __init__(self, *args, **kwargs): 21 | super(vagrant, self).__init__(*args, **kwargs) 22 | self.commands = OrderedDict([ 23 | ('up', self.up), 24 | ('status', self.status), 25 | ('sshargs', self.sshargs), 26 | ('destroy', self.destroy), 27 | ('suspend', self.suspend), 28 | ('resume', self.resume), 29 | ('halt', self.halt), 30 | ('provision', self.provision), 31 | ('reload', self.reload), 32 | ('conf', self.conf), 33 | ('user', self.user), 34 | ('hostname', self.hostname), 35 | ('user_hostname', self.user_hostname), 36 | ('user_hostname_port', self.user_hostname_port), 37 | ('port', self.port), 38 | ('keyfile', self.keyfile), 39 | ('box_list', self.box_list), 40 | ('path', self.path), 41 | ('statedir', self.statedir), 42 | ('snapshot_push', self.snapshot_push), 43 | ('snapshot_pop', self.snapshot_pop), 44 | ('snapshot_save', self.snapshot_save), 45 | ('snapshot_restore', self.snapshot_restore), 46 | ('snapshot_delete', self.snapshot_delete), 47 | ('snapshot_list', self.snapshot_list), 48 | ('version', self.version), 49 | ]) 50 | self.vagrant = None 51 | 52 | def run(self, *args, **options): 53 | if not args: 54 | raise self.Error('missing subcommand. Try --help?') 55 | try: 56 | self.vagrant = self.create_session(**options) 57 | return self.run_command(*args, **options) 58 | except KeyboardInterrupt: 59 | print('interrupted by user: exiting...', file=self.stdout) 60 | 61 | def create_session(self, root, quiet_stdout, quiet_stderr, **kwargs): 62 | # make sure ``rm -rf $(celery vagrant statedir)`` does not 63 | # do horrible things. 64 | if root == '/': 65 | raise RuntimeError('Vagrant dir cannot be root (/) !!!') 66 | if '*' in root: 67 | raise RuntimeError('Vagrant dir cannot contain the * character') 68 | return self.Vagrant( 69 | root=root, 70 | quiet_stdout=quiet_stdout, 71 | quiet_stderr=quiet_stderr, 72 | ) 73 | 74 | def usage(self, command): 75 | return '%prog {command} [{commands}] [options] {0.args}'.format( 76 | self, command=command, commands='|'.join(self.commands)) 77 | 78 | def up(self, name, provision_with, **kwargs): 79 | self.vagrant.up(vm_name=name, provision_with=provision_with) 80 | 81 | def status(self, name, **kwargs): 82 | print(self.vagrant.status(vm_name=name), file=self.stdout) 83 | 84 | def sshargs(self, name, **kwargs): 85 | config = self.vagrant.conf(vm_name=name) 86 | print('{0}@{1} -p {2} -i {3}'.format( 87 | config['User'], 88 | config['HostName'], 89 | config['Port'], 90 | config['IdentityFile'], 91 | ), file=self.stdout) 92 | 93 | def destroy(self, name, **kwargs): 94 | self.vagrant.destroy(vm_name=name) 95 | 96 | def provision(self, name, provision_with, **kwargs): 97 | self.vagrant.provision(vm_name=name, provision_with=provision_with) 98 | 99 | def reload(self, name, provision_with, **kwargs): 100 | self.vagrant.reload(vm_name=name, provision_with=provision_with) 101 | 102 | def suspend(self, name, **kwargs): 103 | self.vagrant.suspend(vm_name=name) 104 | 105 | def resume(self, name, **kwargs): 106 | self.vagrant.resume(vm_name=name) 107 | 108 | def halt(self, name, force, **kwargs): 109 | self.vagrant.halt(vm_name=name, force=force) 110 | 111 | def conf(self, name, arguments, **kwargs): 112 | self.pretty_json(self.vagrant.conf( 113 | vm_name=name, 114 | ssh_config=arguments and arguments[0] or None, 115 | )) 116 | 117 | def path(self, **kwargs): 118 | print(self.vagrant.root, file=self.stdout) 119 | 120 | def statedir(self, **kwargs): 121 | print(os.path.join(self.vagrant.root, '.vagrant'), file=self.stdout) 122 | 123 | def user(self, name, **kwargs): 124 | print(self.vagrant.user(vm_name=name), file=self.stdout) 125 | 126 | def hostname(self, name, **kwargs): 127 | print(self.vagrant.hostname(vm_name=name), file=self.stdout) 128 | 129 | def user_hostname(self, name, **kwargs): 130 | print(self.vagrant.user_hostname(vm_name=name), file=self.stdout) 131 | 132 | def port(self, name, **kwargs): 133 | print(self.vagrant.port(vm_name=name), file=self.stdout) 134 | 135 | def user_hostname_port(self, name, **kwargs): 136 | print(self.vagrant.user_hostname_port(vm_name=name), file=self.stdout) 137 | 138 | def keyfile(self, name, **kwargs): 139 | print(self.vagrant.keyfile(vm_name=name), file=self.stdout) 140 | 141 | def version(self, **kwargs): 142 | print(self.vagrant.version(), file=self.stdout) 143 | 144 | def box_list(self, **kwargs): 145 | print(self.vagrant.box_list(), file=self.stdout) 146 | 147 | def snapshot_push(self, **kwargs): 148 | self.vagrant.snapshot_push() 149 | 150 | def snapshot_pop(self, **kwargs): 151 | self.vagrant.snapshot_pop() 152 | 153 | def snapshot_save(self, arguments, **kwargs): 154 | self.vagrant.snapshot_save(name=arguments[0]) 155 | 156 | def snapshot_restore(self, arguments, **kwargs): 157 | self.vagrant.snapshot_restore(name=arguments[0]) 158 | 159 | def snapshot_delete(self, arguments, **kwargs): 160 | self.vagrant.snapshot_delete(name=arguments[0]) 161 | 162 | def snapshot_list(self, **kwargs): 163 | print(self.vagrant.snapshot_list(), file=self.stdout) 164 | 165 | def run_command(self, command, *args, **options): 166 | try: 167 | handler = self.commands[command] 168 | except KeyError: 169 | raise self.Error( 170 | 'Unknown command: {0}. Try --help?'.format(command)) 171 | else: 172 | return handler(arguments=args, **options) 173 | 174 | def pretty_json(self, obj): 175 | json.dump( 176 | obj, self.stdout, 177 | sort_keys=True, indent=4, separators=(',', ': '), 178 | ) 179 | print(file=self.stdout) 180 | 181 | def get_options(self): 182 | return ( 183 | Option('--root', default=path(), 184 | help='Directory holding Vagrantfile.'), 185 | Option('--name', default=None, 186 | help='Optional VM name.'), 187 | Option('--provision-with', default=None, 188 | type='string', action='callback', 189 | callback=csv_list_option, 190 | help='Optional comma-separated list of provisioners.'), 191 | Option('--quiet-stdout', action='store_true', 192 | help='Disable output to stdout.'), 193 | Option('--quiet-stderr', action='store_true', 194 | help='Disable output to stderr.'), 195 | Option('--force', action='store_true', 196 | help='Force action (applies to e.g. halt).'), 197 | ) 198 | 199 | 200 | def main(argv=None): 201 | return vagrant(app='cyanide.app:app').execute_from_commandline(argv=argv) 202 | 203 | 204 | if __name__ == '__main__': 205 | main() 206 | -------------------------------------------------------------------------------- /cyanide/compat.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | import sys 4 | 5 | __all__ = ['bytes_if_py2'] 6 | 7 | PY3 = sys.version_info[0] >= 3 8 | 9 | if PY3: # pragma: no cover 10 | def bytes_if_py2(s): 11 | return s 12 | else: # pragma: no cover 13 | def bytes_if_py2(s): # noqa 14 | if isinstance(s, unicode): 15 | return s.encode() 16 | return s 17 | -------------------------------------------------------------------------------- /cyanide/data.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import absolute_import, unicode_literals 3 | 4 | import datetime 5 | import decimal 6 | import uuid 7 | 8 | from .compat import bytes_if_py2 9 | 10 | try: 11 | import simplejson as json 12 | from simplejson.decoder import JSONDecodeError as _DecodeError 13 | _json_extra_kwargs = {'use_decimal': False} 14 | except ImportError: # pragma: no cover 15 | import json # noqa 16 | _json_extra_kwargs = {} # noqa 17 | 18 | class _DecodeError(Exception): # noqa 19 | pass 20 | 21 | 22 | _encoder_cls = type(json._default_encoder) 23 | type_registry = {} 24 | 25 | 26 | class JSONEncoder(_encoder_cls): 27 | """Kombu custom json encoder.""" 28 | 29 | def default(self, obj, 30 | dates=(datetime.datetime, datetime.date), 31 | times=(datetime.time,), 32 | textual=(decimal.Decimal, uuid.UUID), 33 | isinstance=isinstance, 34 | datetime=datetime.datetime): 35 | try: 36 | return super(JSONEncoder, self).default(obj) 37 | except TypeError: 38 | reducer = getattr(obj, '__to_json__', None) 39 | if reducer: 40 | return reducer() 41 | if isinstance(obj, dates): 42 | if not isinstance(obj, datetime): 43 | obj = datetime(obj.year, obj.month, obj.day, 0, 0, 0, 0) 44 | r = obj.isoformat() 45 | if r.endswith("+00:00"): 46 | r = r[:-6] + "Z" 47 | return r 48 | elif isinstance(obj, times): 49 | return obj.isoformat() 50 | elif isinstance(obj, textual): 51 | return text_t(obj) 52 | raise 53 | 54 | 55 | def decode_hook(d): 56 | try: 57 | d = d['py/obj'] 58 | except KeyError: 59 | return d 60 | type_registry[d['type']](**d['attrs']) 61 | 62 | 63 | def install_json(): 64 | json._default_encoder = JSONEncoder() 65 | json._default_decoder.object_hook = decode_hook 66 | try: 67 | from kombu.utils import json as kombujson 68 | except ImportError: 69 | pass 70 | else: 71 | kombujson._default_encoder = JSONEncoder 72 | install_json() # ugh, ugly but it's a test suite after all 73 | 74 | 75 | # this imports kombu.utils.json, so can only import after install_json() 76 | from celery.utils.debug import humanbytes # noqa 77 | from celery.utils.imports import qualname # noqa 78 | 79 | 80 | def json_reduce(obj, attrs): 81 | return {'py/obj': {'type': qualname(obj), 'attrs': attrs}} 82 | 83 | 84 | def jsonable(cls): 85 | type_registry[qualname(cls)] = cls.__from_json__ 86 | return cls 87 | 88 | 89 | @jsonable 90 | class Data(object): 91 | 92 | def __init__(self, label, data): 93 | self.label = label 94 | self.data = data 95 | 96 | def __str__(self): 97 | return bytes_if_py2(''.format( 98 | self.label, humanbytes(len(self.data)), 99 | )) 100 | 101 | def __repr__(self): 102 | return str(self) 103 | 104 | def __to_json__(self): 105 | return json_reduce(self, {'label': self.label, 'data': self.data}) 106 | 107 | @classmethod 108 | def __from_json__(cls, label=None, data=None, **kwargs): 109 | return cls(label, data) 110 | 111 | def __reduce__(self): 112 | return Data, (self.label, self.data) 113 | 114 | BIG = Data('BIG', 'x' * 2 ** 20 * 8) 115 | SMALL = Data('SMALL', 'e' * 1024) 116 | -------------------------------------------------------------------------------- /cyanide/fbi.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, print_function, unicode_literals 2 | 3 | import socket 4 | import sys 5 | 6 | from contextlib import contextmanager 7 | 8 | from celery import states 9 | 10 | 11 | class FBI(object): 12 | 13 | def __init__(self, app): 14 | self.app = app 15 | self.receiver = None 16 | self.state = self.app.events.State() 17 | self.connection = None 18 | self.enabled = False 19 | 20 | def enable(self, enabled): 21 | self.enabled = enabled 22 | 23 | @contextmanager 24 | def investigation(self): 25 | if self.enabled: 26 | with self.app.connection() as conn: 27 | receiver = self.app.events.Receiver( 28 | conn, handlers={'*': self.state.event}, 29 | ) 30 | with receiver.consumer_context() as (conn, _, _): 31 | self.connection = conn 32 | try: 33 | yield self 34 | finally: 35 | self.ffwd() 36 | else: 37 | yield 38 | 39 | def ffwd(self): 40 | while 1: 41 | try: 42 | self.connection.drain_events(timeout=1) 43 | except socket.error: 44 | break 45 | 46 | def state_of(self, tid): 47 | try: 48 | task = self.state.tasks[tid] 49 | except KeyError: 50 | return 'No events for {0}'.format(tid) 51 | 52 | if task.state in states.READY_STATES: 53 | return 'Task {0.uuid} completed with {0.state}'.format(task) 54 | elif task.state in states.UNREADY_STATES: 55 | return 'Task {0.uuid} waiting in {0.state} state'.format(task) 56 | else: 57 | return 'Task {0.uuid} in other state {0.state}'.format(task) 58 | 59 | def query(self, ids): 60 | return self.app.control.inspect().query_task(id) 61 | 62 | def diag(self, ids, file=sys.stderr): 63 | if self.enabled: 64 | self.ffwd() 65 | for tid in ids: 66 | print(self.state_of(tid), file=file) 67 | -------------------------------------------------------------------------------- /cyanide/suite.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, print_function, unicode_literals 2 | 3 | import celery 4 | import cyanide 5 | 6 | import inspect 7 | import platform 8 | import socket 9 | import sys 10 | import traceback 11 | 12 | from collections import OrderedDict, defaultdict, namedtuple 13 | from functools import partial 14 | from itertools import count, cycle 15 | 16 | from celery.exceptions import TimeoutError 17 | from celery.five import items, monotonic, range, values 18 | from celery.utils.debug import blockdetection 19 | from celery.utils.imports import qualname 20 | from celery.utils.text import pluralize, truncate 21 | from celery.utils.term import colored 22 | from kombu.utils import retry_over_time 23 | 24 | from .fbi import FBI 25 | from .tasks import marker, _marker 26 | 27 | try: 28 | from celery.platforms import isatty 29 | except ImportError: 30 | from celery.utils import isatty 31 | 32 | try: 33 | import celery.utils.time as timeutils 34 | except ImportError: 35 | from celery.utils import timeutils 36 | 37 | BANNER = """\ 38 | Cyanide v{version} [celery {celery_version}] 39 | 40 | {platform} 41 | 42 | [config] 43 | .> app: {app} 44 | .> broker: {conninfo} 45 | .> suite: {suite} 46 | 47 | [toc: {total} {TESTS} total] 48 | {toc} 49 | """ 50 | 51 | F_PROGRESS = """\ 52 | {0.index:2d}: {0.test.__name__:<36} {status} \ 53 | ({0.iteration}/{0.total_iterations}) rep#{0.repeats} runtime: \ 54 | {runtime}/{elapsed}\ 55 | """ 56 | 57 | E_STILL_WAITING = """\ 58 | Still waiting for {0}. Trying again {when}: {exc!r}\ 59 | """ 60 | 61 | Progress = namedtuple('Progress', ( 62 | 'test', 'iteration', 'total_iterations', 63 | 'index', 'repeats', 'runtime', 'elapsed', 'completed', 64 | )) 65 | 66 | 67 | Inf = float('Inf') 68 | 69 | 70 | def assert_equal(a, b): 71 | assert a == b, '{0!r} != {1!r}'.format(a, b) 72 | 73 | 74 | class StopSuite(Exception): 75 | pass 76 | 77 | 78 | class Sentinel(Exception): 79 | pass 80 | 81 | 82 | def humanize_seconds(secs, prefix='', sep='', now='now', **kwargs): 83 | s = timeutils.humanize_seconds(secs, prefix, sep, now, **kwargs) 84 | if s == now and float(secs) > 0: 85 | return '{prefix}{sep}{0:.2f} seconds'.format( 86 | float(secs), prefix=prefix, sep=sep) 87 | return s 88 | 89 | 90 | def pstatus(p, status=None): 91 | runtime = format(monotonic() - p.runtime, '.4f') 92 | elapsed = format(monotonic() - p.elapsed, '.4f') 93 | return F_PROGRESS.format( 94 | p, 95 | status=status or '', 96 | runtime=humanize_seconds(runtime, now=runtime), 97 | elapsed=humanize_seconds(elapsed, now=elapsed), 98 | ) 99 | 100 | 101 | class Speaker(object): 102 | 103 | def __init__(self, gap=5.0, file=None): 104 | self.gap = gap 105 | self.file = sys.stdout if file is None else file 106 | self.last_noise = monotonic() - self.gap * 2 107 | 108 | def beep(self): 109 | now = monotonic() 110 | if now - self.last_noise >= self.gap: 111 | self.emit() 112 | self.last_noise = now 113 | 114 | def emit(self): 115 | print('\a', file=self.file, end='') 116 | 117 | 118 | class Meter(object): 119 | 120 | def __init__(self, s='.', end='', cursor='/-\\', file=None): 121 | self.s = s 122 | self.end = end 123 | self.file = sys.stdout if file is None else file 124 | self.counter = 0 125 | self.cursor = cycle(cursor) 126 | 127 | def emit(self, *args, **kwargs): 128 | self.counter += len(self.s) 129 | print(self.s * (self.counter - 1) + next(self.cursor), 130 | end='\r', file=self.file) 131 | self.file.flush() 132 | 133 | def revert(self): 134 | self.counter = 0 135 | 136 | 137 | class DummyMeter(object): 138 | 139 | def __init__(self, *args, **kwargs): 140 | pass 141 | 142 | def emit(self, *args, **kwargs): 143 | pass 144 | 145 | def revert(self): 146 | pass 147 | 148 | 149 | def testgroup(*funs): 150 | return OrderedDict((fun.__name__, fun) for fun in funs) 151 | 152 | 153 | class ManagerMixin(object): 154 | TaskPredicate = StopSuite 155 | Meter = Meter 156 | 157 | def _init_manager(self, app, 158 | block_timeout=30 * 60, stdout=None, stderr=None): 159 | self.stdout = sys.stdout if stdout is None else stdout 160 | self.stderr = sys.stderr if stderr is None else stderr 161 | self.connerrors = self.app.connection().recoverable_connection_errors 162 | self.block_timeout = block_timeout 163 | self.progress = None 164 | self.speaker = Speaker(file=self.stdout) 165 | self.fbi = FBI(app) 166 | 167 | def new_meter(self): 168 | return self.Meter(file=self.stdout) 169 | 170 | def missing_results(self, r): 171 | return [res.id for res in r if res.id not in res.backend._cache] 172 | 173 | def wait_for(self, fun, catch, 174 | desc='thing', args=(), kwargs={}, errback=None, 175 | max_retries=10, interval_start=0.1, interval_step=0.5, 176 | interval_max=5.0, emit_warning=False, **options): 177 | meter = self.new_meter() 178 | 179 | def on_error(exc, intervals, retries): 180 | interval = next(intervals) 181 | if emit_warning: 182 | self.warn(E_STILL_WAITING.format( 183 | desc, when=humanize_seconds(interval, 'in', ' '), exc=exc, 184 | )) 185 | else: 186 | meter.emit() 187 | if errback: 188 | errback(exc, interval, retries) 189 | return interval 190 | 191 | return self.retry_over_time( 192 | fun, catch, 193 | args=args, kwargs=kwargs, 194 | errback=on_error, max_retries=max_retries, 195 | interval_start=interval_start, interval_step=interval_step, 196 | **options 197 | ) 198 | 199 | def ensure_not_for_a_while(self, fun, catch, 200 | desc='thing', max_retries=20, 201 | interval_start=0.1, interval_step=0.02, 202 | interval_max=1.0, emit_warning=False, 203 | **options): 204 | meter = self.new_meter() 205 | try: 206 | return self.wait_for( 207 | fun, catch, desc=desc, max_retries=max_retries, 208 | interval_start=interval_start, interval_step=interval_step, 209 | interval_max=interval_max, emit_warning=emit_warning, 210 | errback=meter.emit, 211 | ) 212 | except catch: 213 | pass 214 | else: 215 | raise AssertionError('Should not have happened: {0}'.format(desc)) 216 | finally: 217 | meter.revert() 218 | 219 | def retry_over_time(self, *args, **kwargs): 220 | return retry_over_time(*args, **kwargs) 221 | 222 | def join(self, r, propagate=False, max_retries=10, **kwargs): 223 | if self.no_join: 224 | return 225 | received = [] 226 | 227 | def on_result(task_id, value): 228 | received.append(task_id) 229 | 230 | for i in range(max_retries) if max_retries else count(0): 231 | received[:] = [] 232 | try: 233 | return r.get(callback=on_result, propagate=propagate, **kwargs) 234 | except (socket.timeout, TimeoutError) as exc: 235 | waiting_for = self.missing_results(r) 236 | self.speaker.beep() 237 | marker( 238 | 'Still waiting for {0}/{1}: [{2}]: {3!r}'.format( 239 | len(r) - len(received), len(r), 240 | truncate(', '.join(waiting_for)), exc), '!', 241 | ) 242 | self.fbi.diag(waiting_for) 243 | except self.connerrors as exc: 244 | self.speaker.beep() 245 | marker('join: connection lost: {0!r}'.format(exc), '!') 246 | raise self.TaskPredicate('Test failed: Missing task results') 247 | 248 | def inspect(self, timeout=1): 249 | return self.app.control.inspect(timeout=timeout) 250 | 251 | def query_tasks(self, ids, timeout=0.5): 252 | for reply in items(self.inspect(timeout).query_task(ids) or []): 253 | yield reply 254 | 255 | def query_task_states(self, ids, timeout=0.5): 256 | states = defaultdict(set) 257 | for hostname, reply in self.query_tasks(ids, timeout=timeout): 258 | for task_id, (state, _) in items(reply): 259 | states[state].add(task_id) 260 | return states 261 | 262 | def assert_accepted(self, ids, interval=0.5, 263 | desc='waiting for tasks to be accepted', **policy): 264 | return self.assert_task_worker_state( 265 | self.is_accepted, ids, interval=interval, desc=desc, **policy 266 | ) 267 | 268 | def assert_received(self, ids, interval=0.5, 269 | desc='waiting for tasks to be received', **policy): 270 | return self.assert_task_worker_state( 271 | self.is_accepted, ids, interval=interval, desc=desc, **policy 272 | ) 273 | 274 | def assert_task_worker_state(self, fun, ids, interval=0.5, **policy): 275 | return self.wait_for( 276 | partial(self.true_or_raise, fun, ids, timeout=interval), 277 | (Sentinel,), **policy 278 | ) 279 | 280 | def is_received(self, ids, **kwargs): 281 | return self._ids_matches_state( 282 | ['reserved', 'active', 'ready'], ids, **kwargs) 283 | 284 | def is_accepted(self, ids, **kwargs): 285 | return self._ids_matches_state(['active', 'ready'], ids, **kwargs) 286 | 287 | def _ids_matches_state(self, expected_states, ids, timeout=0.5): 288 | states = self.query_task_states(ids, timeout=timeout) 289 | return all( 290 | any(t in s for s in [states[k] for k in expected_states]) 291 | for t in ids 292 | ) 293 | 294 | def true_or_raise(self, fun, *args, **kwargs): 295 | res = fun(*args, **kwargs) 296 | if not res: 297 | raise Sentinel() 298 | return res 299 | 300 | 301 | class Suite(ManagerMixin): 302 | 303 | TaskPredicate = StopSuite 304 | 305 | def __init__(self, app, no_color=False, **kwargs): 306 | self.app = app 307 | self._init_manager(app, **kwargs) 308 | if not isatty(self.stdout): 309 | no_color = True 310 | self.colored = colored(enabled=not no_color) 311 | self.init_groups() 312 | 313 | def setup(self): 314 | pass 315 | 316 | def teardown(self): 317 | pass 318 | 319 | def print(self, message, file=None): 320 | print(message, file=self.stdout if file is None else file) 321 | 322 | def error(self, message): 323 | print(self.colored.red(message), file=self.stderr) 324 | 325 | def warn(self, message): 326 | print(self.colored.cyan(message), file=self.stdout) 327 | 328 | def init_groups(self): 329 | acc = defaultdict(list) 330 | for attr in dir(self): 331 | if not _is_descriptor(self, attr): 332 | meth = getattr(self, attr) 333 | try: 334 | groups = meth.__func__.__testgroup__ 335 | except AttributeError: 336 | pass 337 | else: 338 | for g in groups: 339 | acc[g].append(meth) 340 | # sort the tests by the order in which they are defined in the class 341 | for g in values(acc): 342 | g[:] = sorted(g, key=lambda m: m.__func__.__testsort__) 343 | self.groups = dict( 344 | (name, testgroup(*tests)) for name, tests in items(acc) 345 | ) 346 | 347 | def run(self, names=None, iterations=50, offset=0, 348 | numtests=None, list_all=False, repeat=0, group='all', 349 | diag=False, no_join=False, **kw): 350 | self.no_join = no_join 351 | self.fbi.enable(diag) 352 | tests = self.filtertests(group, names)[offset:numtests or None] 353 | if list_all: 354 | return self.print(self.testlist(tests)) 355 | self.print(self.banner(tests)) 356 | self.print('+enable worker task events...') 357 | self.app.control.enable_events() 358 | it = count() if repeat == Inf else range(int(repeat) or 1) 359 | for i in it: 360 | marker( 361 | '{0} (repetition {1})'.format( 362 | self.colored.bold('suite start'), i + 1), 363 | '+', 364 | ) 365 | for j, test in enumerate(tests): 366 | self.runtest(test, iterations, j + 1, i + 1) 367 | marker( 368 | '{0} (repetition {1})'.format( 369 | self.colored.bold('suite end'), i + 1), 370 | '+', 371 | ) 372 | 373 | def assert_equal(self, a, b): 374 | return assert_equal(a, b) 375 | 376 | def filtertests(self, group, names): 377 | tests = self.groups[group] 378 | try: 379 | return ([tests[n] for n in names] if names 380 | else list(values(tests))) 381 | except KeyError as exc: 382 | raise KeyError('Unknown test name: {0}'.format(exc)) 383 | 384 | def testlist(self, tests): 385 | return ',\n'.join( 386 | '.> {0}) {1}'.format(i + 1, t.__name__) 387 | for i, t in enumerate(tests) 388 | ) 389 | 390 | def banner(self, tests): 391 | app = self.app 392 | return BANNER.format( 393 | app='{0}:0x{1:x}'.format(app.main or '__main__', id(app)), 394 | version=cyanide.__version__, 395 | celery_version=celery.VERSION_BANNER, 396 | conninfo=app.connection().as_uri(), 397 | platform=platform.platform(), 398 | toc=self.testlist(tests), 399 | TESTS=pluralize(len(tests), 'test'), 400 | total=len(tests), 401 | suite=':'.join(qualname(self).rsplit('.', 1)), 402 | ) 403 | 404 | def runtest(self, fun, n=50, index=0, repeats=1): 405 | n = getattr(fun, '__iterations__', None) or n 406 | header = '[[[{0}({1})]]]'.format(fun.__name__, n) 407 | if repeats > 1: 408 | header = '{0} #{1}'.format(header, repeats) 409 | self.print(header) 410 | with blockdetection(self.block_timeout): 411 | with self.fbi.investigation(): 412 | runtime = elapsed = monotonic() 413 | i = 0 414 | failed = False 415 | self.progress = Progress( 416 | fun, i, n, index, repeats, elapsed, runtime, 0, 417 | ) 418 | _marker.delay(pstatus(self.progress)) 419 | 420 | try: 421 | for i in range(n): 422 | runtime = monotonic() 423 | self.progress = Progress( 424 | fun, i + 1, n, index, repeats, runtime, elapsed, 0, 425 | ) 426 | self.execute_test(fun) 427 | 428 | except Exception: 429 | failed = True 430 | self.speaker.beep() 431 | raise 432 | finally: 433 | if n > 1 or failed: 434 | self.print('{0} {1} iterations in {2}'.format( 435 | 'failed after' if failed else 'completed', 436 | i + 1, humanize_seconds(monotonic() - elapsed), 437 | ), file=self.stderr if failed else self.stdout) 438 | if not failed: 439 | self.progress = Progress( 440 | fun, i + 1, n, index, repeats, runtime, elapsed, 1, 441 | ) 442 | 443 | def execute_test(self, fun): 444 | self.setup() 445 | try: 446 | try: 447 | fun() 448 | except StopSuite: 449 | raise 450 | except AssertionError as exc: 451 | self.on_test_error(exc, 'FAILED') 452 | except Exception as exc: 453 | self.on_test_error(exc, 'ERROR') 454 | else: 455 | self.print(pstatus(self.progress, self.colored.green('OK'))) 456 | finally: 457 | self.teardown() 458 | 459 | def on_test_error(self, exc, status): 460 | self.error('-> {0!r}'.format(exc)) 461 | self.error(traceback.format_exc()) 462 | self.error(pstatus(self.progress, self.colored.red(status))) 463 | 464 | def dump_progress(self): 465 | return pstatus(self.progress) if self.progress else 'No test running' 466 | 467 | _creation_counter = count(0) 468 | 469 | 470 | def testcase(*groups, **kwargs): 471 | if not groups: 472 | raise ValueError('@testcase requires at least one group name') 473 | 474 | def _mark_as_case(fun): 475 | fun.__testgroup__ = groups 476 | fun.__testsort__ = next(_creation_counter) 477 | fun.__iterations__ = kwargs.get('iterations') 478 | return fun 479 | 480 | return _mark_as_case 481 | 482 | 483 | def _is_descriptor(obj, attr): 484 | try: 485 | cattr = getattr(obj.__class__, attr) 486 | except AttributeError: 487 | pass 488 | else: 489 | ismethod = inspect.ismethod(cattr) or inspect.isfunction(cattr) 490 | return not ismethod and hasattr(cattr, '__get__') 491 | return False 492 | -------------------------------------------------------------------------------- /cyanide/suites/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/celery/cyanide/9b7d78dd4d804ed74c50f0ab9492f19ac06dc9ca/cyanide/suites/__init__.py -------------------------------------------------------------------------------- /cyanide/suites/default.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | import random 4 | 5 | from time import sleep 6 | 7 | from celery import group 8 | 9 | from cyanide.tasks import ( 10 | add, any_, exiting, kill, sleeping, 11 | sleeping_ignore_limits, any_returning, 12 | ) 13 | from cyanide.data import BIG, SMALL 14 | from cyanide.suite import Suite, testcase 15 | 16 | 17 | class Default(Suite): 18 | 19 | @testcase('all', 'green') 20 | def manyshort(self): 21 | self.join(group(add.s(i, i) for i in range(1000))(), 22 | timeout=10, propagate=True) 23 | 24 | @testcase('all') 25 | def always_timeout(self): 26 | self.join( 27 | group(sleeping.s(1).set(time_limit=0.1) 28 | for _ in range(100))(), 29 | timeout=10, propagate=False, 30 | ) 31 | 32 | @testcase('all') 33 | def termbysig(self): 34 | self._evil_groupmember(kill) 35 | 36 | @testcase('green') 37 | def group_with_exit(self): 38 | self._evil_groupmember(exiting) 39 | 40 | @testcase('all') 41 | def timelimits(self): 42 | self._evil_groupmember(sleeping, 2, time_limit=1) 43 | 44 | @testcase('all') 45 | def timelimits_soft(self): 46 | self._evil_groupmember(sleeping_ignore_limits, 2, 47 | soft_time_limit=1, time_limit=1.1) 48 | 49 | @testcase('all') 50 | def alwayskilled(self): 51 | g = group(kill.s() for _ in range(10)) 52 | self.join(g(), timeout=10) 53 | 54 | @testcase('all', 'green') 55 | def alwaysexits(self): 56 | g = group(exiting.s() for _ in range(10)) 57 | self.join(g(), timeout=10) 58 | 59 | def _evil_groupmember(self, evil_t, *eargs, **opts): 60 | g1 = group(add.s(2, 2).set(**opts), evil_t.s(*eargs).set(**opts), 61 | add.s(4, 4).set(**opts), add.s(8, 8).set(**opts)) 62 | g2 = group(add.s(3, 3).set(**opts), add.s(5, 5).set(**opts), 63 | evil_t.s(*eargs).set(**opts), add.s(7, 7).set(**opts)) 64 | self.join(g1(), timeout=10) 65 | self.join(g2(), timeout=10) 66 | 67 | @testcase('all', 'green') 68 | def bigtasksbigvalue(self): 69 | g = group(any_returning.s(BIG, sleep=0.3) for i in range(8)) 70 | r = g() 71 | try: 72 | self.join(r, timeout=10) 73 | finally: 74 | # very big values so remove results from backend 75 | try: 76 | r.forget() 77 | except NotImplementedError: 78 | pass 79 | 80 | @testcase('all', 'green') 81 | def bigtasks(self, wait=None): 82 | self._revoketerm(wait, False, False, BIG) 83 | 84 | @testcase('all', 'green') 85 | def smalltasks(self, wait=None): 86 | self._revoketerm(wait, False, False, SMALL) 87 | 88 | @testcase('all') 89 | def revoketermfast(self, wait=None): 90 | self._revoketerm(wait, True, False, SMALL) 91 | 92 | @testcase('all') 93 | def revoketermslow(self, wait=5): 94 | self._revoketerm(wait, True, True, BIG) 95 | 96 | def _revoketerm(self, wait=None, terminate=True, 97 | joindelay=True, data=BIG): 98 | g = group(any_.s(data, sleep=wait) for i in range(8)) 99 | r = g() 100 | if terminate: 101 | if joindelay: 102 | sleep(random.choice(range(4))) 103 | r.revoke(terminate=True) 104 | self.join(r, timeout=10) 105 | -------------------------------------------------------------------------------- /cyanide/tasks.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import absolute_import, print_function, unicode_literals 3 | 4 | import os 5 | import signal 6 | import sys 7 | 8 | from time import sleep 9 | 10 | from celery.exceptions import SoftTimeLimitExceeded 11 | from celery.utils.log import get_task_logger 12 | 13 | from .app import app 14 | 15 | E_MARKER_DELAY_ERROR = """\ 16 | Retrying marker.delay(). It failed to start: {0}\ 17 | """ 18 | 19 | logger = get_task_logger(__name__) 20 | 21 | 22 | def marker(s, sep='-'): 23 | """Marker is a task that logs something to the worker logs. 24 | 25 | :param s: Text to log. 26 | 27 | """ 28 | print('{0}{1}'.format(sep, s)) 29 | while True: 30 | try: 31 | return _marker.delay(s, sep) 32 | except Exception as exc: 33 | print(E_MARKER_DELAY_ERROR.format(exc)) 34 | 35 | 36 | @app.task 37 | def _marker(s, sep='---'): 38 | print('{sep} {0} {sep}'.format(s, sep=sep)) 39 | 40 | 41 | @app.task 42 | def add(x, y): 43 | """Add two numbers.""" 44 | return x + y 45 | 46 | 47 | @app.task(bind=True) 48 | def ids(self, i): 49 | """Returns a tuple of ``root_id``, ``parent_id`` and 50 | the argument passed as ``i``.""" 51 | return self.request.root_id, self.request.parent_id, i 52 | 53 | 54 | @app.task(bind=True) 55 | def collect_ids(self, res, i): 56 | """Used as a callback in a chain or group where the previous tasks 57 | are :task:`ids`: returns a tuple of:: 58 | 59 | (previous_result, (root_id, parent_id, i)) 60 | 61 | """ 62 | return res, ids(i) 63 | 64 | 65 | @app.task 66 | def xsum(x): 67 | """Takes a list of numbers and returns the total.""" 68 | return sum(x) 69 | 70 | 71 | @app.task 72 | def any_(*args, **kwargs): 73 | """Task taking any argument, returning nothing. 74 | 75 | This is useful for testing related to large arguments: big values, 76 | an insane number of positional arguments, etc. 77 | 78 | :keyword sleep: Optional number of seconds to sleep for before returning. 79 | 80 | """ 81 | wait = kwargs.get('sleep') 82 | if wait: 83 | sleep(wait) 84 | 85 | 86 | @app.task 87 | def any_returning(*args, **kwargs): 88 | """The same as :task:`any` except it returns the arguments given 89 | as a tuple of ``(args, kwargs)``.""" 90 | any_(*args, **kwargs) 91 | return args, kwargs 92 | 93 | 94 | @app.task 95 | def exiting(status=0): 96 | """Task calling ``sys.exit(status)`` to terminate its own worker 97 | process.""" 98 | sys.exit(status) 99 | 100 | 101 | @app.task 102 | def kill(sig=getattr(signal, 'SIGKILL', None) or signal.SIGTERM): 103 | """Task sending signal to process currently executing itself. 104 | 105 | :keyword sig: Signal to send as signal number, default is :sig:`KILL` 106 | on platforms that supports that signal, for other platforms (i.e Windows) 107 | it will be :sig:`TERM`. 108 | 109 | """ 110 | os.kill(os.getpid(), sig) 111 | 112 | 113 | @app.task 114 | def sleeping(i, **_): 115 | """Task sleeping for ``i`` seconds, and returning nothing.""" 116 | sleep(i) 117 | 118 | 119 | @app.task 120 | def sleeping_ignore_limits(i, **_): 121 | """Task sleeping for ``i`` seconds, while ignoring soft time limits. 122 | 123 | If the task is signalled with 124 | :exc:`~celery.exceptions.SoftTimeLimitExceeded` the signal is ignored 125 | and the task will sleep for ``i`` seconds again, which will trigger 126 | the hard time limit (if enabled). 127 | 128 | """ 129 | try: 130 | sleep(i) 131 | except SoftTimeLimitExceeded: 132 | sleep(i) 133 | 134 | 135 | @app.task(bind=True) 136 | def retries(self, n=1, countdown=1, return_value=10): 137 | """Task that retries itself ``n`` times. 138 | 139 | :param n: Number of times to retry. 140 | :param n: Seconds to wait (``int``/``float``) between each retry (default 141 | is one second). 142 | :param return_value: Value to return when task finally succeeds. 143 | Default is 10 (don't ask, I guess it's a random true value). 144 | 145 | """ 146 | if not self.request.retries or self.request.retries < n: 147 | raise self.retry(countdown=countdown) 148 | return return_value 149 | 150 | 151 | @app.task 152 | def print_unicode(log_message='hå它 valmuefrø', print_message='hiöäüß'): 153 | """Task that both logs and print strings containing funny characters.""" 154 | logger.warning(log_message) 155 | print(print_message) 156 | 157 | 158 | @app.task 159 | def segfault(): 160 | """Task causing a segfault, abruptly terminating the process 161 | executing the task.""" 162 | import ctypes 163 | ctypes.memset(0, 0, 1) 164 | assert False, 'should not get here' 165 | 166 | 167 | @app.task(bind=True) 168 | def chord_adds(self, x): 169 | """Task that adds a new task to the current chord in a workflow.""" 170 | self.add_to_chord(add.s(x, x)) 171 | return 42 172 | 173 | 174 | @app.task(bind=True) 175 | def chord_replace(self, x): 176 | """Task that replaces itself in the current chord.""" 177 | return self.replace_in_chord(add.s(x, x)) 178 | 179 | 180 | @app.task 181 | def raising(exc=KeyError()): 182 | """Task raising exception. 183 | 184 | :param exc: Exception to raise. 185 | 186 | """ 187 | raise exc 188 | 189 | 190 | @app.task 191 | def errback(request, exc, traceback): 192 | print('Task id {0!r} raised exception: {1!r}\n{2}'.format( 193 | request.id, exc, traceback, 194 | )) 195 | 196 | 197 | @app.task 198 | def old_errback(task_id): 199 | print('Task id %r raised exception' % (task_id,)) 200 | 201 | 202 | @app.task 203 | def logs(msg, p=False): 204 | """Log a message to the worker logs. 205 | 206 | :keyword p: If set to :const:`True` the message will be printed instead 207 | of logged, thus being redirected to the stdout logger. 208 | 209 | """ 210 | print(msg) if p else logger.info(msg) 211 | -------------------------------------------------------------------------------- /cyanide/templates.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | import celery 4 | import os 5 | 6 | from functools import partial 7 | 8 | from celery.five import items 9 | from kombu import Queue 10 | from kombu.utils import symbol_by_name 11 | 12 | CYANIDE_TRANS = os.environ.get('CYANIDE_TRANS', False) 13 | default_queue = 'c.stress.trans' if CYANIDE_TRANS else 'c.stress' 14 | CYANIDE_QUEUE = os.environ.get('CYANIDE_QUEUE', default_queue) 15 | 16 | templates = {} 17 | 18 | IS_CELERY_4 = celery.VERSION[0] >= 4 19 | 20 | 21 | def template(name=None): 22 | 23 | def _register(cls): 24 | templates[name or cls.__name__] = '.'.join([__name__, cls.__name__]) 25 | return cls 26 | return _register 27 | 28 | 29 | if IS_CELERY_4: 30 | 31 | def use_template(app, template='default'): 32 | if isinstance(template, list) and len(template) == 1: 33 | # weird argparse thing 34 | template = template[0] 35 | template = template.split(',') 36 | 37 | # mixin the rest of the templates when the config is needed 38 | @app.on_after_configure.connect(weak=False) 39 | def load_template(sender, source, **kwargs): 40 | mixin_templates(template[1:], source) 41 | 42 | app.config_from_object(templates[template[0]]) 43 | else: 44 | 45 | def use_template(app, template='default'): # noqa 46 | template = template.split(',') 47 | app.after_configure = partial(mixin_templates, template[1:]) 48 | app.config_from_object(templates[template[0]]) 49 | 50 | 51 | def mixin_templates(templates, conf): 52 | return [mixin_template(template, conf) for template in templates] 53 | 54 | 55 | def mixin_template(template, conf): 56 | cls = symbol_by_name(templates[template]) 57 | conf.update(dict( 58 | (k, v) for k, v in items(vars(cls)) 59 | if not k.startswith('_') 60 | )) 61 | 62 | 63 | def template_names(): 64 | return ', '.join(templates) 65 | 66 | 67 | @template() 68 | class default(object): 69 | CELERY_ACCEPT_CONTENT = ['json'] 70 | BROKER_URL = os.environ.get('CYANIDE_BROKER', 'pyamqp://') 71 | BROKER_HEARTBEAT = 30 72 | CELERY_RESULT_BACKEND = os.environ.get('CYANIDE_BACKEND', 'rpc://') 73 | CELERY_RESULT_SERIALIZER = 'json' 74 | CELERY_RESULT_PERSISTENT = True 75 | CELERY_RESULT_EXPIRES = 300 76 | CELERY_MAX_CACHED_RESULTS = 100 77 | CELERY_DEFAULT_QUEUE = CYANIDE_QUEUE 78 | CELERY_IMPORTS = ['cyanide.tasks'] 79 | CELERY_TRACK_STARTED = True 80 | CELERY_QUEUES = [ 81 | Queue(CYANIDE_QUEUE, 82 | durable=not CYANIDE_TRANS, 83 | no_ack=CYANIDE_TRANS), 84 | ] 85 | CELERY_TASK_SERIALIZER = 'json' 86 | CELERY_TASK_PUBLISH_RETRY_POLICY = { 87 | 'max_retries': 100, 88 | 'interval_max': 2, 89 | 'interval_step': 0.1, 90 | } 91 | CELERY_TASK_PROTOCOL = 2 92 | if CYANIDE_TRANS: 93 | CELERY_DEFAULT_DELIVERY_MODE = 1 94 | CELERYD_PREFETCH_MULTIPLIER = int(os.environ.get('CYANIDE_PREFETCH', 10)) 95 | 96 | 97 | @template() 98 | class redis(default): 99 | BROKER_URL = os.environ.get('CYANIDE_BROKER', 'redis://') 100 | BROKER_TRANSPORT_OPTIONS = { 101 | 'fanout_prefix': True, 102 | 'fanout_patterns': True, 103 | } 104 | CELERY_RESULT_BACKEND = os.environ.get('CYANIDE_BACKEND', 'redis://') 105 | 106 | 107 | @template() 108 | class redistore(default): 109 | CELERY_RESULT_BACKEND = 'redis://' 110 | 111 | 112 | @template() 113 | class acks_late(default): 114 | CELERY_ACKS_LATE = True 115 | 116 | 117 | @template() 118 | class pickle(default): 119 | CELERY_ACCEPT_CONTENT = ['pickle', 'json'] 120 | CELERY_TASK_SERIALIZER = 'pickle' 121 | CELERY_RESULT_SERIALIZER = 'pickle' 122 | 123 | 124 | @template() 125 | class confirms(default): 126 | BROKER_URL = 'pyamqp://' 127 | BROKER_TRANSPORT_OPTIONS = {'confirm_publish': True} 128 | 129 | 130 | @template() 131 | class events(default): 132 | CELERY_SEND_EVENTS = True 133 | CELERY_SEND_TASK_SENT_EVENT = True 134 | 135 | 136 | @template() 137 | class execv(default): 138 | CELERYD_FORCE_EXECV = True 139 | 140 | 141 | @template() 142 | class sqs(default): 143 | BROKER_URL = 'sqs://' 144 | BROKER_TRANSPORT_OPTIONS = { 145 | 'region': os.environ.get('AWS_REGION', 'us-east-1'), 146 | } 147 | 148 | 149 | @template() 150 | class proto1(default): 151 | CELERY_TASK_PROTOCOL = 1 152 | 153 | 154 | @template() 155 | class vagrant1(default): 156 | BROKER_URL = 'pyamqp://testing:t3s71ng@192.168.33.123//testing' 157 | 158 | 159 | @template() 160 | class vagrant1_redis(redis): 161 | BROKER_URL = 'redis://192.168.33.123' 162 | CELERY_RESULT_BACKEND = 'redis://192.168.33.123' 163 | -------------------------------------------------------------------------------- /cyanide/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/celery/cyanide/9b7d78dd4d804ed74c50f0ab9492f19ac06dc9ca/cyanide/tests/__init__.py -------------------------------------------------------------------------------- /cyanide/tests/case.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | import case 4 | import sys 5 | 6 | sys.modules[__name__] = case 7 | -------------------------------------------------------------------------------- /cyanide/vagrant/Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | # Vagrantfile API/syntax version. Don't touch unless you know what you're doing! 5 | VAGRANTFILE_API_VERSION = "2" 6 | 7 | Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| 8 | # All Vagrant configuration is done here. The most common configuration 9 | # options are documented and commented below. For a complete reference, 10 | # please see the online documentation at vagrantup.com. 11 | 12 | # Every Vagrant virtual environment requires a box to build off of. 13 | config.vm.box = "ubuntu/trusty64" 14 | 15 | config.vm.provision :shell, path: "provision/provision.sh", 16 | privileged: true 17 | 18 | # Disable automatic box update checking. If you disable this, then 19 | # boxes will only be checked for updates when the user runs 20 | # `vagrant box outdated`. This is not recommended. 21 | # config.vm.box_check_update = false 22 | 23 | # Create a forwarded port mapping which allows access to a specific port 24 | # within the machine from a port on the host machine. In the example below, 25 | # accessing "localhost:8080" will access port 80 on the guest machine. 26 | # config.vm.network "forwarded_port", guest: 80, host: 8080 27 | 28 | # Create a private network, which allows host-only access to the machine 29 | # using a specific IP. 30 | config.vm.network "private_network", ip: "192.168.33.123" 31 | 32 | # Create a public network, which generally matched to bridged network. 33 | # Bridged networks make the machine appear as another physical device on 34 | # your network. 35 | # config.vm.network "public_network" 36 | 37 | # If true, then any SSH connections made will enable agent forwarding. 38 | # Default value: false 39 | # config.ssh.forward_agent = true 40 | 41 | # Share an additional folder to the guest VM. The first argument is 42 | # the path on the host to the actual folder. The second argument is 43 | # the path on the guest to mount the folder. And the optional third 44 | # argument is a set of non-required options. 45 | # config.vm.synced_folder "../data", "/vagrant_data" 46 | 47 | # Provider-specific configuration so you can fine-tune various 48 | # backing providers for Vagrant. These expose provider-specific options. 49 | # Example for VirtualBox: 50 | # 51 | config.vm.provider "virtualbox" do |vb| 52 | # # Don't boot with headless mode 53 | # vb.gui = true 54 | # 55 | # # Use VBoxManage to customize the VM. For example to change memory: 56 | vb.customize ["modifyvm", :id, "--memory", "1024"] 57 | end 58 | # 59 | # View the documentation for the provider you're using for more 60 | # information on available options. 61 | 62 | # Enable provisioning with CFEngine. CFEngine Community packages are 63 | # automatically installed. For example, configure the host as a 64 | # policy server and optionally a policy file to run: 65 | # 66 | # config.vm.provision "cfengine" do |cf| 67 | # cf.am_policy_hub = true 68 | # # cf.run_file = "motd.cf" 69 | # end 70 | # 71 | # You can also configure and bootstrap a client to an existing 72 | # policy server: 73 | # 74 | # config.vm.provision "cfengine" do |cf| 75 | # cf.policy_server_address = "10.0.2.15" 76 | # end 77 | 78 | # Enable provisioning with Puppet stand alone. Puppet manifests 79 | # are contained in a directory path relative to this Vagrantfile. 80 | # You will need to create the manifests directory and a manifest in 81 | # the file default.pp in the manifests_path directory. 82 | # 83 | # config.vm.provision "puppet" do |puppet| 84 | # puppet.manifests_path = "manifests" 85 | # puppet.manifest_file = "site.pp" 86 | # end 87 | 88 | # Enable provisioning with chef solo, specifying a cookbooks path, roles 89 | # path, and data_bags path (all relative to this Vagrantfile), and adding 90 | # some recipes and/or roles. 91 | # 92 | # config.vm.provision "chef_solo" do |chef| 93 | # chef.cookbooks_path = "../my-recipes/cookbooks" 94 | # chef.roles_path = "../my-recipes/roles" 95 | # chef.data_bags_path = "../my-recipes/data_bags" 96 | # chef.add_recipe "mysql" 97 | # chef.add_role "web" 98 | # 99 | # # You may also specify custom JSON attributes: 100 | # chef.json = { :mysql_password => "foo" } 101 | # end 102 | 103 | # Enable provisioning with chef server, specifying the chef server URL, 104 | # and the path to the validation key (relative to this Vagrantfile). 105 | # 106 | # The Opscode Platform uses HTTPS. Substitute your organization for 107 | # ORGNAME in the URL and validation key. 108 | # 109 | # If you have your own Chef Server, use the appropriate URL, which may be 110 | # HTTP instead of HTTPS depending on your configuration. Also change the 111 | # validation key to validation.pem. 112 | # 113 | # config.vm.provision "chef_client" do |chef| 114 | # chef.chef_server_url = "https://api.opscode.com/organizations/ORGNAME" 115 | # chef.validation_key_path = "ORGNAME-validator.pem" 116 | # end 117 | # 118 | # If you're using the Opscode platform, your validator client is 119 | # ORGNAME-validator, replacing ORGNAME with your organization name. 120 | # 121 | # If you have your own Chef Server, the default validation client name is 122 | # chef-validator, unless you changed the configuration. 123 | # 124 | # chef.validation_client_name = "ORGNAME-validator" 125 | end 126 | -------------------------------------------------------------------------------- /cyanide/vagrant/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | import os 4 | import vagrant 5 | 6 | 7 | def path(): 8 | return os.path.abspath(os.path.dirname(__file__)) 9 | 10 | 11 | class Vagrant(vagrant.Vagrant): 12 | 13 | def __init__(self, root=None, quiet_stdout=False, quiet_stderr=False, 14 | *args, **kwargs): 15 | super(Vagrant, self).__init__( 16 | root or path(), 17 | quiet_stdout=quiet_stdout, 18 | quiet_stderr=quiet_stderr, 19 | *args, **kwargs) 20 | -------------------------------------------------------------------------------- /cyanide/vagrant/provision/celeryd-init.config: -------------------------------------------------------------------------------- 1 | CELERYD_NODES="worker1" 2 | CELERY_BIN="/usr/local/bin/celery" 3 | CELERY_APP="cyanide" 4 | CELERYD_CHDIR="/opt/devel/cyanide" 5 | CELERYD_OPTS="-c10 --maxtasksperchild=256 -Z vagrant1" 6 | CELERYD_LOG_FILE="/var/log/celery/%n%I.log" 7 | CELERYD_PID_FILE="/var/run/celery/%n.pid" 8 | 9 | CELERYD_USER="celery" 10 | CELERYD_GROUP="celery" 11 | 12 | CELERY_CREATE_DIRS=1 13 | -------------------------------------------------------------------------------- /cyanide/vagrant/provision/provision.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | APT_SOURCES_LST="/etc/apt/sources.list.d/" 4 | 5 | DEVEL_DIR="/opt/devel" 6 | 7 | WGET="wget" 8 | RABBITMQCTL="rabbitmqctl" 9 | 10 | RABBITMQ_APT_URL="http://www.rabbitmq.com/debian/" 11 | RABBITMQ_APT_VER="testing main" 12 | RABBITMQ_APT_KEY="https://www.rabbitmq.com/rabbitmq-release-signing-key.asc" 13 | RABBITMQ_DEB="rabbitmq-server" 14 | 15 | RABBITMQ_USERNAME="testing" 16 | RABBITMQ_PASSWORD="t3s71ng" 17 | RABBITMQ_VHOST="/testing" 18 | 19 | REDIS_DEB="redis-server" 20 | REDIS_CONF="/etc/redis/redis.conf" 21 | 22 | GIT_ROOT="${DEVEL_DIR}" 23 | 24 | GITHUB_ROOT="https://github.com/" 25 | CELERY_GITHUB_USER="celery" 26 | CELERY_USER="celery" 27 | CELERY_GROUP="celery" 28 | CELERY_DIR="${GIT_ROOT}/celery" 29 | CELERY_CONFIG_DST="/etc/default/celeryd" 30 | CYANIDE_DIR="${GIT_ROOT}/cyanide" 31 | CELERY_CONFIG_SRC="${CYANIDE_DIR}/cyanide/vagrant/provision/celeryd-init.config" 32 | 33 | 34 | die () { 35 | echo $* 36 | exit 1 37 | } 38 | 39 | # --- grent 40 | 41 | add_real_user () { 42 | user_shell=${3:-/bin/bash} 43 | addgroup $2 44 | echo creating user "$1 group='$2' shell='${user_shell}'" 45 | echo | adduser -q "$1" --shell="${user_shell}" \ 46 | --ingroup="$2" \ 47 | --disabled-password 1>/dev/null 2>&1 48 | id "$1" || die "Not able to create user" 49 | } 50 | 51 | # --- system 52 | 53 | make_directories () { 54 | mkdir -p "${DEVEL_DIR}" 55 | } 56 | 57 | enable_bash_vi_mode () { 58 | echo "set -o vi" >> /etc/bash.bashrc 59 | } 60 | 61 | configure_system () { 62 | make_directories 63 | enable_bash_vi_mode 64 | } 65 | 66 | 67 | # --- apt 68 | 69 | apt_update() { 70 | apt-get update 71 | } 72 | 73 | add_apt_source () { 74 | echo "deb $1" >> "${APT_SOURCES_LST}/rabbitmq.list" 75 | } 76 | 77 | add_apt_key() { 78 | "$WGET" --quiet -O - "$1" | apt-key add - 79 | } 80 | 81 | apt_install () { 82 | apt-get install -y "$1" 83 | } 84 | 85 | # --- rabbitmq 86 | 87 | rabbitmq_add_user () { 88 | "$RABBITMQCTL" add_user "$1" "$2" 89 | } 90 | 91 | rabbitmq_add_vhost () { 92 | "$RABBITMQCTL" add_vhost "$1" 93 | } 94 | 95 | rabbitmq_set_perm () { 96 | "$RABBITMQCTL" set_permissions -p $1 $2 '.*' '.*' '.*' 97 | } 98 | 99 | install_rabbitmq() { 100 | add_apt_source "${RABBITMQ_APT_URL} ${RABBITMQ_APT_VER}" 101 | add_apt_key "${RABBITMQ_APT_KEY}" 102 | apt_update 103 | apt_install "${RABBITMQ_DEB}" 104 | 105 | rabbitmq_add_user "${RABBITMQ_USERNAME}" "${RABBITMQ_PASSWORD}" 106 | rabbitmq_add_vhost "${RABBITMQ_VHOST}" 107 | rabbitmq_set_perm "${RABBITMQ_VHOST}" "${RABBITMQ_USERNAME}" 108 | } 109 | 110 | # --- redis 111 | 112 | restart_redis () { 113 | service redis-server restart 114 | } 115 | 116 | 117 | install_redis () { 118 | apt_install "${REDIS_DEB}" 119 | sed -i 's/^bind .*$/#bind 127.0.0.1/' "${REDIS_CONF}" 120 | restart_redis 121 | } 122 | 123 | # --- git 124 | 125 | install_git () { 126 | apt_install git 127 | } 128 | 129 | 130 | github_clone () { 131 | dir="$3" 132 | mkdir "$dir" 133 | chown "${CELERY_USER}" "$dir" 134 | (cd "${GIT_ROOT}"; sudo -u celery git clone "${GITHUB_ROOT}/${1}/${2}") 135 | } 136 | 137 | # --- pip 138 | 139 | pip_install () { 140 | pip install -U "$1" 141 | } 142 | 143 | install_pip () { 144 | apt_install python-setuptools 145 | easy_install pip 146 | pip_install virtualenv 147 | apt_install python-dev 148 | pip_install setproctitle 149 | } 150 | 151 | # --- celery 152 | 153 | restart_celery () { 154 | service celeryd restart 155 | } 156 | 157 | 158 | install_celery_service () { 159 | cp "${CELERY_DIR}/extra/generic-init.d/celeryd" /etc/init.d/ 160 | chmod +x "/etc/init.d/celeryd" 161 | update-rc.d celeryd defaults 162 | echo "cp \'${CELERY_CONFIG_SRC}\' \'${CELERY_CONFIG_DST}'" 163 | cp "${CELERY_CONFIG_SRC}" "${CELERY_CONFIG_DST}" 164 | update-rc.d celeryd enable 165 | restart_celery 166 | } 167 | 168 | install_celery () { 169 | pip_install celery 170 | add_real_user "${CELERY_USER}" "${CELERY_GROUP}" 171 | echo github_clone "'${CELERY_GITHUB_USER}'" "'celery'" 172 | github_clone "${CELERY_GITHUB_USER}" celery "${CELERY_DIR}" 173 | (cd ${CELERY_DIR}; pip install -r requirements/dev.txt); 174 | (cd ${CELERY_DIR}; python setup.py develop); 175 | } 176 | 177 | install_cyanide () { 178 | echo github_clone "'${CELERY_GITHUB_USER}'" "'cyanide'" 179 | github_clone "${CELERY_GITHUB_USER}" cyanide "${CYANIDE_DIR}" 180 | } 181 | 182 | # --- MAIN 183 | 184 | provision () { 185 | apt_update 186 | configure_system 187 | apt_install powertop 188 | apt_install htop 189 | install_git 190 | install_rabbitmq 191 | install_redis 192 | install_pip 193 | install_celery 194 | install_cyanide 195 | install_celery_service 196 | } 197 | 198 | provision 199 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don\'t have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help 23 | help: 24 | @echo "Please use \`make ' where is one of" 25 | @echo " html to make standalone HTML files" 26 | @echo " dirhtml to make HTML files named index.html in directories" 27 | @echo " singlehtml to make a single large HTML file" 28 | @echo " pickle to make pickle files" 29 | @echo " json to make JSON files" 30 | @echo " htmlhelp to make HTML files and a HTML help project" 31 | @echo " qthelp to make HTML files and a qthelp project" 32 | @echo " applehelp to make an Apple Help Book" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " epub3 to make an epub3" 36 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 37 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 38 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 39 | @echo " text to make text files" 40 | @echo " man to make manual pages" 41 | @echo " texinfo to make Texinfo files" 42 | @echo " info to make Texinfo files and run them through makeinfo" 43 | @echo " gettext to make PO message catalogs" 44 | @echo " changes to make an overview of all changed/added/deprecated items" 45 | @echo " xml to make Docutils-native XML files" 46 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 47 | @echo " linkcheck to check all external links for integrity" 48 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 49 | @echo " coverage to run coverage check of the documentation (if enabled)" 50 | @echo " apicheck to verify that all modules are present in autodoc" 51 | 52 | .PHONY: clean 53 | clean: 54 | rm -rf $(BUILDDIR)/* 55 | 56 | .PHONY: html 57 | html: 58 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 59 | @echo 60 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 61 | 62 | .PHONY: dirhtml 63 | dirhtml: 64 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 65 | @echo 66 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 67 | 68 | .PHONY: singlehtml 69 | singlehtml: 70 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 71 | @echo 72 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 73 | 74 | .PHONY: pickle 75 | pickle: 76 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 77 | @echo 78 | @echo "Build finished; now you can process the pickle files." 79 | 80 | .PHONY: json 81 | json: 82 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 83 | @echo 84 | @echo "Build finished; now you can process the JSON files." 85 | 86 | .PHONY: htmlhelp 87 | htmlhelp: 88 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 89 | @echo 90 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 91 | ".hhp project file in $(BUILDDIR)/htmlhelp." 92 | 93 | .PHONY: qthelp 94 | qthelp: 95 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 96 | @echo 97 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 98 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 99 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/PROJ.qhcp" 100 | @echo "To view the help file:" 101 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/PROJ.qhc" 102 | 103 | .PHONY: applehelp 104 | applehelp: 105 | $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp 106 | @echo 107 | @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." 108 | @echo "N.B. You won't be able to view it unless you put it in" \ 109 | "~/Library/Documentation/Help or install it in your application" \ 110 | "bundle." 111 | 112 | .PHONY: devhelp 113 | devhelp: 114 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 115 | @echo 116 | @echo "Build finished." 117 | @echo "To view the help file:" 118 | @echo "# mkdir -p $$HOME/.local/share/devhelp/PROJ" 119 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/PROJ" 120 | @echo "# devhelp" 121 | 122 | .PHONY: epub 123 | epub: 124 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 125 | @echo 126 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 127 | 128 | .PHONY: epub3 129 | epub3: 130 | $(SPHINXBUILD) -b epub3 $(ALLSPHINXOPTS) $(BUILDDIR)/epub3 131 | @echo 132 | @echo "Build finished. The epub3 file is in $(BUILDDIR)/epub3." 133 | 134 | .PHONY: latex 135 | latex: 136 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 137 | @echo 138 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 139 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 140 | "(use \`make latexpdf' here to do that automatically)." 141 | 142 | .PHONY: latexpdf 143 | latexpdf: 144 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 145 | @echo "Running LaTeX files through pdflatex..." 146 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 147 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 148 | 149 | .PHONY: latexpdfja 150 | latexpdfja: 151 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 152 | @echo "Running LaTeX files through platex and dvipdfmx..." 153 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 154 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 155 | 156 | .PHONY: text 157 | text: 158 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 159 | @echo 160 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 161 | 162 | .PHONY: man 163 | man: 164 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 165 | @echo 166 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 167 | 168 | .PHONY: texinfo 169 | texinfo: 170 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 171 | @echo 172 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 173 | @echo "Run \`make' in that directory to run these through makeinfo" \ 174 | "(use \`make info' here to do that automatically)." 175 | 176 | .PHONY: info 177 | info: 178 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 179 | @echo "Running Texinfo files through makeinfo..." 180 | make -C $(BUILDDIR)/texinfo info 181 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 182 | 183 | .PHONY: gettext 184 | gettext: 185 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 186 | @echo 187 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 188 | 189 | .PHONY: changes 190 | changes: 191 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 192 | @echo 193 | @echo "The overview file is in $(BUILDDIR)/changes." 194 | 195 | .PHONY: linkcheck 196 | linkcheck: 197 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 198 | @echo 199 | @echo "Link check complete; look for any errors in the above output " \ 200 | "or in $(BUILDDIR)/linkcheck/output.txt." 201 | 202 | .PHONY: doctest 203 | doctest: 204 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 205 | @echo "Testing of doctests in the sources finished, look at the " \ 206 | "results in $(BUILDDIR)/doctest/output.txt." 207 | 208 | .PHONY: coverage 209 | coverage: 210 | $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage 211 | @echo "Testing of coverage in the sources finished, look at the " \ 212 | "results in $(BUILDDIR)/coverage/python.txt." 213 | 214 | .PHONY: apicheck 215 | apicheck: 216 | $(SPHINXBUILD) -b apicheck $(ALLSPHINXOPTS) $(BUILDDIR)/apicheck 217 | 218 | .PHONY: xml 219 | xml: 220 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 221 | @echo 222 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 223 | 224 | .PHONY: pseudoxml 225 | pseudoxml: 226 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 227 | @echo 228 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 229 | -------------------------------------------------------------------------------- /docs/_static/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/celery/cyanide/9b7d78dd4d804ed74c50f0ab9492f19ac06dc9ca/docs/_static/.keep -------------------------------------------------------------------------------- /docs/_templates/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/celery/cyanide/9b7d78dd4d804ed74c50f0ab9492f19ac06dc9ca/docs/_templates/.keep -------------------------------------------------------------------------------- /docs/changelog.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../Changelog 2 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import absolute_import, unicode_literals 3 | 4 | import os 5 | 6 | from sphinx_celery import conf 7 | 8 | globals().update(conf.build_config( 9 | 'cyanide', __file__, 10 | project='Cyanide', 11 | # version_dev='2.0', 12 | # version_stable='1.4', 13 | canonical_url='https://cyanide.readthedocs.io', 14 | webdomain='', 15 | github_project='celery/cyanide', 16 | copyright='2013-2016', 17 | html_logo='images/logo.png', 18 | html_favicon='images/favicon.ico', 19 | html_prepend_sidebars=[], 20 | include_intersphinx={'python', 'sphinx'}, 21 | # django_settings='testproj.settings', 22 | # path_additions=[os.path.join(os.pardir, 'testproj')], 23 | apicheck_package='cyanide', 24 | apicheck_ignore_modules=[ 25 | 'cyanide.__main__', 26 | 'cyanide.bin', 27 | 'cyanide.suites', 28 | ], 29 | )) 30 | -------------------------------------------------------------------------------- /docs/copyright.rst: -------------------------------------------------------------------------------- 1 | Copyright 2 | ========= 3 | 4 | *Cyanide User Manual* 5 | 6 | by Ask Solem 7 | 8 | .. |copy| unicode:: U+000A9 .. COPYRIGHT SIGN 9 | 10 | Copyright |copy| 2016, Ask Solem 11 | 12 | All rights reserved. This material may be copied or distributed only 13 | subject to the terms and conditions set forth in the `Creative Commons 14 | Attribution-ShareAlike 4.0 International` 15 | `_ license. 16 | 17 | You may share and adapt the material, even for commercial purposes, but 18 | you must give the original author credit. 19 | If you alter, transform, or build upon this 20 | work, you may distribute the resulting work only under the same license or 21 | a license compatible to this one. 22 | 23 | .. note:: 24 | 25 | While the Cyanide is offered under the 26 | Creative Commons *Attribution-ShareAlike 4.0 International* license 27 | the Cyanide is offered under the 28 | `BSD License (3 Clause) `_ 29 | -------------------------------------------------------------------------------- /docs/glossary.rst: -------------------------------------------------------------------------------- 1 | .. _glossary: 2 | 3 | Glossary 4 | ======== 5 | 6 | .. glossary:: 7 | :sorted: 8 | 9 | term 10 | Description of term 11 | -------------------------------------------------------------------------------- /docs/images/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/celery/cyanide/9b7d78dd4d804ed74c50f0ab9492f19ac06dc9ca/docs/images/favicon.ico -------------------------------------------------------------------------------- /docs/images/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/celery/cyanide/9b7d78dd4d804ed74c50f0ab9492f19ac06dc9ca/docs/images/logo.png -------------------------------------------------------------------------------- /docs/includes/installation.txt: -------------------------------------------------------------------------------- 1 | .. _installation: 2 | 3 | Installation 4 | ============ 5 | 6 | You can install cyanide either via the Python Package Index (PyPI) 7 | or from source. 8 | 9 | To install using `pip`: 10 | 11 | .. code-block:: console 12 | 13 | $ pip install -U cyanide 14 | 15 | .. _installing-from-source: 16 | 17 | Downloading and installing from source 18 | -------------------------------------- 19 | 20 | Download the latest version of cyanide from 21 | http://pypi.python.org/pypi/cyanide 22 | 23 | You can install it by doing the following: 24 | 25 | .. code-block:: console 26 | 27 | $ tar xvfz cyanide-0.0.0.tar.gz 28 | $ cd cyanide-0.0.0 29 | $ python setup.py build 30 | # python setup.py install 31 | 32 | The last command must be executed as a privileged user if 33 | you are not currently using a virtualenv. 34 | 35 | .. _installing-from-git: 36 | 37 | Using the development version 38 | ----------------------------- 39 | 40 | With pip 41 | ~~~~~~~~ 42 | 43 | You can install the latest snapshot of cyanide using the following 44 | pip command: 45 | 46 | .. code-block:: console 47 | 48 | $ pip install https://github.com/celery/cyanide/zipball/master#egg=cyanide 49 | -------------------------------------------------------------------------------- /docs/includes/introduction.txt: -------------------------------------------------------------------------------- 1 | :Version: 1.3.0 2 | :Web: https://cyanide.readthedocs.io/ 3 | :Download: http://pypi.python.org/pypi/cyanide/ 4 | :Source: http://github.com/celery/cyanide/ 5 | :Keywords: celery, stress, integration, functional, testing 6 | 7 | .. contents:: 8 | :local: 9 | 10 | Introduction 11 | ============ 12 | 13 | This stress test suite will attempt to break the Celery worker in different 14 | ways, and can also be used to write new stress test suites for projects 15 | depending on Celery infrastructure. 16 | 17 | The worker must currently be started separately, and it's encouraged 18 | to repeat the suite using workers started with different configuration values. 19 | 20 | Ideas include: 21 | 22 | #. Default, single process: 23 | 24 | .. code-block:: console 25 | 26 | $ celery -A cyanide worker -c 1 27 | 28 | #. Default, multiple processes: 29 | 30 | .. code-block:: console 31 | 32 | $ celery -A cyanide worker -c 8 33 | 34 | #. Frequent ``maxtasksperchild`` recycling, single child process: 35 | 36 | .. code-block:: console 37 | 38 | $ celery -A cyanide worker -c 1 --maxtasksperchild=1 39 | 40 | #. Frequent autoscale scale down & ``maxtasksperchild``, single child process: 41 | 42 | .. code-block:: console 43 | 44 | $ AUTOSCALE_KEEPALIVE=0.01 celery -A cyanide worker \ 45 | > --autoscale=1,0 --maxtasksperchild=1 46 | 47 | #. Frequent ``maxtasksperchild``, multiple child processes: 48 | 49 | .. code-block:: console 50 | 51 | $ celery -A cyanide worker -c 8 --maxtasksperchild=1 52 | 53 | #. Processes terminated by time limits: 54 | 55 | .. code-block:: console 56 | 57 | $ celery -A cyanide worker --time-limit=1 58 | 59 | #. Frequent ``maxtasksperchild``, single child process with late ack: 60 | 61 | .. code-block:: console 62 | 63 | $ celery -A cyanide worker -c1 --maxtasksperchild=1 -Z acks_late 64 | 65 | #. Worker using the :pypi:`eventlet` pool: 66 | 67 | Start the worker, here having a thousand green-threads: 68 | 69 | .. code-block:: console 70 | 71 | $ celery -A cyanide worker -c1000 -P eventlet 72 | 73 | You must activate the `green` test group when starting the test suite: 74 | 75 | .. code-block:: console 76 | 77 | $ celery cyanide -g green 78 | 79 | #. Worker using the :pypi:`gevent` pool: 80 | 81 | Start the worker, here having a thousand green-threads: 82 | 83 | .. code-block:: console 84 | 85 | $ celery -A cyanide worker -c1000 -P gevent 86 | 87 | You must activate the `green` test group when starting the test suite: 88 | 89 | .. code-block:: console 90 | 91 | $ celery cyanide -g green 92 | 93 | Tips 94 | ==== 95 | 96 | It's a good idea to include the :option:`--purge ` 97 | argument to clear out tasks from previous runs. 98 | 99 | Note that the stress client will probably hang if the test fails, so this 100 | test suite is currently not suited for automatic runs. 101 | 102 | Configuration Templates 103 | ======================= 104 | 105 | You can select a configuration template using the `-Z` command-line argument 106 | to any :program:`celery -A cyanide` command or the :program:`celery cyanide` 107 | command used to execute the test suite. 108 | 109 | The templates available are: 110 | 111 | * ``default`` 112 | 113 | Using AMQP as a broker, RPC as a result backend, 114 | and using JSON serialization for task and result messages. 115 | 116 | Both broker and result store is expected to run at localhost. 117 | 118 | * ``vagrant1`` 119 | 120 | Use the VM started by :command:`celery vagrant up` as the broker 121 | and result backend (RabbitMQ). 122 | 123 | * ``vagrant1_redis`` 124 | 125 | Use the VM started by :command:`celery vagrant up` as the broker 126 | and result backend (Redis). 127 | 128 | * ``redis`` 129 | 130 | Using Redis as a broker and result backend. 131 | 132 | * ``redistore`` 133 | 134 | Using Redis as a result backend only. 135 | 136 | * ``acks_late`` 137 | 138 | Enables late ack globally. 139 | 140 | * ``pickle`` 141 | 142 | Using pickle as the serializer for tasks and results 143 | (also allowing the worker to receive and process pickled messages) 144 | 145 | * ``confirms`` 146 | 147 | Enables RabbitMQ publisher confirmations. 148 | 149 | * ``events`` 150 | 151 | Configure workers to send task events. 152 | 153 | * ``proto1`` 154 | 155 | Use version 1 of the task message protocol (pre 4.0) 156 | 157 | You can see the resulting configuration from any template by running 158 | the command: 159 | 160 | .. code-block:: console 161 | 162 | $ celery -A cyanide report -Z redis 163 | 164 | Examples 165 | -------- 166 | 167 | Example running the stress test using the ``redis`` configuration template: 168 | 169 | .. code-block:: console 170 | 171 | $ cyanide -Z redis 172 | 173 | Example running the worker using the ``redis`` configuration template: 174 | 175 | .. code-block:: console 176 | 177 | $ celery -A cyanide worker -Z redis 178 | 179 | You can also mix several templates by providing a comma-separated list: 180 | 181 | .. code-block:: console 182 | 183 | $ celery -A cyanide worker -Z redis,acks_late 184 | 185 | In this example (``redis,acks_late``) the ``redis`` template will be used 186 | as main configuration, and then the additional keys from the ``acks_late`` template 187 | will be merged as changes. 188 | 189 | Test Suite Options 190 | ================== 191 | 192 | After one or more worker instances are running, you can start executing the 193 | tests. 194 | 195 | By default the complete test suite will be executed: 196 | 197 | .. code-block:: console 198 | 199 | $ celery cyanide 200 | 201 | You can also specify what test cases to run by providing one or more names 202 | as arguments: 203 | 204 | .. code-block:: console 205 | 206 | $ celery cyanide revoketermfast revoketermslow 207 | 208 | A full list of test case names can be retrieved with the 209 | :option:`-l ` switch: 210 | 211 | .. code-block:: console 212 | 213 | $ celery cyanide -l 214 | .> 1) chain, 215 | .> 2) chaincomplex, 216 | .> 3) parentids_chain, 217 | .> 4) parentids_group, 218 | .> 5) manyshort, 219 | .> 6) unicodetask, 220 | .> 7) always_timeout, 221 | .> 8) termbysig, 222 | .> 9) timelimits, 223 | .> 10) timelimits_soft, 224 | .> 11) alwayskilled, 225 | .> 12) alwaysexits, 226 | .> 13) bigtasksbigvalue, 227 | .> 14) bigtasks, 228 | .> 15) smalltasks, 229 | .> 16) revoketermfast, 230 | .> 17) revoketermslow 231 | 232 | You can also start from an offset within this list, e.g. to skip the first two 233 | tests use :option:`--offset=2 `: 234 | 235 | .. code-block:: console 236 | 237 | $ celery cyanide --offset=2 238 | 239 | See :command:`celery cyanide --help` for a list of all available 240 | command-line options. 241 | 242 | Vagrant 243 | ======= 244 | 245 | Starting 246 | -------- 247 | 248 | Cyanide ships with a complete virtual machine solution to run your tests. 249 | The image ships with Celery, Cyanide, RabbitMQ and Redis and can be deployed 250 | simply by running the :program:`celery vagrant` command: 251 | 252 | .. code-block:: console 253 | 254 | $ celery vagrant up 255 | 256 | 257 | The IP address of the new virtual machine will be 192.168.33.123, 258 | and you can easily tell both the worker and cyanide test suite to use 259 | it by specifying the ``vagrant1`` (RabbitMQ) or ``vagrant1_redis`` templates: 260 | 261 | .. code-block:: console 262 | 263 | $ celery -A worker -Z vagrant1 264 | $ celery cyanide -Z vagrant1 265 | 266 | SSH 267 | --- 268 | 269 | To open an SSH session with the virtual machine after starting 270 | with :command:`celery vagrant up` do: 271 | 272 | .. code-block:: console 273 | 274 | $ ssh $(celery vagrant sshargs) 275 | 276 | Stopping 277 | -------- 278 | 279 | To shutdown the virtual machine run the command: 280 | 281 | .. code-block:: console 282 | 283 | $ celery vagrant halt 284 | 285 | To destroy the instance run the command: 286 | 287 | .. code-block:: console 288 | 289 | $ celery vagrant destroy 290 | 291 | 292 | .. note:: 293 | 294 | To completely wipe your instance you need to remove the 295 | :file:`.vagrant` directory. 296 | 297 | The location of this directory can be retrieved by executing 298 | the following: 299 | 300 | .. code-block:: console 301 | 302 | $ celery vagrant statedir 303 | /opt/devel/cyanide/cyanide/vagrant/.vagrant 304 | 305 | You can combine this with :command:`rm` to force removal of this 306 | directory: 307 | 308 | .. code-block:: console 309 | 310 | $ rm -rf $(celery vagrant statedir) 311 | 312 | Environment Variables 313 | ===================== 314 | 315 | ``CYANIDE_TRANS`` 316 | ----------------- 317 | 318 | If the :envvar:`CYANIDE_TRANS` environment variable is set 319 | the stress test suite will use transient task messages instead of persisting 320 | messages to disk. 321 | 322 | To avoid declaration collision the ``cstress.trans`` queue name will be used 323 | when this option is enabled. 324 | 325 | ``CYANIDE_BROKER`` 326 | ------------------ 327 | 328 | You can set the :envvar:`CYANIDE_BROKER` environment variable 329 | to change the default broker used: 330 | 331 | .. code-block:: console 332 | 333 | $ CYANIDE_BROKER='amqp://' celery -A cyanide worker # … 334 | $ CYANIDE_BROKER='amqp://' celery cyanide 335 | 336 | ``CYANIDE_BACKEND`` 337 | ------------------- 338 | 339 | You can set the :envvar:`CYANIDE_BACKEND` environment variable to change 340 | the result backend used: 341 | 342 | .. code-block:: console 343 | 344 | $ CYANIDE_BACKEND='amqp://' celery -A cyanide worker # … 345 | $ CYANIDE_BACKEND='amqp://' celery cyanide 346 | 347 | ``CYANIDE_QUEUE`` 348 | ----------------- 349 | 350 | A queue named ``c.stress`` is created and used by default for all task 351 | communication. 352 | 353 | You can change the name of this queue using the :envvar:`CYANIDE_QUEUE` 354 | environment variable: 355 | 356 | .. code-block:: console 357 | 358 | $ CYANIDE_QUEUE='cyanide' celery -A cyanide worker # … 359 | $ CYANIDE_QUEUE='cyanide' celery cyanide 360 | 361 | ``CYANIDE_PREFETCH`` 362 | -------------------- 363 | 364 | The :envvar:`CYANIDE_PREFETCH` environment variable sets the default prefetch 365 | multiplier (default value is 10). 366 | 367 | ``AWS_REGION`` 368 | -------------- 369 | 370 | The :envvar:`AWS_REGION` environment variable changes the Amazon AWS region 371 | to something other than the default ``us-east-1``, to be used with the 372 | ``sqs`` template. 373 | 374 | 375 | Custom Suites 376 | ============= 377 | 378 | You can define custom suites (look at source code of 379 | :mod:`cyanide.suites.default` for inspiration), and tell cyanide to use that 380 | suite by specifying the :option:`celery cyanide -S` option: 381 | 382 | .. code-block:: console 383 | 384 | $ celery cyanide -S proj.funtests:MySuite 385 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | ======================================================================= 2 | Cyanide - Celery stress testing and integration test support. 3 | ======================================================================= 4 | 5 | .. include:: includes/introduction.txt 6 | 7 | Contents 8 | ======== 9 | 10 | .. toctree:: 11 | :maxdepth: 1 12 | 13 | copyright 14 | 15 | .. toctree:: 16 | :maxdepth: 2 17 | 18 | reference/index 19 | 20 | .. toctree:: 21 | :maxdepth: 1 22 | 23 | changelog 24 | glossary 25 | 26 | Indices and tables 27 | ================== 28 | 29 | * :ref:`genindex` 30 | * :ref:`modindex` 31 | * :ref:`search` 32 | 33 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. epub3 to make an epub3 31 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 32 | echo. text to make text files 33 | echo. man to make manual pages 34 | echo. texinfo to make Texinfo files 35 | echo. gettext to make PO message catalogs 36 | echo. changes to make an overview over all changed/added/deprecated items 37 | echo. xml to make Docutils-native XML files 38 | echo. pseudoxml to make pseudoxml-XML files for display purposes 39 | echo. linkcheck to check all external links for integrity 40 | echo. doctest to run all doctests embedded in the documentation if enabled 41 | echo. coverage to run coverage check of the documentation if enabled 42 | goto end 43 | ) 44 | 45 | if "%1" == "clean" ( 46 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 47 | del /q /s %BUILDDIR%\* 48 | goto end 49 | ) 50 | 51 | 52 | REM Check if sphinx-build is available and fallback to Python version if any 53 | %SPHINXBUILD% 1>NUL 2>NUL 54 | if errorlevel 9009 goto sphinx_python 55 | goto sphinx_ok 56 | 57 | :sphinx_python 58 | 59 | set SPHINXBUILD=python -m sphinx.__init__ 60 | %SPHINXBUILD% 2> nul 61 | if errorlevel 9009 ( 62 | echo. 63 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 64 | echo.installed, then set the SPHINXBUILD environment variable to point 65 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 66 | echo.may add the Sphinx directory to PATH. 67 | echo. 68 | echo.If you don't have Sphinx installed, grab it from 69 | echo.http://sphinx-doc.org/ 70 | exit /b 1 71 | ) 72 | 73 | :sphinx_ok 74 | 75 | 76 | if "%1" == "html" ( 77 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 78 | if errorlevel 1 exit /b 1 79 | echo. 80 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 81 | goto end 82 | ) 83 | 84 | if "%1" == "dirhtml" ( 85 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 86 | if errorlevel 1 exit /b 1 87 | echo. 88 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 89 | goto end 90 | ) 91 | 92 | if "%1" == "singlehtml" ( 93 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 94 | if errorlevel 1 exit /b 1 95 | echo. 96 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 97 | goto end 98 | ) 99 | 100 | if "%1" == "pickle" ( 101 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 102 | if errorlevel 1 exit /b 1 103 | echo. 104 | echo.Build finished; now you can process the pickle files. 105 | goto end 106 | ) 107 | 108 | if "%1" == "json" ( 109 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 110 | if errorlevel 1 exit /b 1 111 | echo. 112 | echo.Build finished; now you can process the JSON files. 113 | goto end 114 | ) 115 | 116 | if "%1" == "htmlhelp" ( 117 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 118 | if errorlevel 1 exit /b 1 119 | echo. 120 | echo.Build finished; now you can run HTML Help Workshop with the ^ 121 | .hhp project file in %BUILDDIR%/htmlhelp. 122 | goto end 123 | ) 124 | 125 | if "%1" == "qthelp" ( 126 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 127 | if errorlevel 1 exit /b 1 128 | echo. 129 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 130 | .qhcp project file in %BUILDDIR%/qthelp, like this: 131 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\PROJ.qhcp 132 | echo.To view the help file: 133 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\PROJ.ghc 134 | goto end 135 | ) 136 | 137 | if "%1" == "devhelp" ( 138 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 139 | if errorlevel 1 exit /b 1 140 | echo. 141 | echo.Build finished. 142 | goto end 143 | ) 144 | 145 | if "%1" == "epub" ( 146 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 147 | if errorlevel 1 exit /b 1 148 | echo. 149 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 150 | goto end 151 | ) 152 | 153 | if "%1" == "epub3" ( 154 | %SPHINXBUILD% -b epub3 %ALLSPHINXOPTS% %BUILDDIR%/epub3 155 | if errorlevel 1 exit /b 1 156 | echo. 157 | echo.Build finished. The epub3 file is in %BUILDDIR%/epub3. 158 | goto end 159 | ) 160 | 161 | if "%1" == "latex" ( 162 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 163 | if errorlevel 1 exit /b 1 164 | echo. 165 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 166 | goto end 167 | ) 168 | 169 | if "%1" == "latexpdf" ( 170 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 171 | cd %BUILDDIR%/latex 172 | make all-pdf 173 | cd %~dp0 174 | echo. 175 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 176 | goto end 177 | ) 178 | 179 | if "%1" == "latexpdfja" ( 180 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 181 | cd %BUILDDIR%/latex 182 | make all-pdf-ja 183 | cd %~dp0 184 | echo. 185 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 186 | goto end 187 | ) 188 | 189 | if "%1" == "text" ( 190 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 191 | if errorlevel 1 exit /b 1 192 | echo. 193 | echo.Build finished. The text files are in %BUILDDIR%/text. 194 | goto end 195 | ) 196 | 197 | if "%1" == "man" ( 198 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 199 | if errorlevel 1 exit /b 1 200 | echo. 201 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 202 | goto end 203 | ) 204 | 205 | if "%1" == "texinfo" ( 206 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 207 | if errorlevel 1 exit /b 1 208 | echo. 209 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 210 | goto end 211 | ) 212 | 213 | if "%1" == "gettext" ( 214 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 215 | if errorlevel 1 exit /b 1 216 | echo. 217 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 218 | goto end 219 | ) 220 | 221 | if "%1" == "changes" ( 222 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 223 | if errorlevel 1 exit /b 1 224 | echo. 225 | echo.The overview file is in %BUILDDIR%/changes. 226 | goto end 227 | ) 228 | 229 | if "%1" == "linkcheck" ( 230 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 231 | if errorlevel 1 exit /b 1 232 | echo. 233 | echo.Link check complete; look for any errors in the above output ^ 234 | or in %BUILDDIR%/linkcheck/output.txt. 235 | goto end 236 | ) 237 | 238 | if "%1" == "doctest" ( 239 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 240 | if errorlevel 1 exit /b 1 241 | echo. 242 | echo.Testing of doctests in the sources finished, look at the ^ 243 | results in %BUILDDIR%/doctest/output.txt. 244 | goto end 245 | ) 246 | 247 | if "%1" == "coverage" ( 248 | %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage 249 | if errorlevel 1 exit /b 1 250 | echo. 251 | echo.Testing of coverage in the sources finished, look at the ^ 252 | results in %BUILDDIR%/coverage/python.txt. 253 | goto end 254 | ) 255 | 256 | if "%1" == "xml" ( 257 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 258 | if errorlevel 1 exit /b 1 259 | echo. 260 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 261 | goto end 262 | ) 263 | 264 | if "%1" == "pseudoxml" ( 265 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 266 | if errorlevel 1 exit /b 1 267 | echo. 268 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 269 | goto end 270 | ) 271 | 272 | :end 273 | -------------------------------------------------------------------------------- /docs/reference/cyanide.app.rst: -------------------------------------------------------------------------------- 1 | ===================================================== 2 | cyanide.app 3 | ===================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cyanide.app 8 | 9 | .. automodule:: cyanide.app 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cyanide.bin.cyanide.rst: -------------------------------------------------------------------------------- 1 | ===================================================== 2 | cyanide.bin.cyanide 3 | ===================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cyanide.bin.cyanide 8 | 9 | .. automodule:: cyanide.bin.cyanide 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cyanide.bin.vagrant.rst: -------------------------------------------------------------------------------- 1 | ===================================================== 2 | cyanide.bin.vagrant 3 | ===================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cyanide.bin.vagrant 8 | 9 | .. automodule:: cyanide.bin.vagrant 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cyanide.compat.rst: -------------------------------------------------------------------------------- 1 | ===================================================== 2 | cyanide.compat 3 | ===================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cyanide.compat 8 | 9 | .. automodule:: cyanide.compat 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cyanide.data.rst: -------------------------------------------------------------------------------- 1 | ===================================================== 2 | cyanide.data 3 | ===================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cyanide.data 8 | 9 | .. automodule:: cyanide.data 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cyanide.fbi.rst: -------------------------------------------------------------------------------- 1 | ===================================================== 2 | cyanide.fbi 3 | ===================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cyanide.fbi 8 | 9 | .. automodule:: cyanide.fbi 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cyanide.rst: -------------------------------------------------------------------------------- 1 | ===================================================== 2 | cyanide 3 | ===================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cyanide 8 | 9 | .. automodule:: cyanide 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cyanide.suite.rst: -------------------------------------------------------------------------------- 1 | ===================================================== 2 | cyanide.suite 3 | ===================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cyanide.suite 8 | 9 | .. automodule:: cyanide.suite 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cyanide.suites.default.rst: -------------------------------------------------------------------------------- 1 | ===================================================== 2 | cyanide.suites.default 3 | ===================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cyanide.suites.default 8 | 9 | .. automodule:: cyanide.suites.default 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cyanide.tasks.rst: -------------------------------------------------------------------------------- 1 | ===================================================== 2 | cyanide.tasks 3 | ===================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cyanide.tasks 8 | 9 | .. automodule:: cyanide.tasks 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cyanide.templates.rst: -------------------------------------------------------------------------------- 1 | ===================================================== 2 | cyanide.templates 3 | ===================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cyanide.templates 8 | 9 | .. automodule:: cyanide.templates 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cyanide.vagrant.rst: -------------------------------------------------------------------------------- 1 | ===================================================== 2 | cyanide.vagrant 3 | ===================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cyanide.vagrant 8 | 9 | .. automodule:: cyanide.vagrant 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/index.rst: -------------------------------------------------------------------------------- 1 | .. _apiref: 2 | 3 | =============== 4 | API Reference 5 | =============== 6 | 7 | :Release: |version| 8 | :Date: |today| 9 | 10 | .. toctree:: 11 | :maxdepth: 1 12 | 13 | cyanide 14 | cyanide.app 15 | cyanide.bin.cyanide 16 | cyanide.bin.vagrant 17 | cyanide.suite 18 | cyanide.suites.default 19 | cyanide.tasks 20 | cyanide.vagrant 21 | cyanide.templates 22 | cyanide.data 23 | cyanide.fbi 24 | cyanide.compat 25 | -------------------------------------------------------------------------------- /docs/templates/readme.txt: -------------------------------------------------------------------------------- 1 | ===================================================================== 2 | Cyanide - Celery stress testing and integration test support. 3 | ===================================================================== 4 | 5 | |build-status| |coverage| |license| |wheel| |pyversion| |pyimp| 6 | 7 | .. include:: ../includes/introduction.txt 8 | 9 | .. include:: ../includes/installation.txt 10 | 11 | .. |build-status| image:: https://secure.travis-ci.org/celery/cyanide.png?branch=master 12 | :alt: Build status 13 | :target: https://travis-ci.org/celery/cyanide 14 | 15 | .. |coverage| image:: https://codecov.io/github/celery/cyanide/coverage.svg?branch=master 16 | :target: https://codecov.io/github/celery/cyanide?branch=master 17 | 18 | .. |license| image:: https://img.shields.io/pypi/l/cyanide.svg 19 | :alt: BSD License 20 | :target: https://opensource.org/licenses/BSD-3-Clause 21 | 22 | .. |wheel| image:: https://img.shields.io/pypi/wheel/cyanide.svg 23 | :alt: Cyanide can be installed via wheel 24 | :target: http://pypi.python.org/pypi/cyanide/ 25 | 26 | .. |pyversion| image:: https://img.shields.io/pypi/pyversions/cyanide.svg 27 | :alt: Supported Python versions. 28 | :target: http://pypi.python.org/pypi/cyanide/ 29 | 30 | .. |pyimp| image:: https://img.shields.io/pypi/implementation/cyanide.svg 31 | :alt: Support Python implementations. 32 | :target: http://pypi.python.org/pypi/cyanide/ 33 | -------------------------------------------------------------------------------- /extra/appveyor/install.ps1: -------------------------------------------------------------------------------- 1 | # Sample script to install Python and pip under Windows 2 | # Authors: Olivier Grisel and Kyle Kastner 3 | # License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ 4 | 5 | $BASE_URL = "https://www.python.org/ftp/python/" 6 | $GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py" 7 | $GET_PIP_PATH = "C:\get-pip.py" 8 | 9 | 10 | function DownloadPython ($python_version, $platform_suffix) { 11 | $webclient = New-Object System.Net.WebClient 12 | $filename = "python-" + $python_version + $platform_suffix + ".msi" 13 | $url = $BASE_URL + $python_version + "/" + $filename 14 | 15 | $basedir = $pwd.Path + "\" 16 | $filepath = $basedir + $filename 17 | if (Test-Path $filename) { 18 | Write-Host "Reusing" $filepath 19 | return $filepath 20 | } 21 | 22 | # Download and retry up to 5 times in case of network transient errors. 23 | Write-Host "Downloading" $filename "from" $url 24 | $retry_attempts = 3 25 | for($i=0; $i -lt $retry_attempts; $i++){ 26 | try { 27 | $webclient.DownloadFile($url, $filepath) 28 | break 29 | } 30 | Catch [Exception]{ 31 | Start-Sleep 1 32 | } 33 | } 34 | Write-Host "File saved at" $filepath 35 | return $filepath 36 | } 37 | 38 | 39 | function InstallPython ($python_version, $architecture, $python_home) { 40 | Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home 41 | if (Test-Path $python_home) { 42 | Write-Host $python_home "already exists, skipping." 43 | return $false 44 | } 45 | if ($architecture -eq "32") { 46 | $platform_suffix = "" 47 | } else { 48 | $platform_suffix = ".amd64" 49 | } 50 | $filepath = DownloadPython $python_version $platform_suffix 51 | Write-Host "Installing" $filepath "to" $python_home 52 | $args = "/qn /i $filepath TARGETDIR=$python_home" 53 | Write-Host "msiexec.exe" $args 54 | Start-Process -FilePath "msiexec.exe" -ArgumentList $args -Wait -Passthru 55 | Write-Host "Python $python_version ($architecture) installation complete" 56 | return $true 57 | } 58 | 59 | 60 | function InstallPip ($python_home) { 61 | $pip_path = $python_home + "/Scripts/pip.exe" 62 | $python_path = $python_home + "/python.exe" 63 | if (-not(Test-Path $pip_path)) { 64 | Write-Host "Installing pip..." 65 | $webclient = New-Object System.Net.WebClient 66 | $webclient.DownloadFile($GET_PIP_URL, $GET_PIP_PATH) 67 | Write-Host "Executing:" $python_path $GET_PIP_PATH 68 | Start-Process -FilePath "$python_path" -ArgumentList "$GET_PIP_PATH" -Wait -Passthru 69 | } else { 70 | Write-Host "pip already installed." 71 | } 72 | } 73 | 74 | function InstallPackage ($python_home, $pkg) { 75 | $pip_path = $python_home + "/Scripts/pip.exe" 76 | & $pip_path install $pkg 77 | } 78 | 79 | function main () { 80 | InstallPython $env:PYTHON_VERSION $env:PYTHON_ARCH $env:PYTHON 81 | InstallPip $env:PYTHON 82 | InstallPackage $env:PYTHON wheel 83 | } 84 | 85 | main 86 | -------------------------------------------------------------------------------- /extra/appveyor/run_with_compiler.cmd: -------------------------------------------------------------------------------- 1 | :: To build extensions for 64 bit Python 3, we need to configure environment 2 | :: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of: 3 | :: MS Windows SDK for Windows 7 and .NET Framework 4 (SDK v7.1) 4 | :: 5 | :: To build extensions for 64 bit Python 2, we need to configure environment 6 | :: variables to use the MSVC 2008 C++ compilers from GRMSDKX_EN_DVD.iso of: 7 | :: MS Windows SDK for Windows 7 and .NET Framework 3.5 (SDK v7.0) 8 | :: 9 | :: 32 bit builds do not require specific environment configurations. 10 | :: 11 | :: Note: this script needs to be run with the /E:ON and /V:ON flags for the 12 | :: cmd interpreter, at least for (SDK v7.0) 13 | :: 14 | :: More details at: 15 | :: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows 16 | :: http://stackoverflow.com/a/13751649/163740 17 | :: 18 | :: Author: Olivier Grisel 19 | :: License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ 20 | @ECHO OFF 21 | 22 | SET COMMAND_TO_RUN=%* 23 | SET WIN_SDK_ROOT=C:\Program Files\Microsoft SDKs\Windows 24 | 25 | SET MAJOR_PYTHON_VERSION="%PYTHON_VERSION:~0,1%" 26 | IF %MAJOR_PYTHON_VERSION% == "2" ( 27 | SET WINDOWS_SDK_VERSION="v7.0" 28 | ) ELSE IF %MAJOR_PYTHON_VERSION% == "3" ( 29 | SET WINDOWS_SDK_VERSION="v7.1" 30 | ) ELSE ( 31 | ECHO Unsupported Python version: "%MAJOR_PYTHON_VERSION%" 32 | EXIT 1 33 | ) 34 | 35 | IF "%PYTHON_ARCH%"=="64" ( 36 | ECHO Configuring Windows SDK %WINDOWS_SDK_VERSION% for Python %MAJOR_PYTHON_VERSION% on a 64 bit architecture 37 | SET DISTUTILS_USE_SDK=1 38 | SET MSSdk=1 39 | "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION% 40 | "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x64 /release 41 | ECHO Executing: %COMMAND_TO_RUN% 42 | call %COMMAND_TO_RUN% || EXIT 1 43 | ) ELSE ( 44 | ECHO Using default MSVC build environment for 32 bit architecture 45 | ECHO Executing: %COMMAND_TO_RUN% 46 | call %COMMAND_TO_RUN% || EXIT 1 47 | ) 48 | -------------------------------------------------------------------------------- /requirements/default.txt: -------------------------------------------------------------------------------- 1 | python-vagrant 2 | -------------------------------------------------------------------------------- /requirements/docs.txt: -------------------------------------------------------------------------------- 1 | sphinx_celery>=1.1 2 | celery 3 | -------------------------------------------------------------------------------- /requirements/pkgutils.txt: -------------------------------------------------------------------------------- 1 | setuptools>=20.6.7 2 | wheel>=0.29.0 3 | flake8>=2.5.4 4 | flakeplus>=1.1 5 | tox>=2.3.1 6 | sphinx2rst>=1.0 7 | bumpversion 8 | -------------------------------------------------------------------------------- /requirements/test-ci.txt: -------------------------------------------------------------------------------- 1 | coverage>=3.0 2 | codecov 3 | -------------------------------------------------------------------------------- /requirements/test.txt: -------------------------------------------------------------------------------- 1 | case 2 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [nosetests] 2 | where = cyanide/tests 3 | 4 | [flake8] 5 | # classes can be lowercase, arguments and variables can be uppercase 6 | # whenever it makes the code more readable. 7 | ignore = N806, N802, N801, N803 8 | 9 | [pep257] 10 | ignore = D102,D104,D203,D105,D213 11 | 12 | 13 | [wheel] 14 | universal = 1 15 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from setuptools import setup, find_packages 5 | 6 | import os 7 | import re 8 | import sys 9 | import codecs 10 | 11 | try: 12 | import platform 13 | _pyimp = platform.python_implementation 14 | except (AttributeError, ImportError): 15 | def _pyimp(): 16 | return 'Python' 17 | 18 | NAME = 'cyanide' 19 | 20 | E_UNSUPPORTED_PYTHON = '%s 1.0 requires %%s %%s or later!' % (NAME,) 21 | 22 | PYIMP = _pyimp() 23 | PY26_OR_LESS = sys.version_info < (2, 7) 24 | PY3 = sys.version_info[0] == 3 25 | PY33_OR_LESS = PY3 and sys.version_info < (3, 4) 26 | PYPY_VERSION = getattr(sys, 'pypy_version_info', None) 27 | PYPY = PYPY_VERSION is not None 28 | PYPY24_ATLEAST = PYPY_VERSION and PYPY_VERSION >= (2, 4) 29 | 30 | if PY26_OR_LESS: 31 | raise Exception(E_UNSUPPORTED_PYTHON % (PYIMP, '2.7')) 32 | elif PY33_OR_LESS and not PYPY24_ATLEAST: 33 | raise Exception(E_UNSUPPORTED_PYTHON % (PYIMP, '3.4')) 34 | 35 | # -*- Classifiers -*- 36 | 37 | classes = """ 38 | Development Status :: 2 - Pre-Alpha 39 | License :: OSI Approved :: BSD License 40 | Programming Language :: Python 41 | Programming Language :: Python :: 2 42 | Programming Language :: Python :: 2.7 43 | Programming Language :: Python :: 3 44 | Programming Language :: Python :: 3.4 45 | Programming Language :: Python :: 3.5 46 | Programming Language :: Python :: Implementation :: CPython 47 | Programming Language :: Python :: Implementation :: PyPy 48 | Programming Language :: Python :: Implementation :: Jython 49 | Operating System :: OS Independent 50 | """ 51 | classifiers = [s.strip() for s in classes.split('\n') if s] 52 | 53 | # -*- Distribution Meta -*- 54 | 55 | re_meta = re.compile(r'__(\w+?)__\s*=\s*(.*)') 56 | re_doc = re.compile(r'^"""(.+?)"""') 57 | 58 | 59 | def add_default(m): 60 | attr_name, attr_value = m.groups() 61 | return ((attr_name, attr_value.strip("\"'")),) 62 | 63 | 64 | def add_doc(m): 65 | return (('doc', m.groups()[0]),) 66 | 67 | pats = {re_meta: add_default, re_doc: add_doc} 68 | here = os.path.abspath(os.path.dirname(__file__)) 69 | with open(os.path.join(here, NAME, '__init__.py')) as fh: 70 | meta = {} 71 | for line in fh: 72 | if line.strip() == '# -eof meta-': 73 | break 74 | for pattern, handler in pats.items(): 75 | m = pattern.match(line.strip()) 76 | if m: 77 | meta.update(handler(m)) 78 | 79 | # -*- Installation Requires -*- 80 | 81 | 82 | def strip_comments(l): 83 | return l.split('#', 1)[0].strip() 84 | 85 | 86 | def _pip_requirement(req): 87 | if req.startswith('-r '): 88 | _, path = req.split() 89 | return reqs(*path.split('/')) 90 | return [req] 91 | 92 | 93 | def _reqs(*f): 94 | return [ 95 | _pip_requirement(r) for r in ( 96 | strip_comments(l) for l in open( 97 | os.path.join(os.getcwd(), 'requirements', *f)).readlines() 98 | ) if r] 99 | 100 | 101 | def reqs(*f): 102 | return [req for subreq in _reqs(*f) for req in subreq] 103 | 104 | # -*- Long Description -*- 105 | 106 | if os.path.exists('README.rst'): 107 | long_description = codecs.open('README.rst', 'r', 'utf-8').read() 108 | else: 109 | long_description = 'See http://pypi.python.org/pypi/%s' % (NAME,) 110 | 111 | # -*- %%% -*- 112 | 113 | setup( 114 | name=NAME, 115 | version=meta['version'], 116 | description=meta['doc'], 117 | keywords='celery integration test suite', 118 | author=meta['author'], 119 | author_email=meta['contact'], 120 | url=meta['homepage'], 121 | platforms=['any'], 122 | license='BSD', 123 | packages=find_packages(exclude=['ez_setup', 'tests', 'tests.*']), 124 | include_package_data=True, 125 | zip_safe=False, 126 | install_requires=reqs('default.txt'), 127 | tests_require=reqs('test.txt'), 128 | test_suite='nose.collector', 129 | classifiers=classifiers, 130 | long_description=long_description, 131 | entry_points={ 132 | 'celery.commands': [ 133 | 'cyanide = cyanide.bin.cyanide:cyanide', 134 | 'vagrant = cyanide.bin.vagrant:vagrant', 135 | ], 136 | }, 137 | ) 138 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = 2.7,pypy,3.4,3.5,pypy3,flake8,flakeplus,apicheck,configcheck 3 | 4 | [testenv] 5 | deps= 6 | -r{toxinidir}/requirements/default.txt 7 | -r{toxinidir}/requirements/test.txt 8 | -r{toxinidir}/requirements/test-ci.txt 9 | 10 | linkcheck,apicheck: -r{toxinidir}/requirements/docs.txt 11 | flake8,flakeplus: -r{toxinidir}/requirements/pkgutils.txt 12 | sitepackages = False 13 | recreate = False 14 | commands = nosetests -xsv --with-coverage \ 15 | --cover-inclusive --cover-min-percentage=90 --cover-erase [] 16 | 17 | basepython = 18 | 2.7,flake8,flakeplus,apicheck,linkcheck,configcheck: python2.7 19 | 3.4: python3.4 20 | 3.5: python3.5 21 | pypy: pypy 22 | pypy3: pypy3 23 | 24 | [testenv:apicheck] 25 | commands = 26 | pip install -U -r{toxinidir}/requirements/dev.txt 27 | sphinx-build -W -b apicheck -d {envtmpdir}/doctrees docs docs/_build/apicheck 28 | 29 | [testenv:configcheck] 30 | commands = 31 | pip install -U -r{toxinidir}/requirements/dev.txt 32 | sphinx-build -W -b configcheck -d {envtmpdir}/doctrees docs docs/_build/configcheck 33 | 34 | [testenv:linkcheck] 35 | commands = 36 | pip install -U -r{toxinidir}/requirements/dev.txt 37 | sphinx-build -W -b linkcheck -d {envtmpdir}/doctrees docs docs/_build/linkcheck 38 | 39 | [testenv:flake8] 40 | commands = 41 | flake8 --ignore=X999 {toxinidir}/cyanide 42 | 43 | [testenv:flakeplus] 44 | commands = 45 | flakeplus --2.7 {toxinidir}/cyanide 46 | --------------------------------------------------------------------------------