├── .gitignore ├── .hgignore ├── AUTHORS ├── Changelog ├── LICENSE ├── MANIFEST.in ├── README ├── README.rst ├── TODO ├── contrib ├── possible_names.rst └── release │ ├── bump_version.py │ ├── doc4allmods │ ├── flakeplus.py │ ├── prepy3ktest │ ├── py3k-run-tests │ ├── removepyc.sh │ └── verify-reference-index.sh ├── cyme ├── __init__.py ├── admin.py ├── api │ ├── __init__.py │ ├── urls.py │ ├── views.py │ └── web.py ├── bin │ ├── __init__.py │ ├── base.py │ ├── cyme.py │ ├── cyme_branch.py │ └── cyme_list_branches.py ├── branch │ ├── __init__.py │ ├── controller.py │ ├── httpd.py │ ├── intsup.py │ ├── managers.py │ ├── metrics.py │ ├── signals.py │ ├── state.py │ ├── supervisor.py │ └── thread.py ├── client │ ├── __init__.py │ └── base.py ├── conf.py ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ ├── base.py │ │ ├── cyme.py │ │ └── cyme_branch.py ├── models │ ├── __init__.py │ └── managers.py ├── settings.py ├── status.py ├── tasks.py ├── tests │ ├── __init__.py │ ├── test_branch.py │ └── test_models.py └── utils │ ├── __init__.py │ ├── actors.py │ └── dictshield.py ├── docs ├── .static │ └── .keep ├── .templates │ ├── page.html │ ├── sidebarintro.html │ └── sidebarlogo.html ├── Makefile ├── _ext │ ├── applyxrefs.py │ ├── celerydocs.py │ └── literals_to_xrefs.py ├── _theme │ └── celery │ │ ├── static │ │ └── celery.css_t │ │ └── theme.conf ├── changelog.rst ├── conf.py ├── images │ ├── celery-icon-128.png │ ├── celery-icon-32.png │ ├── celery-icon-64.png │ ├── celery_favicon_128.png │ ├── favicon.ico │ └── favicon.png ├── index.rst ├── introduction.rst └── reference │ ├── cyme.api.views.rst │ ├── cyme.api.web.rst │ ├── cyme.bin.base.rst │ ├── cyme.bin.cyme.rst │ ├── cyme.bin.cyme_branch.rst │ ├── cyme.branch.controller.rst │ ├── cyme.branch.httpd.rst │ ├── cyme.branch.intsup.rst │ ├── cyme.branch.managers.rst │ ├── cyme.branch.metrics.rst │ ├── cyme.branch.rst │ ├── cyme.branch.signals.rst │ ├── cyme.branch.state.rst │ ├── cyme.branch.supervisor.rst │ ├── cyme.branch.thread.rst │ ├── cyme.client.base.rst │ ├── cyme.client.rst │ ├── cyme.management.commands.cyme.rst │ ├── cyme.management.commands.cyme_branch.rst │ ├── cyme.models.managers.rst │ ├── cyme.models.rst │ ├── cyme.status.rst │ ├── cyme.tasks.rst │ ├── cyme.utils.rst │ └── index.rst ├── funtests ├── setup.cfg ├── setup.py └── suite │ ├── __init__.py │ ├── test_basic.py │ └── utils.py ├── pavement.py ├── requirements ├── default.txt ├── docs.txt └── test.txt ├── setup.cfg ├── setup.py └── tests ├── __init__.py ├── manage.py └── settings.py /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.pyc 3 | *$py.class 4 | *~ 5 | .*.sw[pon] 6 | dist/ 7 | *.egg-info 8 | *.egg 9 | *.egg/ 10 | doc/__build/* 11 | build/ 12 | .build/ 13 | pip-log.txt 14 | .directory 15 | erl_crash.dump 16 | *.db 17 | *.sqlite 18 | Documentation/ 19 | .tox/ 20 | .ropeproject/ 21 | .project 22 | .pydevproject 23 | instances/ 24 | dump.rdb 25 | 26 | -------------------------------------------------------------------------------- /.hgignore: -------------------------------------------------------------------------------- 1 | syntax: glob 2 | .DS_Store 3 | *.pyc 4 | *~ 5 | .*.sw[pon] 6 | dist/ 7 | *.egg-info 8 | doc/__build/* 9 | build/ 10 | .build/ 11 | pip-log.txt 12 | .directory 13 | erl_crash.dump 14 | *.db 15 | Documentation/ 16 | *.sqlite 17 | instances/ 18 | dump.rdb 19 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Ordered by date of first contribution: 2 | Ask Solem 3 | -------------------------------------------------------------------------------- /Changelog: -------------------------------------------------------------------------------- 1 | ================ 2 | Change history 3 | ================ 4 | 5 | .. contents:: 6 | :local: 7 | 8 | .. _version-0.0.1: 9 | 10 | 0.0.1 11 | ===== 12 | :status: in development 13 | 14 | * Initial commit. 15 | 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011-2013, GoPivotal, Inc. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, 8 | this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | 13 | Neither the name of GoPivotal, Inc. nor the names of its contributors may be used 14 | to endorse or promote products derived from this software without specific 15 | prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 19 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 | PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS 21 | BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | POSSIBILITY OF SUCH DAMAGE. 28 | 29 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include AUTHORS 2 | include Changelog 3 | include LICENSE 4 | include MANIFEST.in 5 | include README 6 | include README.rst 7 | include TODO 8 | include setup.cfg 9 | recursive-include cyme *.py 10 | recursive-include docs * 11 | recursive-include tests * 12 | recursive-include funtests *.py 13 | recursive-include requirements *.txt 14 | include funtests/setup.py 15 | include funtests/setup.cfg 16 | prune *.db 17 | prune tests/*.pyc 18 | prune cyme/*.pyc 19 | prune docs/.build 20 | prune funtests/suite/instances/* 21 | prune docs/.build 22 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | README.rst -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | =============================================== 2 | Cyme - Celery instance manager 3 | =============================================== 4 | 5 | :Version: 0.0.5 6 | :Web: http://celeryproject.org/ 7 | :Download: http://pypi.python.org/pypi/cyme/ 8 | :Keywords: celery, task queue, job queue, cloud, service 9 | 10 | -- 11 | 12 | Installation 13 | ============= 14 | 15 | You can install ``cyme`` either via the Python Package Index (PyPI) 16 | or from source. 17 | 18 | To install using ``pip``,:: 19 | 20 | $ pip install cyme 21 | 22 | To install using ``easy_install``,:: 23 | 24 | $ easy_install cyme 25 | 26 | Getting Help 27 | ============ 28 | 29 | Mailing list 30 | ------------ 31 | 32 | For discussions about the usage, development, and future of celery, 33 | please join the `celery-users`_ mailing list. 34 | 35 | .. _`celery-users`: http://groups.google.com/group/celery-users/ 36 | 37 | IRC 38 | --- 39 | 40 | Come chat with us on IRC. The `#celery`_ channel is located at the `Freenode`_ 41 | network. 42 | 43 | .. _`#celery`: irc://irc.freenode.net/celery 44 | .. _`Freenode`: http://freenode.net 45 | 46 | 47 | License 48 | ======= 49 | 50 | This software is licensed under the ``New BSD License``. See the ``LICENSE`` 51 | file in the top distribution directory for the full license text. 52 | 53 | .. # vim: syntax=rst expandtab tabstop=4 shiftwidth=4 shiftround 54 | 55 | -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | * When deleting an app, this should also delete all resources owned by an app. 2 | -------------------------------------------------------------------------------- /contrib/possible_names.rst: -------------------------------------------------------------------------------- 1 | NAMES 2 | ===== 3 | 4 | * umbel 5 | * cardoon 6 | * chervil 7 | * cicely 8 | * chard 9 | * cress 10 | * kale 11 | * leek 12 | * onion 13 | * ginger 14 | * skirret 15 | * yam 16 | 17 | umbel, cardoon, chervil, cicely, chard, cress, kale, leek, onion, ginger, 18 | skirret, yam 19 | -------------------------------------------------------------------------------- /contrib/release/bump_version.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from __future__ import absolute_import 4 | from __future__ import with_statement 5 | 6 | import errno 7 | import os 8 | import re 9 | import sys 10 | import subprocess 11 | 12 | from contextlib import contextmanager 13 | from tempfile import NamedTemporaryFile 14 | 15 | rq = lambda s: s.strip("\"'") 16 | 17 | 18 | def cmd(*args): 19 | return subprocess.Popen(args, stdout=subprocess.PIPE).communicate()[0] 20 | 21 | 22 | @contextmanager 23 | def no_enoent(): 24 | try: 25 | yield 26 | except OSError, exc: 27 | if exc.errno != errno.ENOENT: 28 | raise 29 | 30 | 31 | class StringVersion(object): 32 | 33 | def decode(self, s): 34 | s = rq(s) 35 | text = "" 36 | major, minor, release = s.split(".") 37 | if not release.isdigit(): 38 | pos = release.index(re.split("\d+", release)[1][0]) 39 | release, text = release[:pos], release[pos:] 40 | return int(major), int(minor), int(release), text 41 | 42 | def encode(self, v): 43 | return ".".join(map(str, v[:3])) + v[3] 44 | to_str = StringVersion().encode 45 | from_str = StringVersion().decode 46 | 47 | 48 | class TupleVersion(object): 49 | 50 | def decode(self, s): 51 | v = list(map(rq, s.split(", "))) 52 | return (tuple(map(int, v[0:3])) + 53 | tuple(["".join(v[3:])])) 54 | 55 | def encode(self, v): 56 | v = list(v) 57 | 58 | def quote(lit): 59 | if isinstance(lit, basestring): 60 | return '"%s"' % (lit, ) 61 | return str(lit) 62 | 63 | if not v[-1]: 64 | v.pop() 65 | return ", ".join(map(quote, v)) 66 | 67 | 68 | class VersionFile(object): 69 | 70 | def __init__(self, filename): 71 | self.filename = filename 72 | self._kept = None 73 | 74 | def _as_orig(self, version): 75 | return self.wb % {"version": self.type.encode(version), 76 | "kept": self._kept} 77 | 78 | def write(self, version): 79 | pattern = self.regex 80 | with no_enoent(): 81 | with NamedTemporaryFile() as dest: 82 | with open(self.filename) as orig: 83 | for line in orig: 84 | if pattern.match(line): 85 | dest.write(self._as_orig(version)) 86 | else: 87 | dest.write(line) 88 | os.rename(dest.name, self.filename) 89 | 90 | def parse(self): 91 | pattern = self.regex 92 | gpos = 0 93 | with open(self.filename) as fh: 94 | for line in fh: 95 | m = pattern.match(line) 96 | if m: 97 | if "?P" in pattern.pattern: 98 | self._kept, gpos = m.groupdict()["keep"], 1 99 | return self.type.decode(m.groups()[gpos]) 100 | 101 | 102 | class PyVersion(VersionFile): 103 | regex = re.compile(r'^VERSION\s*=\s*\((.+?)\)') 104 | wb = "VERSION = (%(version)s)\n" 105 | type = TupleVersion() 106 | 107 | 108 | class SphinxVersion(VersionFile): 109 | regex = re.compile(r'^:[Vv]ersion:\s*(.+?)$') 110 | wb = ':Version: %(version)s\n' 111 | type = StringVersion() 112 | 113 | 114 | class CPPVersion(VersionFile): 115 | regex = re.compile(r'^\#\s*define\s*(?P\w*)VERSION\s+(.+)') 116 | wb = '#define %(kept)sVERSION "%(version)s"\n' 117 | type = StringVersion() 118 | 119 | 120 | _filetype_to_type = {"py": PyVersion, 121 | "rst": SphinxVersion, 122 | "c": CPPVersion, 123 | "h": CPPVersion} 124 | 125 | def filetype_to_type(filename): 126 | _, _, suffix = filename.rpartition(".") 127 | return _filetype_to_type[suffix](filename) 128 | 129 | 130 | def bump(*files, **kwargs): 131 | version = kwargs.get("version") 132 | files = [filetype_to_type(f) for f in files] 133 | versions = [v.parse() for v in files] 134 | current = list(reversed(sorted(versions)))[0] # find highest 135 | 136 | if version: 137 | next = from_str(version) 138 | else: 139 | major, minor, release, text = current 140 | if text: 141 | raise Exception("Can't bump alpha releases") 142 | next = (major, minor, release + 1, text) 143 | 144 | print("Bump version from %s -> %s" % (to_str(current), to_str(next))) 145 | 146 | for v in files: 147 | print(" writing %r..." % (v.filename, )) 148 | v.write(next) 149 | 150 | print(cmd("git", "commit", "-m", "Bumps version to %s" % (to_str(next), ), 151 | *[f.filename for f in files])) 152 | print(cmd("git", "tag", "v%s" % (to_str(next), ))) 153 | 154 | 155 | def main(argv=sys.argv, version=None): 156 | if not len(argv) > 1: 157 | print("Usage: distdir [docfile] -- ") 158 | sys.exit(0) 159 | if "--" in argv: 160 | c = argv.index('--') 161 | version = argv[c + 1] 162 | argv = argv[:c] 163 | bump(*argv[1:], version=version) 164 | 165 | if __name__ == "__main__": 166 | main() 167 | -------------------------------------------------------------------------------- /contrib/release/doc4allmods: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | PACKAGE="$1" 4 | SKIP_PACKAGES="$PACKAGE tests management urls" 5 | SKIP_FILES="cyme.admin.rst 6 | cyme.bin.rst 7 | cyme.api.rst 8 | cyme.api.urls.rst 9 | cyme.settings.rst 10 | cyme.conf.rst" 11 | 12 | modules=$(find "$PACKAGE" -name "*.py") 13 | 14 | failed=0 15 | for module in $modules; do 16 | dotted=$(echo $module | sed 's/\//\./g') 17 | name=${dotted%.__init__.py} 18 | name=${name%.py} 19 | rst=$name.rst 20 | skip=0 21 | for skip_package in $SKIP_PACKAGES; do 22 | [ $(echo "$name" | cut -d. -f 2) == "$skip_package" ] && skip=1 23 | done 24 | for skip_file in $SKIP_FILES; do 25 | [ "$skip_file" == "$rst" ] && skip=1 26 | done 27 | 28 | if [ $skip -eq 0 ]; then 29 | if [ ! -f "docs/reference/$rst" ]; then 30 | echo $rst :: FAIL 31 | failed=1 32 | fi 33 | fi 34 | done 35 | 36 | exit $failed 37 | -------------------------------------------------------------------------------- /contrib/release/flakeplus.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import absolute_import 3 | from __future__ import with_statement 4 | 5 | import os 6 | import re 7 | import sys 8 | 9 | from collections import defaultdict 10 | from unipath import Path 11 | 12 | RE_COMMENT = r'^\s*\#' 13 | RE_NOQA = r'.+?\#\s+noqa+' 14 | RE_MULTILINE_COMMENT_O = r'^\s*(?:\'\'\'|""").+?(?:\'\'\'|""")' 15 | RE_MULTILINE_COMMENT_S = r'^\s*(?:\'\'\'|""")' 16 | RE_MULTILINE_COMMENT_E = r'(?:^|.+?)(?:\'\'\'|""")' 17 | RE_WITH = r'(?:^|\s+)with\s+' 18 | RE_WITH_IMPORT = r'''from\s+ __future__\s+ import\s+ with_statement''' 19 | RE_PRINT = r'''(?:^|\s+)print\((?:"|')(?:\W+?)?[A-Z0-9:]{2,}''' 20 | RE_ABS_IMPORT = r'''from\s+ __future__\s+ import\s+ absolute_import''' 21 | 22 | acc = defaultdict(lambda: {"abs": False, "print": False}) 23 | 24 | 25 | def compile(regex): 26 | return re.compile(regex, re.VERBOSE) 27 | 28 | 29 | class FlakePP(object): 30 | re_comment = compile(RE_COMMENT) 31 | re_ml_comment_o = compile(RE_MULTILINE_COMMENT_O) 32 | re_ml_comment_s = compile(RE_MULTILINE_COMMENT_S) 33 | re_ml_comment_e = compile(RE_MULTILINE_COMMENT_E) 34 | re_abs_import = compile(RE_ABS_IMPORT) 35 | re_print = compile(RE_PRINT) 36 | re_with_import = compile(RE_WITH_IMPORT) 37 | re_with = compile(RE_WITH) 38 | re_noqa = compile(RE_NOQA) 39 | map = {"abs": False, "print": False, 40 | "with": False, "with-used": False} 41 | 42 | def __init__(self, verbose=False): 43 | self.verbose = verbose 44 | self.steps = (("abs", self.re_abs_import), 45 | ("with", self.re_with_import), 46 | ("with-used", self.re_with), 47 | ("print", self.re_print)) 48 | 49 | def analyze_fh(self, fh): 50 | steps = self.steps 51 | filename = fh.name 52 | acc = dict(self.map) 53 | index = 0 54 | errors = [0] 55 | 56 | def error(fmt, **kwargs): 57 | errors[0] += 1 58 | self.announce(fmt, **dict(kwargs, filename=filename)) 59 | 60 | for index, line in enumerate(self.strip_comments(fh)): 61 | for key, pattern in steps: 62 | if pattern.match(line): 63 | acc[key] = True 64 | if index: 65 | if not acc["abs"]: 66 | error("%(filename)s: missing abs import") 67 | if acc["with-used"] and not acc["with"]: 68 | error("%(filename)s: missing with import") 69 | if acc["print"]: 70 | error("%(filename)s: left over print statement") 71 | 72 | return filename, errors[0], acc 73 | 74 | def analyze_file(self, filename): 75 | with open(filename) as fh: 76 | return self.analyze_fh(fh) 77 | 78 | def analyze_tree(self, dir): 79 | for dirpath, _, filenames in os.walk(dir): 80 | for path in (Path(dirpath, f) for f in filenames): 81 | if path.endswith(".py"): 82 | yield self.analyze_file(path) 83 | 84 | def analyze(self, *paths): 85 | for path in map(Path, paths): 86 | if path.isdir(): 87 | for res in self.analyze_tree(path): 88 | yield res 89 | else: 90 | yield self.analyze_file(path) 91 | 92 | def strip_comments(self, fh): 93 | re_comment = self.re_comment 94 | re_ml_comment_o = self.re_ml_comment_o 95 | re_ml_comment_s = self.re_ml_comment_s 96 | re_ml_comment_e = self.re_ml_comment_e 97 | re_noqa = self.re_noqa 98 | in_ml = False 99 | 100 | for line in fh.readlines(): 101 | if in_ml: 102 | if re_ml_comment_e.match(line): 103 | in_ml = False 104 | else: 105 | if re_noqa.match(line) or re_ml_comment_o.match(line): 106 | pass 107 | elif re_ml_comment_s.match(line): 108 | in_ml = True 109 | elif re_comment.match(line): 110 | pass 111 | else: 112 | yield line 113 | 114 | def announce(self, fmt, **kwargs): 115 | sys.stderr.write((fmt + "\n") % kwargs) 116 | 117 | 118 | def main(argv=sys.argv, exitcode=0): 119 | for _, errors, _ in FlakePP(verbose=True).analyze(*argv[1:]): 120 | if errors: 121 | exitcode = 1 122 | return exitcode 123 | 124 | 125 | if __name__ == "__main__": 126 | sys.exit(main()) 127 | -------------------------------------------------------------------------------- /contrib/release/prepy3ktest: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | (cd "${1:-.}/.tox/py32/src/kombu"; 3 | 2to3 --no-diff -w --nobackups kombu) 4 | -------------------------------------------------------------------------------- /contrib/release/py3k-run-tests: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | base=${1:-.} 3 | nosetests -vd cyme.tests \ 4 | --with-coverage3 \ 5 | --cover3-branch \ 6 | --cover3-xml \ 7 | --cover3-xml-file="$base/coverage.xml" \ 8 | --cover3-html \ 9 | --cover3-html-dir="$base/cover" \ 10 | --cover3-package=cyme \ 11 | --cover3-exclude=" \ 12 | cyme \ 13 | cyme.tests.* \ 14 | --with-xunit \ 15 | --xunit-file="$base/nosetests.xml" 16 | -------------------------------------------------------------------------------- /contrib/release/removepyc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | (cd "${1:-.}"; 3 | find . -name "*.pyc" | xargs rm -- 2>/dev/null) || echo "ok" 4 | -------------------------------------------------------------------------------- /contrib/release/verify-reference-index.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | verify_index() { 4 | modules=$(grep "celery." "$1" | \ 5 | perl -ple's/^\s*|\s*$//g;s{\.}{/}g;') 6 | retval=0 7 | for module in $modules; do 8 | if [ ! -f "$module.py" ]; then 9 | if [ ! -f "$module/__init__.py" ]; then 10 | echo "Outdated reference: $module" 11 | retval=1 12 | fi 13 | fi 14 | done 15 | 16 | return $retval 17 | } 18 | 19 | verify_index docs/reference/index.rst && \ 20 | verify_index docs/internals/reference/index.rst 21 | 22 | -------------------------------------------------------------------------------- /cyme/__init__.py: -------------------------------------------------------------------------------- 1 | """Cyme - Celery instance manager""" 2 | from __future__ import absolute_import 3 | 4 | import os 5 | 6 | VERSION = (0, 0, 5) 7 | 8 | __version__ = '.'.join(map(str, VERSION[0:3])) + ''.join(VERSION[3:]) 9 | __author__ = 'Ask Solem' 10 | __contact__ = 'ask@celeryproject.org' 11 | __homepage__ = 'http://celeryproject.org' 12 | __docformat__ = 'restructuredtext' 13 | __license__ = 'BSD (3 clause)' 14 | 15 | DEBUG = os.environ.get('CYME_DEBUG') 16 | DEBUG_BLOCK = os.environ.get('CYME_DEBUG_BLOCK') 17 | DEBUG_READERS = os.environ.get('CYME_DEBUG_READERS') 18 | if not os.environ.get('CYME_NO_EVAL'): 19 | from .client import Client # noqa 20 | -------------------------------------------------------------------------------- /cyme/admin.py: -------------------------------------------------------------------------------- 1 | """cyme.admin 2 | 3 | - Django Admin interface extensions for Cyme. 4 | 5 | """ 6 | from __future__ import absolute_import 7 | 8 | from django.contrib import admin 9 | from django.utils.html import escape 10 | from django.utils.translation import ugettext_lazy as _ 11 | 12 | from djcelery.admin_utils import action, display_field, fixedwidth 13 | from djcelery.humanize import naturaldate 14 | 15 | from .models import Broker, Instance, Queue 16 | from .branch.supervisor import supervisor 17 | 18 | 19 | @display_field(_('max/min concurrency'), 'max_concurrency') 20 | def maxmin_concurrency(instance): 21 | return '%s / %s' % (instance.max_concurrency, instance.min_concurrency) 22 | 23 | 24 | @display_field(_('created'), 'created_at') 25 | def created_at(instance): 26 | return """
%s
""" % ( 27 | escape(str(instance.created_at)), 28 | escape(naturaldate(instance.created_at))) 29 | 30 | 31 | @display_field(_('status'), 'is_enabled') 32 | def status(instance): 33 | enabled = 'Enabled' if instance.is_enabled else 'Disabled' 34 | if instance.alive(): 35 | state, color = 'ONLINE', 'green' 36 | else: 37 | state, color = 'OFFLINE', 'red' 38 | return """%s (%s)""" % ( 39 | enabled, color, state) 40 | 41 | 42 | class InstanceAdmin(admin.ModelAdmin): 43 | detail_title = _('Instance detail') 44 | list_page_title = _('Instances') 45 | date_hierarchy = 'created_at' 46 | fieldsets = ( 47 | (None, { 48 | 'fields': ('name', 'max_concurrency', 'min_concurrency', 49 | '_queues', 'is_enabled', '_broker'), 50 | 'classes': ('extrapretty', ), 51 | }), ) 52 | list_display = (fixedwidth('name', pt=10), maxmin_concurrency, 53 | status, 'broker') 54 | read_only_fields = ('created_at', ) 55 | list_filter = ('name', 'max_concurrency', 'min_concurrency', '_queues') 56 | search_fields = ('name', 'max_concurrency', 'min_concurrency', '_queues') 57 | actions = ['disable_instances', 58 | 'enable_instances', 59 | 'restart_instances'] 60 | 61 | @action(_('Disable selected instances')) 62 | def disable_instances(self, request, queryset): 63 | for instance in queryset: 64 | instance.disable() 65 | supervisor.verify(queryset).wait() 66 | 67 | @action(_('Enable selected instances')) 68 | def enable_instances(self, request, queryset): 69 | for instance in queryset: 70 | instance.enable() 71 | supervisor.verify(queryset).wait() 72 | 73 | @action(_('Restart selected instances')) 74 | def restart_instances(self, request, queryset): 75 | supervisor.restart(queryset).wait() 76 | 77 | 78 | admin.site.register(Broker) 79 | admin.site.register(Instance, InstanceAdmin) 80 | admin.site.register(Queue) 81 | -------------------------------------------------------------------------------- /cyme/api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/celery/cyme/c49047318c56f6674bd6d6ea6673b48be4eb7886/cyme/api/__init__.py -------------------------------------------------------------------------------- /cyme/api/urls.py: -------------------------------------------------------------------------------- 1 | """cyme.api.urls""" 2 | 3 | from __future__ import absolute_import 4 | 5 | from django.contrib import admin 6 | from django.conf.urls.defaults import (patterns, include, url, # noqa 7 | handler500, handler404) 8 | 9 | from . import views 10 | 11 | uApp = r'(?P[^/]+)' 12 | uNowait = r'(?P!/)?' 13 | 14 | admin.autodiscover() 15 | 16 | 17 | def _o_(u): 18 | return (u.replace('APP', uApp) 19 | .replace('!', uNowait)) 20 | 21 | urlpatterns = patterns('', 22 | (r'^ping/$', views.ping.as_view()), 23 | (r'^admin/doc/', include('django.contrib.admindocs.urls')), 24 | 25 | (r'^admin/', include(admin.site.urls)), 26 | (r'^branches/(?P.+?)?/?$', views.Branch.as_view()), 27 | (_o_(r'^APP/queue/!(?P.+)'), views.apply.as_view()), 28 | (_o_(r'^APP/queues/!/?$'), views.Queue.as_view()), 29 | (_o_(r'^APP/queues/!(?P.+?)/?$'), views.Queue.as_view()), 30 | (_o_(r'^APP/instances/!(?P.+?)/queues/(?P.+?)?/?$'), 31 | views.Consumer.as_view()), 32 | (_o_(r'^APP/instances/!?(?P.+)?/autoscale/?'), 33 | views.autoscale.as_view()), 34 | (_o_(r'^APP/instances/!(?P.+)?/stats/?'), 35 | views.instance_stats.as_view()), 36 | (_o_(r'^APP/instances/!(?P.+?)?/?$'), views.Instance.as_view()), 37 | (_o_(r'^APP/query/(?P.+?)/state/?'), views.task_state.as_view()), 38 | (_o_(r'^APP/query/(?P.+?)/result/?'), views.task_result.as_view()), 39 | (_o_(r'^APP/query/(?P.+?)/wait/?'), views.task_wait.as_view()), 40 | (_o_(r'^APP?/?$'), views.App.as_view()), 41 | ) 42 | -------------------------------------------------------------------------------- /cyme/api/views.py: -------------------------------------------------------------------------------- 1 | """cyme.api.views""" 2 | 3 | from __future__ import absolute_import 4 | from __future__ import with_statement 5 | 6 | import re 7 | 8 | from celery import current_app as celery 9 | from celery.result import AsyncResult 10 | 11 | from . import web 12 | from cyme.branch.controller import apps, branches, instances, queues 13 | from cyme.tasks import webhook 14 | from cyme.utils import uuid 15 | 16 | 17 | class Branch(web.ApiView): 18 | 19 | def get(self, request, branch=None): 20 | return branches.get(branch) if branch else branches.all() 21 | 22 | 23 | class App(web.ApiView): 24 | 25 | def get(self, request, app=None): 26 | return apps.get(app).as_dict() if app else apps.all() 27 | 28 | def put(self, request, app=None): 29 | return self.Created(apps.add(app or uuid(), 30 | **self.params('broker', 'arguments', 31 | 'extra_config'))) 32 | post = put 33 | 34 | def delete(self, request, app): 35 | return apps.delete(app) 36 | 37 | 38 | class Instance(web.ApiView): 39 | 40 | def get(self, request, app, name=None, nowait=False): 41 | return instances.get(name) if name else instances.all(app=app) 42 | 43 | def delete(self, request, app, name, nowait=False): 44 | return self.Ok(instances.remove(name, nowait=nowait)) 45 | 46 | def post(self, request, app, name=None, nowait=False): 47 | return self.Created(instances.add(name=name, app=app, 48 | nowait=nowait, 49 | **self.params('broker', 'pool', 50 | 'arguments', 51 | 'extra_config'))) 52 | 53 | def put(self, *args, **kwargs): 54 | return self.NotImplemented('Operation is not idempotent: use POST') 55 | 56 | 57 | class Consumer(web.ApiView): 58 | 59 | def get(self, request, app, name, queue=None, nowait=False): 60 | return instances.consuming_from(name) 61 | 62 | def put(self, request, app, name, queue, nowait=False): 63 | return self.Created(instances.add_consumer(name, queue, nowait=nowait)) 64 | post = put 65 | 66 | def delete(self, request, app, name, queue, nowait=False): 67 | return self.Ok(instances.cancel_consumer(name, queue, nowait=nowait)) 68 | 69 | 70 | class Queue(web.ApiView): 71 | 72 | def get(self, request, app, name=None): 73 | return queues.get(name) if name else queues.all() 74 | 75 | def delete(self, request, app, name, nowait=False): 76 | return self.Ok(queues.delete(name)) 77 | 78 | def put(self, request, app, name, nowait=False): 79 | return self.Created(queues.add(name, nowait=nowait, 80 | **self.params('exchange', 'exchange_type', 81 | 'routing_key', 'options'))) 82 | post = put 83 | 84 | 85 | class apply(web.ApiView): 86 | get_methods = frozenset(['GET', 'HEAD']) 87 | re_find_queue = re.compile(r'/?(.+?)/?$') 88 | re_url_in_path = re.compile(r'(.+?/)(\w+://)(.+)') 89 | 90 | def prepare_path(self, rest): 91 | path, url = self._parse_path_containing_url(rest) 92 | if path: 93 | m = self.re_find_queue.match(path) 94 | if m: 95 | return m.groups()[0], url 96 | return None, url 97 | 98 | def dispatch(self, request, app, rest): 99 | gd = lambda m: getattr(request, m) 100 | queue, url = self.prepare_path(rest) 101 | app = apps.get(app) 102 | broker = app.get_broker() 103 | method = request.method.upper() 104 | pargs = {} 105 | if queue: 106 | queue = queues.get(queue) 107 | pargs.update(exchange=queue['exchange'], 108 | exchange_type=queue['exchange_type'], 109 | routing_key=queue['routing_key']) 110 | params = gd(method) if method in self.get_methods else gd('GET') 111 | data = gd(method) if method not in self.get_methods else None 112 | 113 | with broker.producers.acquire(block=True) as producer: 114 | publisher = celery.amqp.TaskPublisher( 115 | connection=producer.connection, 116 | channel=producer.channel) 117 | result = webhook.apply_async((url, method, params, data), 118 | publisher=publisher, retry=True, 119 | **pargs) 120 | return self.Accepted({'uuid': result.task_id, 'url': url, 121 | 'queue': queue, 'method': method, 122 | 'params': params, 'data': data, 123 | 'broker': producer.connection.as_uri()}) 124 | 125 | def _parse_path_containing_url(self, rest): 126 | m = self.re_url_in_path.match(rest) 127 | if m: 128 | first, scheme, last = m.groups() 129 | if scheme: 130 | return first, scheme + last 131 | return first, None 132 | return rest, None 133 | 134 | 135 | class autoscale(web.ApiView): 136 | 137 | def get(self, request, app, name): 138 | instance = instances.get(name) 139 | return {'max': instance['max_concurrency'], 140 | 'min': instance['min_concurrency']} 141 | 142 | def post(self, request, app, name, nowait=False): 143 | return self.Ok(instances.autoscale(name, nowait=nowait, 144 | **self.params(('max', int), ('min', int)))) 145 | 146 | 147 | @web.simple_get 148 | def instance_stats(self, request, app, name): 149 | return instances.stats(name) 150 | 151 | 152 | @web.simple_get 153 | def task_state(self, request, app, uuid): 154 | return {'state': AsyncResult(uuid).state} 155 | 156 | 157 | @web.simple_get 158 | def task_result(self, request, app, uuid): 159 | return {'result': AsyncResult(uuid).result} 160 | 161 | 162 | @web.simple_get 163 | def task_wait(self, request, app, uuid): 164 | return {'result': AsyncResult(uuid).get()} 165 | 166 | 167 | @web.simple_get 168 | def ping(self, request): 169 | return {'ok': 'pong'} 170 | -------------------------------------------------------------------------------- /cyme/api/web.py: -------------------------------------------------------------------------------- 1 | """cyme.api.web 2 | 3 | - Contains utilities for creating our HTTP API. 4 | 5 | """ 6 | 7 | from __future__ import absolute_import 8 | 9 | import httplib as http 10 | import sys 11 | 12 | from functools import partial 13 | from traceback import format_exception 14 | 15 | from django.http import HttpResponse, HttpResponseNotFound 16 | from django.views.generic.base import View 17 | 18 | from anyjson import serialize 19 | from cell.exceptions import NoReplyError, NoRouteError 20 | from kombu.utils.encoding import safe_repr 21 | 22 | # Cross Origin Resource Sharing 23 | # See: http://www.w3.org/TR/cors/ 24 | ACCESS_CONTROL = { 25 | 'Allow-Origin': '*', 26 | 'Allow-Methods': ['GET', 'OPTIONS', 'POST', 'PUT', 'DELETE'], 27 | 'Max-Age': 86400, 28 | } 29 | 30 | 31 | class HttpResponseTimeout(HttpResponse): 32 | """The operation timed out.""" 33 | status_code = http.REQUEST_TIMEOUT 34 | 35 | 36 | class HttpResponseNotImplemented(HttpResponse): 37 | """The requested action is not implemented. 38 | Used for async requests when the operation is inherently sync.""" 39 | status_code = http.NOT_IMPLEMENTED 40 | 41 | 42 | def set_access_control_options(response, options=None): 43 | options = dict(ACCESS_CONTROL, **options or {}) 44 | try: 45 | options['Allow-Methods'] = ', '.join(options['Allow-Methods'] or []) 46 | except KeyError: 47 | pass 48 | 49 | for key, value in ACCESS_CONTROL.iteritems(): 50 | response['Access-Control-%s' % (key, )] = value 51 | 52 | 53 | def JsonResponse(data, status=http.OK, access_control=None, **kwargs): 54 | """Returns a JSON encoded response.""" 55 | if isinstance(data, (basestring, int, float, bool)): 56 | data = {'ok': data} 57 | if data is None or not isinstance(data, (dict, list, tuple)): 58 | return data 59 | kwargs.setdefault('content_type', 'application/json') 60 | response = HttpResponse(serialize(data), 61 | status=status, **kwargs) 62 | set_access_control_options(response, access_control) 63 | response.csrf_exempt = True 64 | return response 65 | Accepted = partial(JsonResponse, status=http.ACCEPTED) 66 | Created = partial(JsonResponse, status=http.CREATED) 67 | Error = partial(JsonResponse, status=http.INTERNAL_SERVER_ERROR) 68 | 69 | 70 | class ApiView(View): 71 | nowait = False # should the current operation be async? 72 | typemap = {int: lambda i: int(i) if i else None, 73 | float: lambda f: float(f) if f else None} 74 | _semipredicate = object() 75 | 76 | def dispatch(self, request, *args, **kwargs): 77 | self.nowait = kwargs.get('nowait', False) 78 | if request.method.lower() == 'get': 79 | kwargs.pop('nowait', None) 80 | if self.nowait: 81 | return self.NotImplemented('Operation cannot be async.') 82 | try: 83 | data = super(ApiView, self).dispatch(request, *args, **kwargs) 84 | except NoRouteError: 85 | return HttpResponseNotFound() 86 | except NoReplyError: 87 | return HttpResponseTimeout() 88 | except Exception, exc: 89 | return Error({'nok': [safe_repr(exc), 90 | ''.join(format_exception(*sys.exc_info()))]}) 91 | return self.Response(data) 92 | 93 | def Response(self, *args, **kwargs): 94 | return JsonResponse(*args, **kwargs) 95 | 96 | def Accepted(self, *args, **kwargs): 97 | return Accepted(*args, **kwargs) 98 | 99 | def Ok(self, data, *args, **kwargs): 100 | if self.nowait: 101 | data = data or {'ok': 'operation scheduled'} 102 | return self.Accepted(data, **kwargs) 103 | return self.Response(data, *args, **kwargs) 104 | 105 | def Created(self, data, *args, **kwargs): 106 | if self.nowait: 107 | data = data or {'ok': 'operation scheduled'} 108 | return self.Accepted(data, **kwargs) 109 | return Created(data, *args, **kwargs) 110 | 111 | def NotImplemented(self, *args, **kwargs): 112 | return HttpResponseNotImplemented(*args, **kwargs) 113 | 114 | def get_or_post(self, key, default=None): 115 | for d in (self.request.GET, self.request.POST): 116 | try: 117 | return d[key] 118 | except KeyError: 119 | pass 120 | return default 121 | 122 | def params(self, *keys): 123 | return dict(self.get_param(key) for key in keys) 124 | 125 | def get_param(self, key, type=None): 126 | semipredicate = self._semipredicate 127 | if isinstance(key, (list, tuple)): 128 | key, type = key 129 | type = self.typemap.get(type, type) 130 | value = self.get_or_post(key, semipredicate) 131 | if type and value != semipredicate: 132 | return key, type(value) 133 | return key, None if value == semipredicate else value 134 | 135 | 136 | def simple_get(fun): 137 | return type(fun.__name__, (ApiView, ), {'get': fun, 138 | '__module__': fun.__module__, '__doc__': fun.__doc__}) 139 | -------------------------------------------------------------------------------- /cyme/bin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/celery/cyme/c49047318c56f6674bd6d6ea6673b48be4eb7886/cyme/bin/__init__.py -------------------------------------------------------------------------------- /cyme/bin/base.py: -------------------------------------------------------------------------------- 1 | """cyme.bin.base 2 | 3 | - Utilities used by command line applications, 4 | basically just to set up Django without having an actual project. 5 | 6 | """ 7 | 8 | from __future__ import absolute_import 9 | from __future__ import with_statement 10 | 11 | from cyme import __version__, DEBUG, DEBUG_BLOCK, DEBUG_READERS 12 | 13 | import django 14 | import getpass 15 | import os 16 | import sys 17 | 18 | from importlib import import_module 19 | 20 | from kombu.utils import cached_property 21 | 22 | from cyme.utils import Path 23 | 24 | 25 | class Env(object): 26 | 27 | def __init__(self, needs_eventlet=False, instance_dir=None): 28 | self.needs_eventlet = needs_eventlet 29 | self.instance_dir = instance_dir 30 | 31 | def __enter__(self): 32 | if self.needs_eventlet: 33 | self.setup_eventlet() 34 | self.configure() 35 | self.setup_pool_limit() 36 | return self 37 | 38 | def __exit__(self, *exc_info): 39 | pass 40 | 41 | def setup_eventlet(self): 42 | import eventlet 43 | import eventlet.debug 44 | eventlet.monkey_patch() 45 | if DEBUG_READERS: 46 | eventlet.debug.hub_prevent_multiple_readers(False) 47 | print('+++ MULTIPLE READERS ALLOWED +++') # noqa+ 48 | if DEBUG_BLOCK: 49 | eventlet.debug.hub_blocking_detection(True) 50 | print('+++ BLOCKING DETECTION ENABLED +++') # noqa+ 51 | 52 | def configure(self): 53 | from django.conf import settings 54 | from cyme import settings as default_settings 55 | from cyme.utils import imerge_settings 56 | mod = os.environ.setdefault('DJANGO_SETTINGS_MODULE', 57 | default_settings.__name__) 58 | 59 | if not settings.configured: 60 | if django.VERSION < (1, 4): 61 | self.management.setup_environ(import_module(mod)) 62 | else: 63 | imerge_settings(settings, default_settings) 64 | if self.instance_dir: 65 | settings.CYME_INSTANCE_DIR = self.instance_dir 66 | 67 | def setup_pool_limit(self, **kwargs): 68 | from kombu import pools 69 | from celery import current_app as celery 70 | limit = kwargs.get('limit', celery.conf.BROKER_POOL_LIMIT) 71 | pools.set_limit(limit if self.needs_eventlet else 1) 72 | celery._pool = pools.connections[celery.broker_connection()] 73 | 74 | def syncdb(self, interactive=True): 75 | from django.conf import settings 76 | from django.db.utils import DEFAULT_DB_ALIAS 77 | dbconf = settings.DATABASES[DEFAULT_DB_ALIAS] 78 | if dbconf['ENGINE'] == 'django.db.backends.sqlite3': 79 | if Path(dbconf['NAME']).absolute().exists(): 80 | return 81 | gp, getpass.getpass = getpass.getpass, getpass.fallback_getpass 82 | try: 83 | self.management.call_command('syncdb', interactive=interactive) 84 | finally: 85 | getpass.getpass = gp 86 | 87 | @cached_property 88 | def management(self): 89 | from django.core import management 90 | return management 91 | 92 | 93 | class BaseApp(object): 94 | env = None 95 | needs_eventlet = False 96 | instance_dir = None 97 | 98 | def get_version(self): 99 | return 'cyme v%s' % (__version__, ) 100 | 101 | def run_from_argv(self, argv=None): 102 | argv = sys.argv if argv is None else argv 103 | if '--version' in argv: 104 | print(self.get_version()) 105 | sys.exit(0) 106 | try: 107 | with (self.env or 108 | Env(self.needs_eventlet, self.instance_dir)) as env: 109 | return self.run(env, argv) 110 | except KeyboardInterrupt: 111 | if DEBUG: 112 | raise 113 | raise SystemExit() 114 | __call__ = run_from_argv 115 | 116 | 117 | def app(**attrs): 118 | x = attrs 119 | 120 | def _app(fun): 121 | 122 | def run(self, *args, **kwargs): 123 | return fun(*args, **kwargs) 124 | 125 | attrs = dict({'run': run, '__module__': fun.__module__, 126 | '__doc__': fun.__doc__}, **x) 127 | 128 | return type(fun.__name__, (BaseApp, ), attrs)() 129 | 130 | return _app 131 | -------------------------------------------------------------------------------- /cyme/bin/cyme.py: -------------------------------------------------------------------------------- 1 | """cyme.bin.cyme""" 2 | 3 | from __future__ import absolute_import 4 | 5 | from .base import app 6 | 7 | 8 | @app() 9 | def cyme(env, argv): 10 | from cyme.management.commands import cyme 11 | cyme.Command(env=env).run_from_argv([argv[0], 'cyme'] + argv[1:]) 12 | 13 | 14 | if __name__ == '__main__': 15 | cyme() 16 | -------------------------------------------------------------------------------- /cyme/bin/cyme_branch.py: -------------------------------------------------------------------------------- 1 | """cyme.bin.cyme_branch 2 | 3 | - This is the script run by the :program:`cyme-branch` script installed 4 | by the Cyme distribution (defined in setup.py's ``entry_points``). 5 | 6 | - It in turn executes the cyme-branch management command. 7 | 8 | """ 9 | 10 | from __future__ import absolute_import 11 | 12 | from .base import app 13 | 14 | 15 | @app(needs_eventlet=True) 16 | def cyme_branch(env, argv): 17 | from cyme.management.commands import cyme_branch 18 | cyme_branch.Command(env).run_from_argv([argv[0], 'cyme-branch'] + argv[1:]) 19 | 20 | 21 | if __name__ == '__main__': 22 | cyme_branch() 23 | -------------------------------------------------------------------------------- /cyme/bin/cyme_list_branches.py: -------------------------------------------------------------------------------- 1 | """cyme.bin.cyme_list_branches""" 2 | from __future__ import absolute_import 3 | 4 | from .base import app 5 | 6 | import anyjson 7 | 8 | 9 | def get_branches(broker=None, limit=None): 10 | from cyme.branch.controller import Branch 11 | from celery import current_app as celery 12 | if limit: 13 | limit = int(limit) 14 | 15 | args = [broker] if broker else [] 16 | conn = celery.broker_connection(*args) 17 | return Branch(connection=conn).all(limit=limit) 18 | 19 | 20 | @app() 21 | def main(env, argv): 22 | opts = dict(arg.split('=', 1) 23 | for arg in argv[1:] if arg.startswith('--')) 24 | print(anyjson.serialize(get_branches(opts.get('--broker'), 25 | opts.get('--limit')))) 26 | 27 | 28 | if __name__ == '__main__': 29 | main() 30 | -------------------------------------------------------------------------------- /cyme/branch/__init__.py: -------------------------------------------------------------------------------- 1 | """cyme.branch 2 | 3 | - This is the Branch thread started by the :program:`cyme-branch` program. 4 | 5 | It starts the HTTP server, the Supervisor, and one or more controllers. 6 | 7 | """ 8 | 9 | from __future__ import absolute_import 10 | 11 | import logging 12 | 13 | from celery import current_app as celery 14 | from celery.utils import LOG_LEVELS, term 15 | from cell.g import Event 16 | from kombu.log import LogMixin 17 | from kombu.utils import gen_unique_id 18 | 19 | from . import signals 20 | from .state import state 21 | from .thread import gThread 22 | 23 | from cyme.utils import find_symbol, instantiate 24 | 25 | 26 | class MockSup(LogMixin): 27 | 28 | def __init__(self, thread, *args): 29 | self.thread = thread 30 | 31 | def start(self): 32 | self.thread.start() 33 | 34 | def stop(self): 35 | return self.thread.stop() 36 | 37 | 38 | class Branch(gThread): 39 | controller_cls = '.controller.Controller' 40 | httpd_cls = '.httpd.HttpServer' 41 | supervisor_cls = '.supervisor.Supervisor' 42 | intsup_cls = '.intsup.gSup' 43 | 44 | _components_ready = {} 45 | _components_shutdown = {} 46 | _presence_ready = 0 47 | _ready = False 48 | 49 | def __init__(self, addrport='', id=None, loglevel=logging.INFO, 50 | logfile=None, without_httpd=False, numc=2, sup_interval=None, 51 | ready_event=None, colored=None, **kwargs): 52 | self.id = id or gen_unique_id() 53 | if isinstance(addrport, basestring): 54 | addr, _, port = addrport.partition(':') 55 | addrport = (addr, int(port) if port else 8000) 56 | self.addrport = addrport 57 | self.connection = celery.broker_connection() 58 | self.without_httpd = without_httpd 59 | self.logfile = logfile 60 | self.loglevel = loglevel 61 | self.numc = numc 62 | self.ready_event = ready_event 63 | self.exit_request = Event() 64 | self.colored = colored or term.colored(enabled=False) 65 | self.httpd = None 66 | gSup = find_symbol(self, self.intsup_cls) 67 | if not self.without_httpd: 68 | self.httpd = MockSup(instantiate(self, self.httpd_cls, addrport), 69 | signals.httpd_ready) 70 | self.supervisor = gSup(instantiate(self, self.supervisor_cls, 71 | sup_interval), signals.supervisor_ready) 72 | self.controllers = [gSup(instantiate(self, self.controller_cls, 73 | id='%s.%s' % (self.id, i), 74 | connection=self.connection, 75 | branch=self), 76 | signals.controller_ready) 77 | for i in xrange(1, numc + 1)] 78 | c = [self.supervisor] + self.controllers + [self.httpd] 79 | c = self.components = list(filter(None, c)) 80 | self._components_ready = dict(zip([z.thread for z in c], 81 | [False] * len(c))) 82 | for controller in self.controllers: 83 | if hasattr(controller.thread, 'presence'): 84 | self._components_ready[controller.thread.presence] = False 85 | self._components_shutdown = dict(self._components_ready) 86 | super(Branch, self).__init__() 87 | 88 | def _component_ready(self, sender=None, **kwargs): 89 | if not self._ready: 90 | self._components_ready[sender] = True 91 | if all(self._components_ready.values()): 92 | signals.branch_ready.send(sender=self) 93 | if self.ready_event: 94 | self.ready_event.send() 95 | self.ready_event = None 96 | self._ready = True 97 | 98 | def on_ready(self, **kwargs): 99 | pass 100 | 101 | def prepare_signals(self): 102 | signals.controller_ready.connect(self._component_ready) 103 | signals.httpd_ready.connect(self._component_ready) 104 | signals.supervisor_ready.connect(self._component_ready) 105 | signals.presence_ready.connect(self._component_ready) 106 | signals.branch_ready.connect(self.on_ready) 107 | signals.thread_post_shutdown.connect(self._component_shutdown) 108 | 109 | def run(self): 110 | state.is_branch = True 111 | signals.branch_startup_request.send(sender=self) 112 | self.prepare_signals() 113 | self.info('Starting with id %r', self.id) 114 | [g.start() for g in self.components] 115 | self.exit_request.wait() 116 | 117 | def stop(self): 118 | self.exit_request.send(1) 119 | super(Branch, self).stop() 120 | 121 | def after(self): 122 | for component in reversed(self.components): 123 | if self._components_ready[component.thread]: 124 | try: 125 | component.stop() 126 | except KeyboardInterrupt: 127 | pass 128 | except BaseException, exc: 129 | component.error('Error in shutdown: %r', exc) 130 | 131 | def _component_shutdown(self, sender, **kwargs): 132 | self._components_shutdown[sender] = True 133 | if all(self._components_shutdown.values()): 134 | signals.branch_shutdown_complete.send(sender=self) 135 | 136 | def about(self): 137 | url = port = None 138 | if self.httpd: 139 | url, port = self.httpd.thread.url, self.httpd.thread.port 140 | port = self.httpd.thread.port if self.httpd else None 141 | return {'id': self.id, 142 | 'loglevel': LOG_LEVELS[self.loglevel], 143 | 'numc': self.numc, 144 | 'sup_interval': self.supervisor.interval, 145 | 'logfile': self.logfile, 146 | 'port': port, 147 | 'url': url} 148 | -------------------------------------------------------------------------------- /cyme/branch/controller.py: -------------------------------------------------------------------------------- 1 | """cyme.branch.controller 2 | 3 | - Actors used to manage entities across all branches. 4 | 5 | """ 6 | 7 | from __future__ import absolute_import 8 | 9 | from functools import partial 10 | 11 | from cell.presence import AwareActorMixin, announce_after 12 | from cell.utils import flatten, first_or_raise, shortuuid 13 | from celery import current_app as celery 14 | from kombu import Exchange 15 | from kombu.common import uuid 16 | 17 | from . import metrics 18 | from . import signals 19 | from .state import state 20 | from .thread import gThread 21 | 22 | from cyme import conf 23 | from cyme import models 24 | from cyme.utils import cached_property, find_symbol, promise 25 | from cyme.utils.actors import Actor, AwareAgent 26 | 27 | 28 | class CymeActor(Actor, AwareActorMixin): 29 | _announced = set() # note: global 30 | 31 | def setup(self): 32 | # retry publishing messages by default if running as cyme-branch. 33 | self.retry = state.is_branch 34 | self.default_fields = {'actor_id': self.id} 35 | 36 | 37 | class ModelActor(CymeActor): 38 | model = None 39 | 40 | def on_agent_ready(self): 41 | if self.name not in self._announced: 42 | self.log.info('%s: %s', self.name_plural, 43 | promise(lambda: ', '.join(self.state.all()))) 44 | self._announced.add(self.name) 45 | 46 | def contribute_to_state(self, state): 47 | state.model = self.model 48 | state.objects = self.model._default_manager 49 | return Actor.contribute_to_state(self, state) 50 | 51 | @cached_property 52 | def name(self): 53 | return unicode(self.model._meta.verbose_name.capitalize()) 54 | 55 | @cached_property 56 | def name_plural(self): 57 | return unicode(self.model._meta.verbose_name_plural).capitalize() 58 | 59 | 60 | class Branch(CymeActor): 61 | exchange = Exchange('cyme.Branch') 62 | default_timeout = 60 63 | types = ('direct', 'scatter', 'round-robin') 64 | meta_lookup_section = 'this' 65 | 66 | def on_agent_ready(self): 67 | if self.name not in self._announced: 68 | self.log.info('Actor ready', self.name) 69 | self._announced.add(self.name) 70 | 71 | class state: 72 | 73 | def id(self): 74 | return self.agent.branch.id 75 | 76 | def url(self): 77 | return self.agent.branch.httpd.thread.url 78 | 79 | def about(self): 80 | return self.agent.branch.about() 81 | 82 | def shutdown(self, id): 83 | if id in [self.id(), '*']: 84 | assert state.is_branch 85 | self.log.warn('Shutdown requested from remote.') 86 | raise SystemExit() 87 | raise self.Next() 88 | 89 | def all(self, **kw): 90 | return flatten(self.scatter('id', **kw)) 91 | 92 | def get(self, id, **kw): 93 | return self.send_to_able('about', to=id, **kw) 94 | 95 | def url(self, id=None, **kw): 96 | if id: 97 | return self.send_to_able('url', to=id, **kw) 98 | return flatten(self.scatter('url', **kw)) 99 | 100 | def shutdown(self, id): 101 | return self.send_to_able('shutdown', {'id': id}, to=id, nowait=True) 102 | 103 | def shutdown_all(self): 104 | return self.scatter('shutdown', {'id': '*'}, nowait=True) 105 | 106 | @property 107 | def meta(self): 108 | return {'this': [self.state.id()]} 109 | branches = Branch() 110 | 111 | 112 | class App(ModelActor): 113 | """Actor for managing the app model.""" 114 | model = models.App 115 | types = ('scatter', ) 116 | exchange = Exchange('cyme.App') 117 | _cache = {} 118 | 119 | class state: 120 | 121 | def all(self): 122 | return [app.name for app in self.objects.all()] 123 | 124 | def add(self, name, broker=None, arguments=None, extra_config=None): 125 | return self.objects.add(name, broker=broker, 126 | arguments=arguments, 127 | extra_config=extra_config).as_dict() 128 | 129 | def delete(self, name): 130 | return self.objects.filter(name=name).delete() and 'ok' 131 | 132 | def get(self, name): 133 | try: 134 | return self.objects.get(name=name).as_dict() 135 | except self.model.DoesNotExist: 136 | raise self.Next() 137 | 138 | def metrics(self): 139 | instance_dir = str(conf.CYME_INSTANCE_DIR) 140 | return {'load_average': metrics.load_average(), 141 | 'disk_use': metrics.df(instance_dir).capacity} 142 | 143 | def all(self): 144 | return flatten(self.scatter('all')) 145 | 146 | def add(self, name, **broker): 147 | self.scatter('add', dict({'name': name}, **broker), nowait=True) 148 | return self.state.add(name, **broker) 149 | 150 | def delete(self, name, **kw): 151 | self._cache.pop(name, None) 152 | return list(self.scatter('delete', dict({'name': name}, **kw))) 153 | 154 | def metrics(self, name=None): 155 | return list(self.scatter('metrics')) 156 | 157 | def get(self, name=None): 158 | objects = self.state.objects 159 | if not name: 160 | return objects.get_default() 161 | if name not in self._cache: 162 | app = self._get(name) 163 | if not app: 164 | raise KeyError(name) 165 | self._cache[name] = objects.recreate(**app) 166 | 167 | return self._cache[name] 168 | 169 | def _get(self, name): 170 | try: 171 | return self.state.get(name) 172 | except self.Next: 173 | return first_or_raise(self.scatter('get', {'name': name}, 174 | propagate=False), 175 | self.NoRouteError(name)) 176 | apps = App() 177 | 178 | 179 | class Instance(ModelActor): 180 | """Actor for managing the Instance model.""" 181 | model = models.Instance 182 | exchange = Exchange('cyme.Instance') 183 | default_timeout = 60 184 | types = ('direct', 'scatter', 'round-robin') 185 | meta_lookup_section = 'instances' 186 | 187 | class state: 188 | 189 | def all(self, app=None): 190 | fun = self.objects.all 191 | if app: 192 | fun = partial(self.objects.filter, app=apps.get(app)) 193 | return [instance.name for instance in fun()] 194 | 195 | def get(self, name, app=None): 196 | try: 197 | x = self.objects.get(name=name) 198 | except self.model.DoesNotExist: 199 | raise self.Next() 200 | return x.as_dict() 201 | 202 | @announce_after 203 | def add(self, name=None, app=None, **kwargs): 204 | return self.local.add(name, app=apps.get(app), **kwargs).as_dict() 205 | 206 | @announce_after 207 | def remove(self, name, app=None): 208 | return self.local.remove(name) and 'ok' 209 | 210 | def restart(self, name, app=None): 211 | return self.local.restart(name) and 'ok' 212 | 213 | def enable(self, name, app=None): 214 | return self.local.enable(name) and 'ok' 215 | 216 | def disable(self, name, app=None): 217 | return self.local.disable(name) and 'ok' 218 | 219 | def add_consumer(self, name, queue): 220 | return self.local.add_consumer(name, queue) and 'ok' 221 | 222 | def cancel_consumer(self, name, queue): 223 | return self.local.cancel_consumer(name, queue) and 'ok' 224 | 225 | def remove_queue_from_all(self, queue): 226 | return [instance.name for instance in 227 | self.objects.remove_queue_from_instances(queue)] 228 | 229 | def autoscale(self, name, max=None, min=None): 230 | instance = self.local.get(name) 231 | instance.autoscale(max=max, min=min) 232 | return {'max': instance.max_concurrency, 233 | 'min': instance.min_concurrency} 234 | 235 | def consuming_from(self, name): 236 | return self.local.get(name).consuming_from() 237 | 238 | def stats(self, name): 239 | return self.local.get(name).stats() 240 | 241 | @cached_property 242 | def local(self): 243 | return find_symbol(self, '.managers.local_instances') 244 | 245 | def get(self, name, app=None, **kw): 246 | return self.send_to_able('get', 247 | {'name': name, 'app': app}, to=name, **kw) 248 | 249 | def all(self, app=None): 250 | return flatten(self.scatter('all', {'app': app})) 251 | 252 | def add(self, name=None, app=None, nowait=False, **kwargs): 253 | if nowait: 254 | name = name if name else uuid() 255 | ret = self.throw('add', dict({'name': name, 'app': app}, **kwargs), 256 | nowait=nowait) 257 | if nowait: 258 | return {'name': name} 259 | return ret 260 | 261 | def remove(self, name, **kw): 262 | return self.send_to_able('remove', {'name': name}, to=name, **kw) 263 | 264 | def restart(self, name, **kw): 265 | return self.send_to_able('restart', {'name': name}, to=name, **kw) 266 | 267 | def enable(self, name, **kw): 268 | return self.send_to_able('enable', args={'name': name}, to=name, **kw) 269 | 270 | def disable(self, name, **kw): 271 | return self.send_to_able('disable', args={'name': name}, to=name, **kw) 272 | 273 | def add_consumer(self, name, queue, **kw): 274 | return self.send_to_able('add_consumer', 275 | {'name': name, 'queue': queue}, to=name, **kw) 276 | 277 | def cancel_consumer(self, name, queue, **kw): 278 | return self.send_to_able('cancel_consumer', 279 | {'name': name, 'queue': queue}, to=name, **kw) 280 | 281 | def remove_queue_from_all(self, queue, **kw): 282 | return flatten(self.scatter('remove_queue_from_all', 283 | {'queue': queue}, **kw) or []) 284 | 285 | def autoscale(self, name, max=None, min=None, **kw): 286 | return self.send_to_able('autoscale', 287 | {'name': name, 'min': min, 'max': max}, to=name, **kw) 288 | 289 | def consuming_from(self, name, **kw): 290 | return self.send_to_able('consuming_from', 291 | {'name': name}, to=name, **kw) 292 | 293 | def stats(self, name, **kw): 294 | return self.send_to_able('stats', {'name': name}, to=name, **kw) 295 | 296 | @property 297 | def meta(self): 298 | return {'instances': self.state.all()} 299 | instances = Instance() 300 | 301 | 302 | class Queue(ModelActor): 303 | """Actor for managing the Queue model.""" 304 | model = models.Queue 305 | exchange = Exchange('cyme.Queue') 306 | types = ('direct', 'scatter', 'round-robin') 307 | default_timeout = 2 308 | meta_lookup_section = 'queues' 309 | 310 | class state: 311 | 312 | def all(self): 313 | return [q.name for q in self.objects.all()] 314 | 315 | def get(self, name): 316 | try: 317 | return self.objects.get(name=name).as_dict() 318 | except self.model.DoesNotExist: 319 | raise KeyError(name) 320 | 321 | @announce_after 322 | def add(self, name, **declaration): 323 | return self.objects._add(name, **declaration).as_dict() 324 | 325 | @announce_after 326 | def delete(self, name): 327 | self.objects.filter(name=name).delete() 328 | return 'ok' 329 | 330 | def all(self): 331 | return flatten(self.scatter('all')) 332 | 333 | def get(self, name): 334 | try: 335 | # see if we have the queue locally. 336 | return self.state.get(name) 337 | except KeyError: 338 | # if not, ask the agents. 339 | return self.send_to_able('get', {'name': name}, to=name) 340 | 341 | def add(self, name, nowait=False, **decl): 342 | return self.throw('add', dict({'name': name}, **decl), nowait=nowait) 343 | 344 | def delete(self, name, **kw): 345 | instances.remove_queue_from_all(name, nowait=True) 346 | return self.send_to_able('delete', {'name': name}, to=name, **kw) 347 | 348 | @property 349 | def meta(self): 350 | return {'queues': self.state.all()} 351 | queues = Queue() 352 | 353 | 354 | class Controller(AwareAgent, gThread): 355 | actors = [Branch(), App(), Instance(), Queue()] 356 | connect_max_retries = celery.conf.BROKER_CONNECTION_MAX_RETRIES 357 | extra_shutdown_steps = 2 358 | _ready_sent = False 359 | _presence_ready_sent = False 360 | 361 | def __init__(self, *args, **kwargs): 362 | self.branch = kwargs.pop('branch', None) 363 | AwareAgent.__init__(self, *args, **kwargs) 364 | gThread.__init__(self) 365 | 366 | def on_awake(self): 367 | # bind global actors to this agent, 368 | # so presence can be used. 369 | for actor in (branches, apps, instances, queues): 370 | actor.agent = self 371 | 372 | def on_connection_revived(self): 373 | state.on_broker_revive() 374 | 375 | def on_consume_ready(self, *args, **kwargs): 376 | if not self._ready_sent: 377 | signals.controller_ready.send(sender=self) 378 | self._ready_sent = True 379 | super(Controller, self).on_consume_ready() 380 | 381 | def on_iteration(self): 382 | self.respond_to_ping() 383 | 384 | def on_connection_error(self, exc, interval): 385 | self.respond_to_ping() 386 | super(Controller, self).on_connection_error(exc, interval) 387 | 388 | def on_presence_ready(self): 389 | if not self._presence_ready_sent: 390 | signals.presence_ready.send(sender=self.presence) 391 | self._presence_ready_sent = True 392 | 393 | def stop(self): 394 | self.should_stop = True 395 | if hasattr(self, 'presence') and self.presence.g: 396 | self.debug('waiting for presence to exit') 397 | signals.thread_shutdown_step.send(sender=self) 398 | self.presence.g.wait() 399 | signals.thread_shutdown_step.send(sender=self) 400 | super(Controller, self).stop() 401 | 402 | @property 403 | def logger_name(self): 404 | return '#'.join([self.__class__.__name__, self._shortid()]) 405 | 406 | def _shortid(self): 407 | if '.' in self.id: 408 | return shortuuid(self.id) + '..' + self.id[-2:] 409 | return shortuuid(self.id) 410 | -------------------------------------------------------------------------------- /cyme/branch/httpd.py: -------------------------------------------------------------------------------- 1 | """cyme.branch.httpd 2 | 3 | - Our embedded WSGI server used to serve the HTTP API. 4 | 5 | """ 6 | 7 | from __future__ import absolute_import 8 | 9 | from eventlet import listen 10 | from eventlet import wsgi 11 | 12 | from django.core.handlers import wsgi as djwsgi 13 | from django.core.servers.basehttp import AdminMediaHandler 14 | from requests import get 15 | 16 | from .thread import gThread 17 | from .signals import httpd_ready 18 | 19 | 20 | class HttpServer(gThread): 21 | joinable = False 22 | 23 | def __init__(self, addrport=None): 24 | host, port = addrport or ('', 8000) 25 | if host == 'localhost': 26 | # dnspython bug? 27 | host = '127.0.0.1' 28 | self.host, self.port = self.addrport = (host, port) 29 | super(HttpServer, self).__init__() 30 | 31 | def server(self, sock, handler): 32 | return wsgi.server(sock, handler, 33 | log=self.create_log(), 34 | protocol=self.create_http_protocol()) 35 | 36 | def run(self): 37 | handler = AdminMediaHandler(djwsgi.WSGIHandler()) 38 | sock = listen(self.addrport) 39 | g = self.spawn(self.server, sock, handler) 40 | self.info('ready') 41 | httpd_ready.send(sender=self, addrport=self.addrport, 42 | handler=handler, sock=sock) 43 | return g.wait() 44 | 45 | def _do_ping(self, timeout): 46 | return get(self.url + '/ping/', timeout=timeout).ok 47 | 48 | def create_log(self): 49 | logger = self 50 | 51 | class _Log(object): 52 | 53 | def write(self, message): 54 | message = message.rstrip('\n') 55 | (logger.debug if '/ping/' in message else logger.info)(message) 56 | 57 | return _Log() 58 | 59 | def create_http_protocol(self): 60 | logger = self 61 | 62 | class HttpProtocol(wsgi.HttpProtocol): 63 | 64 | def get_format_args(self, format, *args): 65 | return ['%s - - [%s] %s', self.address_string(), 66 | self.log_date_time_string(), 67 | format] + args 68 | 69 | def log_message(self, format, *args): 70 | return logger.info(*self.get_format_args(format, *args)) 71 | 72 | def log_error(self, format, *args): 73 | return logger.error(*self.get_format_args(format, *args)) 74 | 75 | return HttpProtocol 76 | 77 | @property 78 | def url(self): 79 | addr, port = self.addrport 80 | if not addr or addr in ('0.0.0.0', ): 81 | addr = '127.0.0.1' 82 | return 'http://%s:%s' % (addr, port) 83 | 84 | @property 85 | def logger_name(self): 86 | return 'wsgi' 87 | -------------------------------------------------------------------------------- /cyme/branch/intsup.py: -------------------------------------------------------------------------------- 1 | """cyme.branch.intsup 2 | 3 | - Internal supervisor used to ensure our threads are still running. 4 | 5 | """ 6 | 7 | from __future__ import absolute_import, with_statement 8 | 9 | from os import _exit 10 | from time import sleep 11 | 12 | from cell.g import Event 13 | 14 | from .thread import gThread 15 | 16 | SUP_ERROR_NOT_STARTED = """\ 17 | found thread not able to start?\ 18 | """ 19 | 20 | SUP_ERROR_PING_TIMEOUT = """\ 21 | suspected thread crash or blocking: %r\ 22 | """ 23 | 24 | 25 | class gSup(gThread): 26 | 27 | def __init__(self, thread, signal, interval=5, timeout=600): 28 | self.thread = thread 29 | self.interval = interval 30 | self.timeout = timeout 31 | self.signal = signal 32 | super(gSup, self).__init__() 33 | 34 | def start_wait_child(self): 35 | self._ready_event = Event() 36 | self.signal.connect(self._on_thread_ready, sender=self.thread) 37 | self.thread.start() 38 | self._ready_event.wait() 39 | assert self._ready_event.ready() 40 | return self.thread 41 | 42 | def _on_thread_ready(self, **kwargs): 43 | self._ready_event.send(1) 44 | self.signal.disconnect(self._on_thread_ready) 45 | 46 | def run(self): 47 | self.debug('starting') 48 | interval = self.interval 49 | thread = self.start_wait_child() 50 | self.info('started') 51 | timeout = self.timeout 52 | critical = self.critical 53 | 54 | while not self.should_stop: 55 | try: 56 | pong = thread.ping(timeout) 57 | except (self.Timeout, Exception), exc: 58 | critical(SUP_ERROR_PING_TIMEOUT % (exc, )) 59 | _exit(0) 60 | if not pong: 61 | critical(SUP_ERROR_NOT_STARTED) 62 | _exit(0) 63 | sleep(interval) 64 | 65 | def stop(self): 66 | super(gSup, self).stop() 67 | self.thread.stop() 68 | 69 | @property 70 | def logger_name(self): 71 | return '%s<%s>' % (self.name, self.thread.name) 72 | -------------------------------------------------------------------------------- /cyme/branch/managers.py: -------------------------------------------------------------------------------- 1 | """cyme.branch.managers 2 | 3 | - Contains the :class:`LocalInstanceManager` instance, 4 | which is the preferred API used to control and manage worker 5 | instances handled by this branch. I.e. it can be used to do synchronous 6 | actions that don't return until the supervisor has performed them. 7 | 8 | """ 9 | 10 | from __future__ import absolute_import 11 | 12 | from .supervisor import supervisor as sup 13 | 14 | from cyme.models import Broker, Instance 15 | from cyme.utils import force_list 16 | 17 | 18 | class LocalInstanceManager(object): 19 | Instances = Instance._default_manager 20 | Brokers = Broker._default_manager 21 | 22 | def get(self, name): 23 | return self.Instances.get(name=name) 24 | 25 | def add(self, name=None, queues=None, 26 | max_concurrency=1, min_concurrency=1, broker=None, 27 | pool=None, app=None, arguments=None, extra_config=None, 28 | nowait=False, **kwargs): 29 | broker = self.Brokers.get_or_create(url=broker)[0] if broker else None 30 | instance = self.Instances.add(name, queues, max_concurrency, 31 | min_concurrency, broker, pool, app, 32 | arguments, extra_config) 33 | return self.maybe_wait(sup.verify, instance, nowait) 34 | 35 | def remove(self, name, nowait=False): 36 | return self.maybe_wait(sup.shutdown, 37 | self.Instances.remove(name), nowait) 38 | 39 | def restart(self, name, nowait=False): 40 | return self.maybe_wait(sup.restart, self.get(name), nowait) 41 | 42 | def enable(self, name, nowait=False): 43 | return self.maybe_wait(sup.verify, 44 | self.Instances.enable(name), nowait) 45 | 46 | def disable(self, name, nowait=False): 47 | return self.maybe_wait(sup.verify, 48 | self.Instances.disable(name), nowait) 49 | 50 | def add_consumer(self, name, queue, nowait=False): 51 | return self.maybe_wait(sup.verify, 52 | self.get(name).add_queue_eventually(queue), nowait) 53 | 54 | def cancel_consumer(self, name, queue, nowait=False): 55 | return self.maybe_wait(sup.verify, 56 | self.Instances.remove_queue_from_instances(queue, name=name), 57 | nowait) 58 | 59 | def remove_queue(self, queue, nowait=False): 60 | return self.maybe_wait(sup.verify, 61 | self.Instances.remove_queue_from_instances(queue), nowait) 62 | 63 | def maybe_wait(self, fun, instances, nowait): 64 | if instances: 65 | g = fun(force_list(instances)) 66 | nowait and g.wait() 67 | return instances 68 | local_instances = LocalInstanceManager() 69 | -------------------------------------------------------------------------------- /cyme/branch/metrics.py: -------------------------------------------------------------------------------- 1 | """cyme.branch.metrics""" 2 | 3 | from __future__ import absolute_import 4 | 5 | import os 6 | from math import ceil 7 | 8 | from cyme.utils import cached_property 9 | 10 | 11 | def load_average(): 12 | return tuple(ceil(l * 1e2) / 1e2 for l in os.getloadavg()) 13 | 14 | 15 | class df(object): 16 | 17 | def __init__(self, path): 18 | self.path = path 19 | 20 | @property 21 | def total_blocks(self): 22 | return self.stat.f_blocks * self.stat.f_frsize / 1024 23 | 24 | @property 25 | def available(self): 26 | return self.stat.f_bavail * self.stat.f_frsize / 1024 27 | 28 | @property 29 | def capacity(self): 30 | avail = self.stat.f_bavail 31 | used = self.stat.f_blocks - self.stat.f_bfree 32 | return int(ceil(used * 100.0 / (used + avail) + 0.5)) 33 | 34 | @cached_property 35 | def stat(self): 36 | return os.statvfs(self.path) 37 | -------------------------------------------------------------------------------- /cyme/branch/signals.py: -------------------------------------------------------------------------------- 1 | """cyme.branch.signals""" 2 | 3 | from __future__ import absolute_import 4 | 5 | from celery.utils.dispatch import Signal 6 | 7 | #: Sent when the http server is ready to accept requests. 8 | #: Arguments: 9 | #: 10 | #: :sender: the :class:`~cyme.httpd.HttpServer` instance. 11 | #: :addrport: the ``(hostname, port)`` tuple. 12 | #: :handler: the WSGI handler used. 13 | #: :sock: the socket used. 14 | httpd_ready = Signal(providing_args=['addrport', 'handler', 'sock']) 15 | 16 | #: Sent when the supervisor is ready. 17 | #: Arguments: 18 | #: 19 | #: :sender: is the :class:`~cyme.supervisor.Supervisor` instance. 20 | supervisor_ready = Signal() 21 | 22 | #: Sent when a controller is ready. 23 | #: 24 | #: Arguments: 25 | #: :sender: is the :class:`~cyme.controller.Controller` instance. 26 | controller_ready = Signal() 27 | 28 | #: Sent when the branch and all its components are ready to serve. 29 | #: 30 | #: Arguments: 31 | #: :sender: is the :class:`~cyme.branch.Branch` instance. 32 | branch_ready = Signal() 33 | 34 | branch_shutdown_complete = Signal() 35 | 36 | 37 | branch_startup_request = Signal() 38 | branch_shutdown_request = Signal() 39 | 40 | 41 | thread_pre_shutdown = Signal() 42 | thread_pre_join = Signal(providing_args=['timeout']) 43 | thread_exit = Signal() 44 | thread_post_join = Signal() 45 | thread_post_shutdown = Signal() 46 | thread_shutdown_step = Signal() 47 | 48 | thread_pre_start = Signal() 49 | thread_post_start = Signal() 50 | thread_startup_step = Signal() 51 | presence_ready = Signal() 52 | -------------------------------------------------------------------------------- /cyme/branch/state.py: -------------------------------------------------------------------------------- 1 | """cyme.branch.state 2 | 3 | - Global branch state. 4 | 5 | - Used to keep track lost connections and so on, which is used by the 6 | supervisor to know if an instance is actually down, or if it is just the 7 | connection being shaky. 8 | 9 | """ 10 | 11 | from __future__ import absolute_import 12 | 13 | from time import time 14 | 15 | from cyme.utils import cached_property, find_symbol 16 | 17 | 18 | class State(object): 19 | broker_last_revived = None 20 | 21 | #: set to true if the process is a cyme-branch 22 | is_branch = False 23 | 24 | def on_broker_revive(self, *args, **kwargs): 25 | self.broker_last_revived = time() 26 | self.supervisor.resume() 27 | 28 | @property 29 | def time_since_broker_revived(self): 30 | return time() - self.broker_last_revived 31 | 32 | @cached_property 33 | def supervisor(self): 34 | return find_symbol(self, '.supervisor.supervisor') 35 | 36 | state = State() 37 | -------------------------------------------------------------------------------- /cyme/branch/supervisor.py: -------------------------------------------------------------------------------- 1 | """cyme.branch.supervisor""" 2 | 3 | from __future__ import absolute_import 4 | from __future__ import with_statement 5 | 6 | from threading import Lock 7 | from Queue import Empty 8 | 9 | from celery.local import Proxy 10 | from eventlet.queue import LightQueue 11 | from eventlet.event import Event 12 | 13 | from .signals import supervisor_ready 14 | from .thread import gThread 15 | 16 | from cyme.status import Status 17 | 18 | __current = None 19 | 20 | 21 | class Supervisor(gThread, Status): 22 | """The supervisor wakes up at intervals to monitor changes in the model. 23 | It can also be requested to perform specific operations, and these 24 | operations can be either async or sync. 25 | 26 | :keyword interval: This is the interval (in seconds as an int/float), 27 | between verifying all the registered instances. 28 | :keyword queue: Custom :class:`~Queue.Queue` instance used to send 29 | and receive commands. 30 | 31 | It is responsible for: 32 | 33 | * Stopping removed instances. 34 | * Starting new instances. 35 | * Restarting unresponsive/killed instances. 36 | * Making sure the instances consumes from the queues 37 | specified in the model, sending ``add_consumer``/- 38 | ``cancel_consumer`` broadcast commands to the instances as it 39 | finds inconsistencies. 40 | * Making sure the max/min concurrency setting is as specified in the 41 | model, sending ``autoscale`` broadcast commands to the noes 42 | as it finds inconsistencies. 43 | 44 | The supervisor is resilient to intermittent connection failures, 45 | and will auto-retry any operation that is dependent on a broker. 46 | 47 | Since workers cannot respond to broadcast commands while the 48 | broker is off-line, the supervisor will not restart affected 49 | instances until the instance has had a chance to reconnect (decided 50 | by the :attr:`wait_after_broker_revived` attribute). 51 | 52 | """ 53 | #: Limit instance restarts to 1/m, so out of control 54 | #: instances will be disabled 55 | restart_max_rate = '1/m' 56 | 57 | #: Default interval_max for ensure_connection is 30 secs. 58 | wait_after_broker_revived = 35.0 59 | 60 | #: Connection errors pauses the supervisor, so events does not accumulate. 61 | paused = False 62 | 63 | #: Default interval (time in seconds as a float to reschedule). 64 | interval = 60.0 65 | 66 | def __init__(self, interval=None, queue=None, set_as_current=True): 67 | self.set_as_current = set_as_current 68 | if self.set_as_current: 69 | set_current(self) 70 | self._orig_queue_arg = queue 71 | self.interval = interval or self.interval 72 | self.queue = LightQueue() if queue is None else queue 73 | self._pause_mutex = Lock() 74 | self._last_update = None 75 | gThread.__init__(self) 76 | Status.__init__(self) 77 | 78 | def __copy__(self): 79 | return self.__class__(self.interval, self._orig_queue_arg) 80 | 81 | def pause(self): 82 | """Pause all timers.""" 83 | self.respond_to_ping() 84 | with self._pause_mutex: 85 | if not self.paused: 86 | self.debug('pausing') 87 | self.paused = True 88 | 89 | def resume(self): 90 | """Resume all timers.""" 91 | with self._pause_mutex: 92 | if self.paused: 93 | self.debug('resuming') 94 | self.paused = False 95 | 96 | def verify(self, instances, ratelimit=False): 97 | """Verify the consistency of one or more instances. 98 | 99 | :param instances: List of instances to verify. 100 | 101 | This operation is asynchronous, and returns a :class:`Greenlet` 102 | instance that can be used to wait for the operation to complete. 103 | 104 | """ 105 | return self._request(instances, self._do_verify_instance, 106 | {'ratelimit': ratelimit}) 107 | 108 | def restart(self, instances): 109 | """Restart one or more instances. 110 | 111 | :param instances: List of instances to restart. 112 | 113 | This operation is asynchronous, and returns a :class:`Greenlet` 114 | instance that can be used to wait for the operation to complete. 115 | 116 | """ 117 | return self._request(instances, self._do_restart_instance) 118 | 119 | def shutdown(self, instances): 120 | """Shutdown one or more instances. 121 | 122 | :param instances: List of instances to stop. 123 | 124 | This operation is asynchronous, and returns a :class:`Greenlet` 125 | instance that can be used to wait for the operation to complete. 126 | 127 | .. warning:: 128 | 129 | Note that the supervisor will automatically restart 130 | any stopped instances unless the corresponding :class:`Instance` 131 | model has been marked as disabled. 132 | 133 | """ 134 | return self._request(instances, self._do_stop_instance) 135 | 136 | def _request(self, instances, action, kwargs={}): 137 | event = Event() 138 | self.queue.put_nowait((instances, event, action, kwargs)) 139 | return event 140 | 141 | def before(self): 142 | self.start_periodic_timer(self.interval, self._verify_all) 143 | 144 | def run(self): 145 | queue = self.queue 146 | self.info('started') 147 | supervisor_ready.send(sender=self) 148 | while not self.should_stop: 149 | try: 150 | instances, event, action, kwargs = queue.get(timeout=1) 151 | except Empty: 152 | self.respond_to_ping() 153 | continue 154 | self.respond_to_ping() 155 | self.debug('wake-up') 156 | try: 157 | for instance in instances: 158 | try: 159 | action(instance, **kwargs) 160 | except Exception, exc: 161 | self.error('Event caused exception: %r', exc) 162 | finally: 163 | event.send(True) 164 | 165 | def _verify_all(self, force=False): 166 | if self._last_update and self._last_update.ready(): 167 | try: 168 | self._last_update.wait() # collect result 169 | except self.GreenletExit: 170 | pass 171 | force = True 172 | if not self._last_update or force: 173 | self._last_update = self.verify(self.all_instances(), 174 | ratelimit=True) 175 | 176 | 177 | class _OfflineSupervisor(object): 178 | 179 | def wait(self): 180 | pass 181 | 182 | def _noop(self, *args, **kwargs): 183 | return self 184 | pause = resume = verify = shutdown = restart = _noop 185 | 186 | 187 | def set_current(sup): 188 | global __current 189 | __current = sup 190 | return __current 191 | 192 | 193 | def get_current(): 194 | if __current is None: 195 | return _OfflineSupervisor() 196 | return __current 197 | 198 | supervisor = Proxy(get_current) 199 | -------------------------------------------------------------------------------- /cyme/branch/thread.py: -------------------------------------------------------------------------------- 1 | """cyme.branch.thread 2 | 3 | - Utilities for working with greenlets. 4 | 5 | """ 6 | 7 | from __future__ import absolute_import 8 | from __future__ import with_statement 9 | 10 | import os 11 | 12 | from Queue import Empty 13 | 14 | from cell.g import Event, spawn, timer, Queue 15 | from kombu.log import LogMixin 16 | from eventlet import Timeout 17 | 18 | from . import signals 19 | 20 | 21 | class AlreadyStartedError(Exception): 22 | """Raised if trying to start a thread instance that is already started.""" 23 | 24 | 25 | class gThread(LogMixin): 26 | AlreadyStarted = AlreadyStartedError 27 | Timeout = Timeout 28 | 29 | #: Name of the thread, used in logs and such. 30 | name = None 31 | 32 | #: Greenlet instance of the thread, set when the thread is started. 33 | g = None 34 | 35 | #: Set when the thread is requested to stop. 36 | should_stop = False 37 | 38 | #: Set this to False if it is not possible to join the thread. 39 | joinable = True 40 | 41 | _exit_event = None 42 | _ping_queue = None 43 | _timers = None 44 | extra_startup_steps = 0 45 | extra_shutdown_steps = 0 46 | 47 | def __init__(self): 48 | self.name = self.name or self.__class__.__name__ 49 | # the exit event is sent when the thread exits, 50 | # and used by `join` to detect this. 51 | self._exit_event = Event() 52 | 53 | # we maintain a list of timers started by the thread, 54 | # so these can be cancelled at shutdown. 55 | self._timers = [] 56 | 57 | def before(self): 58 | """Called at the beginning of :meth:`start`.""" 59 | pass 60 | 61 | def run(self): 62 | raise NotImplementedError("gThreads must implement 'run'") 63 | 64 | def after(self): 65 | """Call after the thread has shut down.""" 66 | pass 67 | 68 | def start_periodic_timer(self, interval, fun, *args, **kwargs): 69 | """Apply function every ``interval`` seconds. 70 | 71 | :param interval: Interval in seconds (int/float). 72 | :param fun: The function to apply. 73 | :param \*args: Additional arguments to pass. 74 | :keyword \*\*kwargs: Additional keyword arguments to pass. 75 | 76 | :returns: entry object, with ``cancel`` and ``kill`` methods. 77 | 78 | """ 79 | entry = timer(interval, fun, *args, **kwargs) 80 | self._timers.append(entry) 81 | return entry 82 | 83 | def start(self): 84 | """Spawn green thread, and returns 85 | :class:`~eventlet.greenthread.GreenThread` instance.""" 86 | if self.g is None: 87 | self.before() 88 | signals.thread_pre_start.send(sender=self) 89 | self._ping_queue = Queue() 90 | g = self.g = self.spawn(self._crashsafe, self.run) 91 | g.link(self._on_exit) 92 | self.debug('%s spawned', self.name) 93 | signals.thread_post_start.send(sender=self) 94 | return g 95 | raise self.AlreadyStarted('cannot start thread twice') 96 | 97 | def stop(self, join=True, timeout=1e100): 98 | """Shutdown the thread. 99 | 100 | This will also cancel+kill any periodic timers registered 101 | by the thread. 102 | 103 | :keyword join: Given that the thread is :attr:`joinable`, if 104 | true will also wait until the thread exits 105 | (by calling :meth:`join`). 106 | :keyword timeout: Timeout for join (default is 1e+100). 107 | 108 | """ 109 | self.debug('shutdown initiated') 110 | signals.thread_pre_shutdown.send(sender=self) 111 | self.should_stop = True 112 | for entry in self._timers: 113 | self.debug('killing timer %r' % (entry, )) 114 | entry.cancel() 115 | entry.kill() 116 | if join and self.joinable: 117 | try: 118 | self.join(timeout) 119 | except self.Timeout: 120 | self.error( 121 | 'exceeded exit timeout (%s), will try to kill', timeout) 122 | self.kill() 123 | self.after() 124 | signals.thread_post_shutdown.send(sender=self) 125 | 126 | def join(self, timeout=None): 127 | """Wait until the thread exits. 128 | 129 | :keyword timeout: Timeout in seconds (int/float). 130 | 131 | :raises eventlet.Timeout: if the thread can't be joined 132 | before the provided timeout. 133 | 134 | 135 | """ 136 | with self.Timeout(timeout): 137 | signals.thread_pre_join.send(sender=self, timeout=timeout) 138 | self.debug('joining (%s)', timeout) 139 | self._exit_event.wait() 140 | signals.thread_post_join.send(sender=self) 141 | 142 | def respond_to_ping(self): 143 | try: 144 | self._ping_queue.get_nowait().send() 145 | except Empty: 146 | pass 147 | 148 | def _do_ping(self, timeout=None): 149 | with self.Timeout(timeout): 150 | event = Event() 151 | self._ping_queue.put_nowait(event) 152 | event.wait() 153 | return event.ready() 154 | 155 | def ping(self, timeout=None): 156 | if self.g and self._ping_queue is not None: 157 | return self._do_ping(timeout) 158 | return False 159 | 160 | def _on_exit(self, g): 161 | # called when the thread exits to unblock `join`. 162 | self._exit_event.send() 163 | 164 | def kill(self): 165 | """Kill the green thread.""" 166 | if self.g is not None: 167 | self.g.kill() 168 | 169 | def _crashsafe(self, fun, *args, **kwargs): 170 | try: 171 | fun(*args, **kwargs) 172 | signals.thread_exit.send(sender=self) 173 | self.debug('exiting') 174 | except Exception, exc: 175 | self.error('Thread crash detected: %r', exc) 176 | os._exit(0) 177 | except self.Timeout, exc: 178 | self.error('Thread raised timeout: %r', exc) 179 | os._exit(0) 180 | 181 | def spawn(self, fun, *args, **kwargs): 182 | return spawn(fun, *args, **kwargs) 183 | 184 | @property 185 | def logger_name(self): 186 | return self.name 187 | -------------------------------------------------------------------------------- /cyme/client/__init__.py: -------------------------------------------------------------------------------- 1 | """cyme.client 2 | 3 | - Python client for the Cyme HTTP API. 4 | 5 | Branches 6 | ~~~~~~~~ 7 | 8 | >>> client = Client('http://localhost:8000') 9 | >>> client.branches 10 | ['cyme1.example.com', 'cyme2.example.com'] 11 | 12 | >>> client.branch_info('cyme1.example.com') 13 | {'sup_interval': 5, 'numc': 2, 'loglevel': 'INFO', 14 | 'logfile': None, 'id': 'cyme1.example.com', 'port': 8000} 15 | 16 | Applications 17 | ~~~~~~~~~~~~ 18 | 19 | >>> app = client.get('foo') 20 | >>> app 21 | 22 | >>> app.info 23 | {'name': 'foo', 'broker': 'amqp://guest:guest@localhost:5672//'} 24 | 25 | Queues 26 | ~~~~~~ 27 | 28 | >>> app.queues.add('myqueue', exchange='myex', routing_key='x') 29 | >>> 30 | >>> my_queue = app.queues.get('my_queue') 31 | 32 | >>> app.queues 33 | [] 34 | 35 | 36 | Instances 37 | ~~~~~~~~~ 38 | 39 | >>> i = app.instances.add() 40 | >>> i 41 | 42 | >>> i.name 43 | u'd87798f3-0bb0-4161-8e0b-a5f069b1d58b' 44 | 45 | >>> i.broker # < inherited from app 46 | u'amqp://guest:guest@localhost:5672//' 47 | 48 | >>> app.instances 49 | [] 50 | 51 | >>> i.autoscale() # current autoscale settings 52 | {'max': 1, 'min': 1} 53 | 54 | >>> i.autoscale(max=10, min=10) # always run 10 processes 55 | {'max': 10, 'min': 10} 56 | 57 | >>> i.stats() 58 | {'total': {}, 59 | 'consumer': {'prefetch_count': 80, 60 | 'broker': {'transport_options': {}, 61 | 'login_method': 'AMQPLAIN', 62 | 'hostname': '127.0.0.1', 63 | 'userid': 'guest', 64 | 'insist': False, 65 | 'connect_timeout': 4, 66 | 'ssl': False, 67 | 'virtual_host': '/', 68 | 'port': 5672, 69 | 'transport': 'amqp'}}, 70 | 'pool': {'timeouts': [None, None], 71 | 'processes': [76003], 72 | 'max-concurrency': 1, 73 | 'max-tasks-per-child': None, 74 | 'put-guarded-by-semaphore': True}, 75 | 'autoscaler': {'current': 1, 'max': 1, 'min': 1, 'qty': 0}} 76 | 77 | Consumers 78 | ~~~~~~~~~ 79 | 80 | >>> instance.consumers.add(my_queue) 81 | {'ok': 'ok'} 82 | 83 | >>> instance_consumers.delete(my_queue) 84 | {'ok': 'ok'} 85 | 86 | >>> instance.consumers 87 | #... consumers with full declarations ... 88 | 89 | 90 | Deleting 91 | ~~~~~~~~ 92 | 93 | This will delete the queue and eventually force all worker instances 94 | to stop consuming from it:: 95 | 96 | >>> my_queue.delete() 97 | 98 | 99 | This will shutdown and delete an instance:: 100 | 101 | >>> i.delete() 102 | 103 | 104 | """ 105 | 106 | from __future__ import absolute_import 107 | 108 | from dictshield import fields 109 | 110 | from . import base 111 | from .base import Path 112 | from cyme.utils import cached_property 113 | from cyme.utils.dictshield import ListField 114 | 115 | # XXX `requests` does not currently seem to support using the 116 | # data argument with PUT requests. 117 | 118 | 119 | class Instance(base.Model): 120 | name = fields.StringField(max_length=200) 121 | broker = fields.StringField(max_length=200) 122 | pool = fields.StringField(max_length=200) 123 | min_concurrency = fields.IntField() 124 | max_concurrency = fields.IntField() 125 | is_enabled = fields.BooleanField() 126 | queue_names = ListField(fields.StringField(max_length=200)) 127 | arguments = fields.StringField(max_length=200) 128 | extra_config = fields.StringField(max_length=200) 129 | 130 | def __repr__(self): 131 | return '' % (self.name, ) 132 | 133 | 134 | class Queue(base.Model): 135 | name = fields.StringField(max_length=200, required=True) 136 | exchange = fields.StringField(max_length=200, required=False) 137 | exchange_type = fields.StringField(max_length=200, required=False) 138 | routing_key = fields.StringField(max_length=200, required=False) 139 | options = fields.StringField(max_length=None) 140 | 141 | def __repr__(self): 142 | return '' % (self.name, ) 143 | 144 | 145 | class Client(base.Client): 146 | app = 'cyme' 147 | 148 | class Instances(base.Section): 149 | 150 | class Model(Instance): 151 | 152 | class Consumers(base.Section): 153 | 154 | def __init__(self, client, name): 155 | base.Section.__init__(self, client) 156 | self.path = self.client.path / name / 'queues' 157 | 158 | def create_model(self, data, *args, **kwargs): 159 | return data 160 | 161 | def __init__(self, *args, **kwargs): 162 | base.Model.__init__(self, *args, **kwargs) 163 | self.consumers = self.Consumers(self.parent, name=self.name) 164 | self.path = self.parent.path / self.name 165 | 166 | def stats(self): 167 | return self.parent.stats(self.name) 168 | 169 | def autoscale(self, max=None, min=None): 170 | return self.parent.autoscale(self.name, max=max, min=min) 171 | 172 | class LazyQueues(object): 173 | 174 | def __init__(self, instance): 175 | self.instance = instance 176 | 177 | def __iter__(self): 178 | return self.instance._get_queues() 179 | 180 | def __getitem__(self, index): 181 | return self._eval[index] 182 | 183 | def __contains__(self, queue): 184 | if isinstance(queue, Queue): 185 | queue = queue.name 186 | return queue in self.instance.queue_names 187 | 188 | def __len__(self): 189 | return len(self._eval) 190 | 191 | def __repr__(self): 192 | return repr(self._eval) 193 | 194 | @cached_property 195 | def _eval(self): 196 | return list(iter(self)) 197 | 198 | def _get_queues(self): 199 | return (self.parent.client.queues.get(name) 200 | for name in self.queue_names) 201 | 202 | @property 203 | def queues(self): 204 | return self.LazyQueues(self) 205 | 206 | def add(self, name=None, broker=None, arguments=None, 207 | config=None, nowait=False): 208 | # name is optional for instances 209 | return base.Section.add(self, name, nowait, 210 | broker=broker, 211 | arguments=arguments, 212 | extra_config=config) 213 | 214 | def stats(self, name): 215 | return self.GET(self.path / name / 'stats') 216 | 217 | def autoscale(self, name, max=None, min=None): 218 | return self.POST(self.path / name / 'autoscale', 219 | params={'max': max, 'min': min}) 220 | 221 | def create_model(self, data, *args, **kwargs): 222 | data['queue_names'] = data.pop('queues', None) 223 | return base.Section.create_model(self, data, *args, **kwargs) 224 | 225 | class Queues(base.Section): 226 | 227 | class Model(Queue): 228 | pass 229 | 230 | def add(self, name, exchange=None, exchange_type=None, 231 | routing_key=None, nowait=False, **options): 232 | options = self.serialize(options) if options else None 233 | return base.Section.add(self, name, nowait, 234 | exchange=exchange, 235 | exchange_type=exchange_type, 236 | routing_key=routing_key, 237 | options=options) 238 | 239 | def __init__(self, url=None, app=None, info=None): 240 | super(Client, self).__init__(url) 241 | self.app = app 242 | self.instances = self.Instances(self) 243 | self.queues = self.Queues(self) 244 | self.info = info or {} 245 | 246 | def add(self, name, broker=None, arguments=None, extra_config=None, 247 | nowait=False): 248 | return self.create_model(name, self.root('POST', 249 | self.maybe_async(name, nowait), 250 | data={'broker': broker, 251 | 'arguments': arguments, 252 | 'extra_config': extra_config})) 253 | 254 | def get(self, name=None): 255 | return self.create_model(name, self.root('GET', name or self.app)) 256 | 257 | def delete(self, name=None): 258 | return self.root('DELETE', name or self.app) 259 | 260 | @property 261 | def branches(self): 262 | return self.root('GET', 'branches') 263 | 264 | def branch_info(self, id): 265 | return self.root('GET', Path('branches') / id) 266 | 267 | def all(self): 268 | return self.root('GET') 269 | 270 | def build_url(self, path): 271 | url = self.url 272 | if self.app: 273 | url += '/' + self.app 274 | return url + str(path) 275 | 276 | def create_model(self, name, info): 277 | return self.clone(app=name, info=base.AttributeDict(info)) 278 | 279 | def clone(self, app=None, info=None): 280 | return self.__class__(url=self.url, app=app, info=info) 281 | 282 | def __repr__(self): 283 | url = self.build_url('') 284 | if self.app: 285 | return '' % url 286 | return '' % url 287 | -------------------------------------------------------------------------------- /cyme/client/base.py: -------------------------------------------------------------------------------- 1 | """cyme.client.base""" 2 | 3 | from __future__ import absolute_import 4 | 5 | import anyjson 6 | import requests 7 | 8 | from urllib import quote 9 | 10 | from celery.datastructures import AttributeDict 11 | from dictshield.document import Document 12 | 13 | from cyme import __version__, DEBUG 14 | from cyme.utils import cached_property 15 | 16 | 17 | class Path(object): 18 | 19 | def __init__(self, s=None, stack=None): 20 | self.stack = (stack or []) + (s.stack if isinstance(s, Path) else [s]) 21 | 22 | def __str__(self, s='/'): 23 | return (s + s.join(map(quote, filter(None, self.stack))).strip(s) + s) 24 | 25 | def __div__(self, other): 26 | return Path(other, self.stack) 27 | 28 | 29 | class Base(object): 30 | 31 | def serialize(self, obj): 32 | return anyjson.serialize(obj) 33 | 34 | def deserialize(self, text): 35 | return anyjson.deserialize(text) 36 | 37 | def __getitem__(self, key): 38 | return self.get(key) 39 | 40 | def __delitem__(self, key): 41 | return self.delete(key) 42 | 43 | def keys(self): 44 | return self.all() 45 | 46 | def __iter__(self): 47 | return iter(self.keys()) 48 | 49 | def maybe_async(self, name, nowait): 50 | if nowait: 51 | return Path('!') / name 52 | return name 53 | 54 | 55 | class Model(Document): 56 | 57 | def __init__(self, parent, *args, **kwargs): 58 | self.parent = parent 59 | super(Model, self).__init__(**self._prepare_kwargs(*args, **kwargs)) 60 | 61 | def _prepare_kwargs(self, *args, **kwargs): 62 | if len(args) == 1 and isinstance(args[0], dict): 63 | kwargs.update(args[0]) 64 | return kwargs 65 | 66 | def delete(self, nowait=False): 67 | return self.parent.delete(self.name, nowait=nowait) 68 | 69 | 70 | class Section(Base): 71 | Model = Model 72 | name = None 73 | path = None 74 | proxy = ['GET', 'POST', 'PUT', 'DELETE'] 75 | 76 | def __init__(self, client): 77 | self.client = client 78 | if self.name is None: 79 | self.name = self.__class__.__name__.lower() 80 | for attr in self.proxy: 81 | setattr(self, attr, getattr(self.client, attr)) 82 | self.path = Path(self.name) if self.path is None else self.path 83 | 84 | def all_names(self): 85 | return self.GET(self.path) 86 | 87 | def all(self): 88 | return (self.get(name) for name in self.all_names()) 89 | 90 | def get(self, name): 91 | return self.GET(self.path / name, type=self.create_model) 92 | 93 | def add(self, name, nowait=False, **data): 94 | if isinstance(name, self.Model): 95 | name = name.name 96 | return self.POST(self.maybe_async(name, nowait), 97 | type=self.create_model, data=data) 98 | 99 | def delete(self, name, nowait=False): 100 | if isinstance(name, self.Model): 101 | name = name.name 102 | return self.DELETE(self.maybe_async(name, nowait)) 103 | 104 | def create_model(self, *args, **kwargs): 105 | model = self.Model(self, 106 | **self.Model(self, *args, **kwargs).to_python()) 107 | model.validate() 108 | return model 109 | 110 | def maybe_async(self, name, nowait): 111 | if nowait: 112 | return self.path / '!' / name 113 | return self.path / name 114 | 115 | def __repr__(self): 116 | return repr(list(self.all())) 117 | 118 | 119 | class Client(Base): 120 | default_url = 'http://127.0.0.1:8000' 121 | 122 | def __init__(self, url=None): 123 | self.url = url.rstrip('/') if url else self.default_url 124 | 125 | def GET(self, path, params=None, type=None): 126 | return self.request('GET', path, params, None, type) 127 | 128 | def PUT(self, path, params=None, data=None, type=None): 129 | return self.request('PUT', path, params, data, type) 130 | 131 | def POST(self, path, params=None, data=None, type=None): 132 | return self.request('POST', path, params, data, type) 133 | 134 | def DELETE(self, path, params=None, data=None, type=None): 135 | return self.request('DELETE', path, params, data, type) 136 | 137 | def request(self, method, path, params=None, data=None, type=None): 138 | return self._request(method, self.build_url(path), params, data, type) 139 | 140 | def _prepare(self, d): 141 | if d: 142 | return dict((key, value if value is not None else '') 143 | for key, value in d.iteritems()) 144 | 145 | def _request(self, method, url, params=None, data=None, type=None): 146 | data = self._prepare(data) 147 | params = self._prepare(params) 148 | if DEBUG: 149 | print(' %s %r data=%r params=%r' % (method, url, # noqa+ 150 | data, params)) 151 | type = type or AttributeDict 152 | r = requests.request(method, str(url), 153 | headers=self.headers, 154 | params=params, data=data) 155 | data = None 156 | if DEBUG: 157 | print(' %r' % (r.text, )) # noqa+ 158 | if r.ok: 159 | ret = self.deserialize(r.text) 160 | if isinstance(ret, dict): 161 | return type(ret) 162 | return ret 163 | r.raise_for_status() 164 | 165 | def root(self, method, path=None, params=None, data=None): 166 | return self._request(method, 167 | self.url + str(Path(path) if path else ''), 168 | params, data) 169 | 170 | def __repr__(self): 171 | return '' % (self.url, ) 172 | 173 | @cached_property 174 | def headers(self): 175 | return {'Accept': 'application/json', 176 | 'User-Agent': 'cyme-client:py %r' % (__version__, )} 177 | -------------------------------------------------------------------------------- /cyme/conf.py: -------------------------------------------------------------------------------- 1 | """cyme.conf""" 2 | 3 | from __future__ import absolute_import 4 | 5 | from django.conf import settings 6 | 7 | from .utils import Path 8 | 9 | CYME_INSTANCE_DIR = Path(getattr(settings, 10 | 'CYME_INSTANCE_DIR', 'instances')).absolute() 11 | CYME_DEFAULT_POOL = getattr(settings, 'CYME_DEFAULT_POOL', 'processes') 12 | -------------------------------------------------------------------------------- /cyme/management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/celery/cyme/c49047318c56f6674bd6d6ea6673b48be4eb7886/cyme/management/__init__.py -------------------------------------------------------------------------------- /cyme/management/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/celery/cyme/c49047318c56f6674bd6d6ea6673b48be4eb7886/cyme/management/commands/__init__.py -------------------------------------------------------------------------------- /cyme/management/commands/base.py: -------------------------------------------------------------------------------- 1 | """cyme.management.commands.base""" 2 | 3 | from __future__ import absolute_import 4 | 5 | import logging 6 | import os 7 | import sys 8 | 9 | from optparse import make_option as Option # noqa 10 | 11 | from celery.utils import get_cls_by_name, instantiate 12 | from django.conf import settings 13 | from djcelery.management.base import CeleryCommand 14 | from kombu.log import LOG_LEVELS 15 | 16 | from cyme import __version__ 17 | from cyme.utils import (cached_property, setup_logging, 18 | redirect_stdouts_to_logger) 19 | 20 | 21 | def die(msg, exitcode=1): 22 | sys.stderr.write('Error: %s\n' % (msg, )) 23 | sys.exit(exitcode) 24 | 25 | 26 | class CymeCommand(CeleryCommand): 27 | __version__ = __version__ 28 | LOG_LEVELS = LOG_LEVELS 29 | 30 | # see http://code.djangoproject.com/changeset/13319. 31 | stdout, stderr = sys.stdout, sys.stderr 32 | 33 | default_detach_pidfile = None 34 | default_detach_logfile = None 35 | 36 | def __init__(self, env=None, *args, **kwargs): 37 | if env is None: 38 | env = instantiate('cyme.bin.base.Env') 39 | self.setup_default_env(env) 40 | self.env = env 41 | super(CymeCommand, self).__init__(*args, **kwargs) 42 | 43 | def setup_default_env(self, env): 44 | pass 45 | 46 | def get_version(self): 47 | return 'cyme v%s' % (self.__version__, ) 48 | 49 | def enter_instance_dir(self): 50 | self.instance_dir.mkdir(parents=True) 51 | self.instance_dir.chdir() 52 | 53 | def install_cry_handler(self): 54 | from celery.apps.worker import install_cry_handler 55 | return install_cry_handler(logging.getLogger()) 56 | 57 | def install_rdb_handler(self): 58 | from celery.apps.worker import install_rdb_handler 59 | return install_rdb_handler(envvar='CYME_RDBSIG') 60 | 61 | def redirect_stdouts_to_logger(self, loglevel='INFO', logfile=None, 62 | redirect_level='WARNING'): 63 | return redirect_stdouts_to_logger(loglevel, logfile, redirect_level) 64 | 65 | def setup_logging(self, loglevel='WARNING', logfile=None, **kwargs): 66 | return setup_logging(loglevel, logfile) 67 | 68 | def prepare_options(self, broker=None, loglevel=None, logfile=None, 69 | pidfile=None, detach=None, instance_dir=None, **kwargs): 70 | if detach: 71 | logfile = logfile or self.default_detach_logfile 72 | pidfile = pidfile or self.default_detach_pidfilE 73 | if broker: 74 | settings.BROKER_HOST = broker 75 | if instance_dir: 76 | settings.CYME_INSTANCE_DIR = instance_dir 77 | if pidfile and not os.path.isabs(pidfile): 78 | pidfile = os.path.join(self.instance_dir, pidfile) 79 | if logfile and not os.path.isabs(logfile): 80 | logfile = os.path.join(self.instance_dir, logfile) 81 | if not isinstance(loglevel, int): 82 | try: 83 | loglevel = self.LOG_LEVELS[loglevel.upper()] 84 | except KeyError: 85 | self.die('Unknown level %r. Please use one of %s.' % ( 86 | loglevel, '|'.join(l for l in self.LOG_LEVELS.keys() 87 | if isinstance(l, basestring)))) 88 | return dict(kwargs, loglevel=loglevel, detach=detach, 89 | logfile=logfile, pidfile=pidfile) 90 | 91 | def print_help(self, prog_name=None, subcommand=None): 92 | return super(CymeCommand, self).print_help('cyme', '') 93 | 94 | @cached_property 95 | def instance_dir(self): 96 | return get_cls_by_name('cyme.conf.CYME_INSTANCE_DIR') 97 | -------------------------------------------------------------------------------- /cyme/management/commands/cyme.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | .. program:: cyme 4 | 5 | ``cyme`` 6 | ======== 7 | 8 | Cyme management utility 9 | 10 | Options 11 | ------- 12 | 13 | .. cmdoption:: -a, --app 14 | 15 | Application to use. Required for all operations except for 16 | when creating, deleting or listing apps. 17 | 18 | .. cmdoption:: -n, --nowait 19 | 20 | Don't want for operations to complete (async). 21 | 22 | .. cmdoption:: -F, --format 23 | 24 | Output format: pretty (default) or json. 25 | 26 | .. cmdoption:: -L, --local 27 | 28 | Flag that if set means that operations will be performed 29 | locally for the current branch only. 30 | This means the instance directory must be properly set. 31 | 32 | .. cmdoption:: -l, --loglevel 33 | 34 | Set custom log level. One of DEBUG/INFO/WARNING/ERROR/CRITICAL. 35 | Default is INFO. 36 | 37 | .. cmdoption:: -f, --logfile 38 | 39 | Set custom logfile path. Default is :file:`` 40 | 41 | .. cmdoption:: -D, --instance-dir 42 | 43 | Custom instance directory (default is :file:`instances/`) 44 | Must be readable by the current user. 45 | 46 | This needs to be properly specified if the :option:`-L` option 47 | is used. 48 | 49 | .. cmdoption:: -b, --broker 50 | 51 | Broker to use for a local `branches` request. 52 | 53 | """ 54 | 55 | from __future__ import absolute_import 56 | 57 | import anyjson 58 | import os 59 | import pprint 60 | 61 | from functools import partial 62 | from inspect import getargspec 63 | 64 | 65 | from celery import current_app as celery 66 | from cyme.client import Client 67 | from cyme.client.base import Model 68 | from cyme.utils import cached_property, instantiate 69 | 70 | from .base import CymeCommand, Option, die 71 | 72 | 73 | try: 74 | import json as _json # Python 2.6+ 75 | json_pretty = partial(_json.dumps, indent=4) 76 | except ImportError: 77 | json_pretty = anyjson.serialize # noqa 78 | 79 | 80 | class I(object): 81 | 82 | def __init__(self, app=None, format=None, nowait=False, 83 | url=None, **kwargs): 84 | self.url = url 85 | self.app = app 86 | self.format = format or 'pretty' 87 | self.nowait = nowait 88 | self.actions = { 89 | 'branches': { 90 | 'all': self.all_branches, 91 | }, 92 | 'apps': { 93 | 'all': self.all_apps, 94 | 'get': self.get_app, 95 | 'add': self.add_app, 96 | 'delete': self.delete_app}, 97 | 'instances': { 98 | 'all': self.all_instances, 99 | 'get': self.get_instance, 100 | 'add': self.add_instance, 101 | 'delete': self.delete_instance, 102 | 'stats': self.instance_stats, 103 | 'autoscale': self.instance_autoscale}, 104 | 'queues': { 105 | 'all': self.all_queues, 106 | 'get': self.get_queue, 107 | 'add': self.add_queue, 108 | 'delete': self.delete_queue}, 109 | 'consumers': { 110 | 'all': self.all_consumers, 111 | 'add': self.add_consumer, 112 | 'delete': self.delete_consumer}, 113 | } 114 | self.needs_app = ('instances', 'queues') 115 | self.formats = {'jsonp': json_pretty, 116 | 'json': anyjson.serialize, 117 | 'pprint': pprint.pformat} 118 | 119 | def getsig(self, fun, opt_args=None): 120 | spec = getargspec(fun) 121 | args = spec.args[:-len(spec.defaults) if spec.defaults else None] 122 | if args[0] == 'self': 123 | args = args[1:] 124 | if spec.defaults: 125 | opt_args = dict(zip(spec.args[len(spec.defaults):], spec.defaults)) 126 | return len(args), args, opt_args 127 | 128 | def _ni(self, *args, **kwargs): 129 | raise NotImplementedError('subclass responsibility') 130 | all_apps = get_app = add_app = delete_app = \ 131 | all_instances = get_instances = add_instance = delete_instance = \ 132 | instance_stats = instance_autoscale = \ 133 | all_queues = get_queue = add_queue = delete_queue = \ 134 | all_consumers = add_consumer = delete_consumer = _ni 135 | 136 | def DISPATCH(self, fqdn, *args): 137 | type, _, action = fqdn.partition('.') 138 | if not self.app and type in self.needs_app: 139 | die('Need to specify --app') 140 | try: 141 | handler = self.actions[type][action or 'all'] 142 | except KeyError: 143 | if type: 144 | die('No action %r for type %r' % (action, type)) 145 | die('Missing type') 146 | try: 147 | response = handler(*args) 148 | except TypeError: 149 | raise 150 | arity, args, optargs = self.getsig(handler) 151 | die('%s.%s requires %s argument%s: %s %s' % ( 152 | type, action, arity, 's' if arity > 1 else '', 153 | ' '.join(args), self.format_optargs(optargs))) 154 | return self.format_response(self.prepare_response(response)) 155 | 156 | def format_optargs(self, optargs): 157 | if optargs: 158 | return ' '.join('[%s]' % (k, ) for k in optargs.keys()) 159 | return '' 160 | 161 | def format_response(self, ret): 162 | return self.formats[self.format](ret) 163 | 164 | def prepare_response(self, ret): 165 | return ret 166 | 167 | 168 | class WebI(I): 169 | 170 | def all_branches(self): 171 | return list(self.client.branches) 172 | 173 | def all_apps(self): 174 | return list(self.client.all()) 175 | 176 | def get_app(self, name): 177 | return self.client.get(name).info 178 | 179 | def add_app(self, name, broker=None, arguments=None, extra_config=None): 180 | return self.client.add(name, nowait=self.nowait, 181 | broker=broker, arguments=arguments, 182 | extra_config=extra_config).info 183 | 184 | def delete_app(self, name): 185 | return self.client.delete(name, nowait=self.nowait) 186 | 187 | def all_instances(self): 188 | return list(self.client.instances.all()) 189 | 190 | def get_instance(self, name): 191 | return self.client.instances.get(name) 192 | 193 | def add_instance(self, name=None, broker=None, arguments=None, 194 | extra_config=None): 195 | return self.client.instances.add(name=name, broker=broker, 196 | arguments=arguments, 197 | extra_config=extra_config, 198 | nowait=self.nowait) 199 | 200 | def delete_instance(self, name): 201 | return self.client.instances.get(name).delete(nowait=self.nowait) 202 | 203 | def instance_stats(self, name): 204 | return self.client.instances.get(name).stats() 205 | 206 | def instance_autoscale(self, name, max=None, min=None): 207 | return self.client.instances.get(name).autoscale(max=max, min=min) 208 | 209 | def all_consumers(self, instance_name): 210 | return list(self.client.instances.get(instance_name).consumers) 211 | 212 | def add_consumer(self, instance_name, queue_name): 213 | return self.client.instances.get(instance_name)\ 214 | .consumers.add(queue_name, nowait=self.nowait) 215 | 216 | def delete_consumer(self, instance_name, queue_name): 217 | return self.client.instances.get(instance_name)\ 218 | .consumers.delete(queue_name, nowait=self.nowait) 219 | 220 | def all_queues(self): 221 | return list(self.client.queues.all()) 222 | 223 | def get_queue(self, name): 224 | return self.client.queues.get(name) 225 | 226 | def add_queue(self, name, exchange=None, exchange_type=None, 227 | routing_key=None, options=None): 228 | options = anyjson.deserialize(options) if options else {} 229 | return self.client.queues.add(name, exchange=exchange, 230 | exchange_type=exchange_type, 231 | routing_key=routing_key, 232 | nowait=self.nowait, 233 | **options) 234 | 235 | def delete_queue(self, name): 236 | return self.client.queues.delete(name, nowait=self.nowait) 237 | 238 | def _part(self, p): 239 | if isinstance(p, Model): 240 | return dict((k, v) for k, v in p.to_python().iteritems() 241 | if not k.startswith('_')) 242 | return p 243 | 244 | def prepare_response(self, ret): 245 | if isinstance(ret, (list, tuple)): 246 | return map(self._part, ret) 247 | return self._part(ret) 248 | 249 | @cached_property 250 | def client(self): 251 | client = Client(self.url) 252 | if self.app: 253 | return client.get(self.app) 254 | return client 255 | 256 | 257 | class LocalI(I): 258 | 259 | def __init__(self, *args, **kwargs): 260 | super(LocalI, self).__init__(*args, **kwargs) 261 | self.broker = kwargs.get('broker') 262 | self.limit = kwargs.get('limit') 263 | from cyme.branch.controller import apps, instances, queues 264 | self.get_app = apps.get 265 | self.apps = apps.state 266 | self.instances = instances.state 267 | self.queues = queues.state 268 | 269 | def all_branches(self): 270 | from cyme.branch.controller import Branch 271 | args = [self.broker] if self.broker else [] 272 | conn = celery.broker_connection(*args) 273 | return Branch(connection=conn).all(limit=self.limit) 274 | 275 | def all_apps(self): 276 | return [app.as_dict() for app in self.apps.objects.all()] 277 | 278 | def get_app(self): 279 | return self.apps.objects.get(app=self.app) 280 | 281 | def add_app(self, name, broker=None, arguments=None, extra_config=None): 282 | return self.apps.add(name, broker=broker, arguments=arguments, 283 | extra_config=extra_config) 284 | 285 | def delete_app(self, name): 286 | self.apps.delete(name) 287 | return {'ok': 'ok'} 288 | 289 | def all_instances(self): 290 | return [instance.as_dict() 291 | for instance in self.instances.objects.filter( 292 | app=self.get_app(self.app))] 293 | 294 | def get_instance(self, name): 295 | return self.instances.get(name, app=self.app) 296 | 297 | def add_instance(self, name=None, broker=None, arguments=None, 298 | extra_config=None): 299 | return self.instances.add(name, broker=broker, app=self.app, 300 | arguments=arguments, 301 | extra_config=extra_config) 302 | 303 | def delete_instance(self, name): 304 | return {'ok': self.instances.remove(name)} 305 | 306 | def instance_stats(self, name): 307 | return self.instances.stats(name) 308 | 309 | def _get_instance(self, name): 310 | return self.instances.objects.get(name=name) 311 | 312 | def instance_autoscale(self, name, max=None, min=None): 313 | return dict(zip(['max', 'min'], 314 | self._get_instance(name=name)._update_autoscale(max, min))) 315 | 316 | def all_consumers(self, instance_name): 317 | return self._get_instance(name=instance_name).consuming_from() 318 | 319 | def add_consumer(self, instance_name, queue_name): 320 | self._get_instance(name=instance_name)\ 321 | .add_queue_eventually(queue_name) 322 | return {'ok': 'ok'} 323 | 324 | def delete_consumer(self, instance_name, queue_name): 325 | self._get_instance(name=instance_name)\ 326 | .remove_queue_eventually(queue_name) 327 | return {'ok': 'ok'} 328 | 329 | def all_queues(self): 330 | return [queue.as_dict() 331 | for queue in self.queues.objects.all()] 332 | 333 | def get_queue(self, name): 334 | return self.queues.get(name) 335 | 336 | def add_queue(self, name, exchange=None, exchange_type=None, 337 | routing_key=None, options=None): 338 | options = anyjson.deserialize(options) if options else {} 339 | return self.queues.add(name, exchange=exchange, 340 | exchange_type=exchange_type, 341 | routing_key=routing_key, 342 | **options) 343 | 344 | def delete_queue(self, name): 345 | self.instances.remove_queue_from_all(name) 346 | return {'ok': self.queues.delete(name)} 347 | 348 | 349 | class Command(CymeCommand): 350 | name = 'cyme' 351 | args = """type command [args] 352 | E.g.: 353 | cyme apps 354 | cyme apps.add [broker URL] [arguments] [extra config] 355 | cyme apps.[get|delete] 356 | 357 | cyme -a instances 358 | cyme -a instances.add [name] [broker URL] [arguments] [extra config] 359 | cyme -a instances.[get|delete|stats] 360 | cyme -a instances.autoscale [max] [min] 361 | 362 | cyme -a queues 363 | cyme -a queues.add [exchange] [type] [rkey] [opts] 364 | cyme -a queues.[get|delete] 365 | 366 | cyme start-all 367 | cyme shutdown-all 368 | 369 | cyme createsuperuser 370 | 371 | cyme shell 372 | """ 373 | option_list = tuple(CymeCommand().option_list) + ( 374 | Option('-a', '--app', 375 | default=None, action='store', dest='app', 376 | help='application to use'), 377 | Option('-n', '--nowait', 378 | default=False, action='store_true', dest='nowait', 379 | help="Don't want for operations to complete (async)."), 380 | Option('-F', '--format', 381 | default='jsonp', action='store', dest='format', 382 | help='Output format: jsonp (default) json or pprint'), 383 | Option('-L', '--local', 384 | default=False, action='store_true', dest='local', 385 | help='Perform operations locally for this branch only.'), 386 | Option('-l', '--loglevel', 387 | default='WARNING', action='store', dest='loglevel', 388 | help='Choose between DEBUG/INFO/WARNING/ERROR/CRITICAL'), 389 | Option('-D', '--instance-dir', 390 | default=None, action='store', dest='instance_dir', 391 | help='Custom instance dir. Default is instances/'), 392 | Option('-u', '--url', 393 | default='http://localhost:8000', dest='url', 394 | help='Custom URL'), 395 | Option('-b', '--broker', 396 | default='None', dest='broker', 397 | help='Broker to use for a local branches request'), 398 | ) 399 | 400 | help = 'Cyme management utility' 401 | 402 | def handle(self, *args, **kwargs): 403 | local = kwargs.pop('local', False) 404 | kwargs = self.prepare_options(**kwargs) 405 | self.commands = {'shell': self.drop_into_shell, 406 | 'sh': self.drop_into_shell, 407 | 'start-all': self.start_all, 408 | 'restart-all': self.restart_all, 409 | 'shutdown-all': self.shutdown_all, 410 | 'createsuperuser': self.create_superuser} 411 | self.setup_logging(**kwargs) 412 | args = list(args) 413 | self.enter_instance_dir() 414 | if local: 415 | self.env.syncdb() 416 | if args: 417 | if args[0] in self.commands: 418 | return self.commands[args[0]]() 419 | print((LocalI if local else WebI)(**kwargs).DISPATCH(*args)) 420 | else: 421 | self.print_help() 422 | 423 | def start_all(self): 424 | self.status.start_all() 425 | 426 | def restart_all(self): 427 | self.status.restart_all() 428 | 429 | def shutdown_all(self): 430 | self.status.shutdown_all() 431 | 432 | def drop_into_shell(self): 433 | from cyme.utils import setup_logging 434 | if os.environ.get('CYME_LOG_DEBUG'): 435 | setup_logging('DEBUG') 436 | self.env.management.call_command('shell') 437 | 438 | def create_superuser(self): 439 | self.env.management.call_command('createsuperuser') 440 | 441 | @cached_property 442 | def status(self): 443 | return instantiate(self, 'cyme.status.Status') 444 | -------------------------------------------------------------------------------- /cyme/management/commands/cyme_branch.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | .. program:: cyme-branch 4 | 5 | ``cyme-branch`` 6 | =============== 7 | 8 | Starts the cyme branch service. 9 | 10 | Options 11 | ------- 12 | 13 | .. cmdoption:: -i, --id 14 | 15 | Set branch id, if not provided one will be automatically generated. 16 | 17 | .. cmdoption:: --without-httpd 18 | 19 | Disable the HTTP server thread. 20 | 21 | .. cmdoption:: -l, --loglevel 22 | 23 | Set custom log level. One of DEBUG/INFO/WARNING/ERROR/CRITICAL. 24 | Default is INFO. 25 | 26 | .. cmdoption:: -f, --logfile 27 | 28 | Set custom log file path. Default is :file:`` 29 | 30 | .. cmdoption:: -D, --instance-dir 31 | 32 | Custom instance directory (default is :file:`instances/``) 33 | Must be writeable by the user cyme-branch runs as. 34 | 35 | .. cmdoption:: -C, --numc 36 | 37 | Number of controllers to start, to handle simultaneous 38 | requests. Each controller requires one AMQP connection. 39 | Default is 2. 40 | 41 | .. cmdoption:: --sup-interval 42 | 43 | Supervisor schedule Interval in seconds. Default is 5. 44 | 45 | """ 46 | 47 | from __future__ import absolute_import 48 | from __future__ import with_statement 49 | 50 | import atexit 51 | import os 52 | 53 | from importlib import import_module 54 | 55 | from celery import current_app as celery 56 | from celery.bin.base import daemon_options 57 | from celery.platforms import (create_pidlock, detached, 58 | signals, set_process_title) 59 | from celery.utils import instantiate 60 | from cell.utils import cached_property, shortuuid 61 | 62 | from .base import CymeCommand, Option 63 | 64 | BANNER = """ 65 | -------------- cyme@%(id)s v%(version)s 66 | ---- **** ----- 67 | --- * *** * -- [Configuration] 68 | -- * - **** --- . url: http://%(addr)s:%(port)s 69 | - ** ---------- . broker: %(broker)s 70 | - ** ---------- . logfile: %(logfile)s@%(loglevel)s 71 | - ** ---------- . sup: interval=%(sup.interval)s 72 | - ** ---------- . presence: interval=%(presence.interval)s 73 | - *** --- * --- . controllers: #%(controllers)s 74 | -- ******* ---- . instancedir: %(instance_dir)s 75 | --- ***** ----- 76 | -------------- http://celeryproject.org 77 | """ 78 | 79 | 80 | class Command(CymeCommand): 81 | branch_cls = 'cyme.branch.Branch' 82 | default_detach_logfile = 'b ranch.log' 83 | default_detach_pidfile = 'branch.pid' 84 | name = 'cyme-branch' 85 | args = '[optional port number, or ipaddr:port]' 86 | help = 'Starts a cyme branch' 87 | option_list = tuple(CymeCommand().option_list) + ( 88 | Option('--broker', '-b', 89 | default=None, action='store', dest='broker', 90 | help="""Broker URL to use for the cyme message bus.\ 91 | Default is amqp://guest:guest@localhost:5672//"""), 92 | Option('--detach', 93 | default=False, action='store_true', dest='detach', 94 | help='Detach and run in the background.'), 95 | Option('-i', '--id', 96 | default=None, action='store', dest='id', 97 | help='Set explicit branch id.'), 98 | Option('-X', '--no-interaction', 99 | default=False, action='store_true', dest='no_interaction', 100 | help="Don't ask questions"), 101 | Option('--without-httpd', 102 | default=False, action='store_true', dest='without_httpd', 103 | help='Disable HTTP server'), 104 | Option('-l', '--loglevel', 105 | default='WARNING', action='store', dest='loglevel', 106 | help='Choose between DEBUG/INFO/WARNING/ERROR/CRITICAL'), 107 | Option('-D', '--instance-dir', 108 | default=None, action='store', dest='instance_dir', 109 | help='Custom instance dir. Default is instances/'), 110 | Option('-C', '--numc', 111 | default=2, action='store', type='int', dest='numc', 112 | help='Number of controllers to start. Default is 2'), 113 | Option('--sup-interval', 114 | default=60, action='store', type='int', dest='sup_interval', 115 | help='Supervisor schedule interval. Default is every minute.'), 116 | ) + daemon_options(default_detach_pidfile) 117 | 118 | _startup_pbar = None 119 | _shutdown_pbar = None 120 | 121 | def handle(self, *args, **kwargs): 122 | kwargs = self.prepare_options(**kwargs) 123 | self.loglevel = kwargs.get('loglevel') 124 | self.logfile = kwargs.get('logfile') 125 | self.enter_instance_dir() 126 | self.env.syncdb(interactive=False) 127 | self.install_cry_handler() 128 | self.install_rdb_handler() 129 | self.colored = celery.log.colored(kwargs.get('logfile')) 130 | self.branch = instantiate(self.branch_cls, *args, 131 | colored=self.colored, **kwargs) 132 | self.connect_signals() 133 | print(str(self.colored.cyan(self.banner()))) 134 | 135 | self.detached = kwargs.get('detach', False) 136 | return (self._detach if self.detached else self._start)(**kwargs) 137 | 138 | def setup_default_env(self, env): 139 | env.setup_eventlet() 140 | env.setup_pool_limit() 141 | 142 | def stop(self): 143 | self.set_process_title('shutdown...') 144 | 145 | def on_branch_ready(self, sender=None, **kwargs): 146 | if self._startup_pbar: 147 | self._startup_pbar.finish() 148 | self._startup_pbar = None 149 | pid = os.getpid() 150 | self.set_process_title('ready') 151 | if not self.detached and \ 152 | not self.branch.is_enabled_for('INFO'): 153 | print('(%s) branch ready' % (pid, )) 154 | sender.info('[READY] (%s)' % (pid, )) 155 | 156 | def on_branch_shutdown(self, sender=None, **kwargs): 157 | if self._shutdown_pbar: 158 | self._shutdown_pbar.finish() 159 | self._shutdown_pbar = None 160 | 161 | def _detach(self, logfile=None, pidfile=None, uid=None, gid=None, 162 | umask=None, working_directory=None, **kwargs): 163 | print('detaching... [pidfile=%s logfile=%s]' % (pidfile, logfile)) 164 | with detached(logfile, pidfile, uid, gid, umask, working_directory): 165 | return self._start(pidfile=pidfile) 166 | 167 | def _start(self, pidfile=None, **kwargs): 168 | self.setup_logging(logfile=self.logfile, loglevel=self.loglevel) 169 | self.set_process_title('boot') 170 | self.install_signal_handlers() 171 | if pidfile: 172 | pidlock = create_pidlock(pidfile).acquire() 173 | atexit.register(pidlock.release) 174 | try: 175 | return self.branch.start().wait() 176 | except SystemExit: 177 | self.branch.stop() 178 | 179 | def banner(self): 180 | branch = self.branch 181 | addr, port = branch.addrport 182 | con = branch.controllers 183 | try: 184 | pres_interval = con[0].thread.presence.interval 185 | except AttributeError: 186 | pres_interval = '(disabled)' 187 | sup = branch.supervisor.thread 188 | return BANNER % {'id': branch.id, 189 | 'version': self.__version__, 190 | 'broker': branch.connection.as_uri(), 191 | 'loglevel': self.LOG_LEVELS[branch.loglevel], 192 | 'logfile': branch.logfile or '[stderr]', 193 | 'addr': addr or 'localhost', 194 | 'port': port or 8000, 195 | 'sup.interval': sup.interval, 196 | 'presence.interval': pres_interval, 197 | 'controllers': len(con), 198 | 'instance_dir': self.instance_dir} 199 | 200 | def install_signal_handlers(self): 201 | 202 | def raise_SystemExit(signum, frame): 203 | raise SystemExit() 204 | 205 | for signal in ('TERM', 'INT'): 206 | signals[signal] = raise_SystemExit 207 | 208 | def set_process_title(self, info): 209 | set_process_title('%s#%s' % (self.name, shortuuid(self.branch.id)), 210 | '%s (-D %s)' % (info, self.instance_dir)) 211 | 212 | def repr_controller_id(self, c): 213 | return shortuuid(c) + c[-2:] 214 | 215 | def connect_signals(self): 216 | sigs = self.signals 217 | sigmap = {sigs.branch_startup_request: (self.setup_startup_progress, 218 | self.setup_shutdown_progress), 219 | sigs.branch_ready: (self.on_branch_ready, ), 220 | sigs.branch_shutdown_complete: (self.on_branch_shutdown, )} 221 | for sig, handlers in sigmap.iteritems(): 222 | for handler in handlers: 223 | sig.connect(handler, sender=self.branch) 224 | 225 | def setup_shutdown_progress(self, sender=None, **kwargs): 226 | from cyme.utils import LazyProgressBar 227 | if sender.is_enabled_for('DEBUG'): 228 | return 229 | c = self.colored 230 | sigs = (self.signals.thread_pre_shutdown, 231 | self.signals.thread_pre_join, 232 | self.signals.thread_post_join, 233 | self.signals.thread_post_shutdown) 234 | estimate = (len(sigs) * ((len(sender.components) + 1) * 2) 235 | + sum(c.thread.extra_shutdown_steps 236 | for c in sender.components)) 237 | text = c.white('Shutdown...').embed() 238 | p = self._shutdown_pbar = LazyProgressBar(estimate, text, 239 | c.reset().embed()) 240 | [sig.connect(p.step) for sig in sigs] 241 | 242 | def setup_startup_progress(self, sender=None, **kwargs): 243 | from cyme.utils import LazyProgressBar 244 | if sender.is_enabled_for('INFO'): 245 | return 246 | c = self.colored 247 | tsigs = (self.signals.thread_pre_start, 248 | self.signals.thread_post_start) 249 | osigs = (self.signals.httpd_ready, 250 | self.signals.supervisor_ready, 251 | self.signals.controller_ready, 252 | self.signals.branch_ready) 253 | 254 | estimate = (len(tsigs) + ((len(sender.components) + 10) * 2) 255 | + len(osigs)) 256 | text = c.white('Startup...').embed() 257 | p = self._startup_pbar = LazyProgressBar(estimate, text, 258 | c.reset().embed()) 259 | [sig.connect(p.step) for sig in tsigs + osigs] 260 | 261 | @cached_property 262 | def signals(self): 263 | return import_module('cyme.branch.signals') 264 | -------------------------------------------------------------------------------- /cyme/models/managers.py: -------------------------------------------------------------------------------- 1 | """cyme.managers 2 | 3 | - These are the managers for our models in :mod:`cyme.models`. 4 | 5 | - They are not to be used directly, but accessed through 6 | the ``objects`` attribute of a Model. 7 | 8 | 9 | """ 10 | 11 | from __future__ import absolute_import 12 | 13 | from anyjson import serialize 14 | from celery import current_app as celery 15 | from djcelery.managers import ExtendedManager 16 | 17 | from cyme.utils import cached_property, uuid 18 | 19 | 20 | class BrokerManager(ExtendedManager): 21 | 22 | def get_default(self): 23 | return self.get_or_create(url=self.default_url)[0] 24 | 25 | @property 26 | def default_url(self): 27 | return celery.broker_connection().as_uri() 28 | 29 | 30 | class AppManager(ExtendedManager): 31 | 32 | def from_json(self, name=None, broker=None): 33 | return {'name': name, 'broker': self.get_broker(broker)} 34 | 35 | def recreate(self, name=None, broker=None, arguments=None, 36 | extra_config=None): 37 | d = self.from_json(name, broker) 38 | return self.get_or_create(name=d['name'], 39 | defaults={'broker': d['broker'], 40 | 'arguments': arguments, 41 | 'extra_config': extra_config})[0] 42 | 43 | def instance(self, name=None, broker=None): 44 | return self.model(**self.from_json(name, broker)) 45 | 46 | def get_broker(self, url): 47 | return self.Brokers.get_or_create(url=url)[0] 48 | 49 | def add(self, name=None, broker=None, arguments=None, extra_config=None): 50 | broker = self.get_broker(broker) if broker else None 51 | return self.get_or_create(name=name, defaults={ 52 | 'broker': broker, 53 | 'arguments': arguments, 54 | 'extra_config': extra_config})[0] 55 | 56 | def get_default(self): 57 | return self.get_or_create(name='cyme')[0] 58 | 59 | @cached_property 60 | def Brokers(self): 61 | return self.model.Broker._default_manager 62 | 63 | 64 | class InstanceManager(ExtendedManager): 65 | 66 | def enabled(self): 67 | return self.filter(is_enabled=True) 68 | 69 | def _maybe_queues(self, queues): 70 | if isinstance(queues, basestring): 71 | queues = queues.split(',') 72 | return [(queue.name if isinstance(queue, self.model.Queue) else queue) 73 | for queue in queues] 74 | 75 | def add(self, name=None, queues=None, max_concurrency=1, 76 | min_concurrency=1, broker=None, pool=None, app=None, 77 | arguments=None, extra_config=None): 78 | instance = self.create(name=name or uuid(), 79 | max_concurrency=max_concurrency, 80 | min_concurrency=min_concurrency, 81 | pool=pool, 82 | app=app, 83 | arguments=arguments, 84 | extra_config=extra_config) 85 | needs_save = False 86 | if queues: 87 | instance.queues = self._maybe_queues(queues) 88 | needs_save = True 89 | if broker: 90 | instance._broker = broker 91 | needs_save = True 92 | if needs_save: 93 | instance.save() 94 | return instance 95 | 96 | def _action(self, name, action, *args, **kwargs): 97 | instance = self.get(name=name) 98 | getattr(instance, action)(*args, **kwargs) 99 | return instance 100 | 101 | def remove(self, name): 102 | return self._action(name, 'delete') 103 | 104 | def enable(self, name): 105 | return self._action(name, 'enable') 106 | 107 | def disable(self, name): 108 | return self._action(name, 'disable') 109 | 110 | def remove_queue_from_instances(self, queue, **query): 111 | instances = [] 112 | for instance in self.filter(**query).iterator(): 113 | if queue in instance.queues: 114 | instance.queues.remove(queue) 115 | instance.save() 116 | instances.append(instance) 117 | return instances 118 | 119 | def add_queue_to_instances(self, queue, **query): 120 | instances = [] 121 | for instance in self.filter(**query).iterator(): 122 | instance.queues.add(queue) 123 | instance.save() 124 | instances.append(instance) 125 | return instances 126 | 127 | 128 | class QueueManager(ExtendedManager): 129 | 130 | def enabled(self): 131 | return self.filter(is_enabled=True) 132 | 133 | def _add(self, name, **declaration): 134 | return self.get_or_create(name=name, defaults=declaration)[0] 135 | 136 | def add(self, name, exchange=None, exchange_type=None, 137 | routing_key=None, **options): 138 | options = serialize(options) if options else None 139 | return self._add(name, exchange=exchange, exchange_type=exchange_type, 140 | routing_key=routing_key, options=options) 141 | -------------------------------------------------------------------------------- /cyme/settings.py: -------------------------------------------------------------------------------- 1 | """Since cyme works as a contained Django APP, this is the default settings 2 | file used when cyme is used outside of a Django project context.""" 3 | from __future__ import absolute_import 4 | 5 | 6 | import os 7 | import djcelery 8 | djcelery.setup_loader() 9 | 10 | DEBUG = True 11 | 12 | # Broker settings. 13 | BROKER_HOST = 'amqp://127.0.0.1:5672//' 14 | BROKER_POOL_LIMIT = 100 15 | 16 | 17 | CELERYD_LOG_FORMAT = """\ 18 | [%(asctime)s: %(levelname)s] %(message)s\ 19 | """.strip() 20 | 21 | DB_NAME = os.environ.get('CYME_DB_NAME') or 'branch.db' 22 | 23 | # Databases 24 | DATABASES = {'default': { 25 | 'ENGINE': 'django.db.backends.sqlite3', 26 | 'NAME': DB_NAME, 27 | }} 28 | 29 | # URL and file paths. 30 | SITE_ID = 1 31 | STATIC_URL = '/static/' 32 | ADMIN_MEDIA_PREFIX = '/adminstatic/' 33 | TEMPLATE_LOADERS = ( 34 | ('django.template.loaders.cached.Loader', ( 35 | 'django.template.loaders.filesystem.Loader', 36 | 'django.template.loaders.app_directories.Loader', 37 | )), 38 | ) 39 | ROOT_URLCONF = 'cyme.api.urls' 40 | 41 | # Time and localization. 42 | TIME_ZONE = 'UTC' 43 | LANGUAGE_CODE = 'en-us' 44 | USE_I18N = USE_L10N = True 45 | 46 | # Apps and middleware. 47 | INSTALLED_APPS = ('django.contrib.auth', 48 | 'django.contrib.contenttypes', 49 | 'django.contrib.sessions', 50 | 'cyme', # cyme must be before admin. 51 | 'cyme.api', 52 | 'django.contrib.admin', 53 | 'django.contrib.admindocs') 54 | 55 | MIDDLEWARE_CLASSES = ( 56 | 'django.middleware.common.CommonMiddleware', 57 | 'django.contrib.sessions.middleware.SessionMiddleware', 58 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 59 | 'django.contrib.messages.middleware.MessageMiddleware', 60 | ) 61 | 62 | # Make this unique, and don't share it with anybody. 63 | SECRET_KEY = '9a3!m32h23psjjkkjl#()hs+-sv@$3*mgq!m3s!encow2&*738' 64 | -------------------------------------------------------------------------------- /cyme/status.py: -------------------------------------------------------------------------------- 1 | """cyme.status""" 2 | 3 | from __future__ import absolute_import, with_statement 4 | 5 | from collections import defaultdict 6 | 7 | from celery.datastructures import TokenBucket 8 | from celery.utils.timeutils import rate 9 | from kombu.common import insured as _insured 10 | from kombu.log import LogMixin 11 | from kombu.utils import fxrangemax 12 | 13 | from .models import Instance 14 | from .branch.state import state 15 | 16 | 17 | class Status(LogMixin): 18 | paused = False 19 | restart_max_rate = '100/s' 20 | 21 | def __init__(self): 22 | self._buckets = defaultdict(lambda: TokenBucket( 23 | rate(self.restart_max_rate))) 24 | 25 | def start_all(self): 26 | for instance in self.all_instances(): 27 | self._do_verify_instance(instance, ratelimit=False) 28 | 29 | def restart_all(self): 30 | for instance in self.all_instances(): 31 | self._do_restart_instance(instance, ratelimit=False) 32 | 33 | def shutdown_all(self): 34 | for instance in self.all_instances(): 35 | try: 36 | self._do_stop_verify_instance(instance) 37 | except Exception, exc: 38 | self.error("Couldn't stop instance %s: %r" % (instance.name, 39 | exc)) 40 | 41 | def all_instances(self): 42 | return Instance.objects.all() 43 | 44 | def insured(self, instance, fun, *args, **kwargs): 45 | """Ensures any function performing a broadcast command completes 46 | despite intermittent connection failures.""" 47 | 48 | def errback(self, exc, interval): 49 | self.error( 50 | 'Error while trying to broadcast %r: %r\n' % (fun, exc)) 51 | self.pause() 52 | 53 | return _insured(instance.broker.pool, fun, args, kwargs, 54 | on_revive=state.on_broker_revive, 55 | errback=errback) 56 | 57 | def ib(self, fun, *args, **kwargs): 58 | """Shortcut to ``self.insured( 59 | fun.im_self, fun(*args, **kwargs))``""" 60 | return self.insured(fun.im_self, fun, *args, **kwargs) 61 | 62 | def pause(self): 63 | self.paused = True # only used in supervisor. 64 | 65 | def resume(self): 66 | self.paused = False 67 | 68 | def respond_to_ping(self): 69 | pass 70 | 71 | def _verify_restart_instance(self, instance): 72 | """Restarts the instance, and verifies that the instance is 73 | actually able to start.""" 74 | self.info('%s instance.restart' % (instance, )) 75 | instance.restart() 76 | is_alive = False 77 | for i in fxrangemax(0.1, 1, 0.4, 30): 78 | self.info('%s pingWithTimeout: %s', instance, i) 79 | self.respond_to_ping() 80 | if self.insured(instance, instance.responds_to_ping, timeout=i): 81 | is_alive = True 82 | break 83 | if is_alive: 84 | self.info('%s successfully restarted' % (instance, )) 85 | else: 86 | self.info("%s instance doesn't respond after restart" % ( 87 | instance, )) 88 | 89 | def _can_restart(self): 90 | """Returns true if the supervisor is allowed to restart 91 | an instance at this point.""" 92 | return True 93 | 94 | def _do_restart_instance(self, instance, ratelimit=False): 95 | bucket = self._buckets[instance.restart] 96 | if ratelimit: 97 | if self._can_restart(): 98 | if bucket.can_consume(1): 99 | self._verify_restart_instance(instance) 100 | else: 101 | self.error( 102 | '%s instance.disabled: Restarted too often', instance) 103 | instance.disable() 104 | self._buckets.pop(instance.restart) 105 | else: 106 | self._buckets.pop(instance.restart, None) 107 | self._verify_restart_instance(instance) 108 | 109 | def _do_stop_instance(self, instance): 110 | self.info('%s instance.shutdown' % (instance, )) 111 | instance.stop() 112 | 113 | def _do_stop_verify_instance(self, instance): 114 | self.info('%s instance.shutdown' % (instance, )) 115 | instance.stop_verify() 116 | 117 | def _do_verify_instance(self, instance, ratelimit=False): 118 | if not self.paused: 119 | if instance.is_enabled and instance.pk: 120 | if not self.ib(instance.alive): 121 | self._do_restart_instance(instance, ratelimit=ratelimit) 122 | self._verify_instance_processes(instance) 123 | self._verify_instance_queues(instance) 124 | else: 125 | if self.ib(instance.alive): 126 | self._do_stop_instance(instance) 127 | 128 | def _verify_instance_queues(self, instance): 129 | """Verify that the queues the instance is consuming from matches 130 | the queues listed in the model.""" 131 | queues = set(instance.queues) 132 | reply = self.ib(instance.consuming_from) 133 | if reply is None: 134 | return 135 | consuming_from = set(reply.keys()) 136 | 137 | for queue in consuming_from ^ queues: 138 | if queue in queues: 139 | self.info('%s: instance.consume_from: %s' % (instance, queue)) 140 | self.ib(instance.add_queue, queue) 141 | elif queue == instance.direct_queue: 142 | pass 143 | else: 144 | self.info( 145 | '%s: instance.cancel_consume: %s' % (instance, queue)) 146 | self.ib(instance.cancel_queue, queue) 147 | 148 | def _verify_instance_processes(self, instance): 149 | """Verify that the max/min concurrency settings of the 150 | instance matches that which is specified in the model.""" 151 | max, min = instance.max_concurrency, instance.min_concurrency 152 | try: 153 | current = self.insured(instance, instance.stats)['autoscaler'] 154 | except (TypeError, KeyError): 155 | return 156 | if max != current['max'] or min != current['min']: 157 | self.info('%s: instance.set_autoscale max=%r min=%r' % ( 158 | instance, max, min)) 159 | self.ib(instance.autoscale, max, min) 160 | -------------------------------------------------------------------------------- /cyme/tasks.py: -------------------------------------------------------------------------------- 1 | """cyme.tasks 2 | 3 | - We have our own version of the webhook task. 4 | 5 | - It simply forwards the original request, not depending on any semantics 6 | present in the query string of the request, nor in the data returned 7 | in the response. 8 | 9 | """ 10 | from __future__ import absolute_import 11 | 12 | from celery.task import task 13 | from requests import request 14 | 15 | from . import __version__ 16 | 17 | #: Cyme User Agent string. 18 | UA = 'Celery/cyme v%s' % (__version__, ) 19 | 20 | #: Default HTTP headers to pass to the dispatched request. 21 | DEFAULT_HEADERS = {'User-Agent': UA} 22 | 23 | 24 | def response_to_dict(r): 25 | return dict(status_code=r.status_code, url=r.url, 26 | headers=r.headers, content=r.read()) 27 | 28 | 29 | @task(timeout=60) 30 | def webhook(url, method='GET', params={}, data={}, headers=None, **kwargs): 31 | kwargs['timeout'] = kwargs.get('timeout', webhook.timeout) 32 | headers = {} if headers is None else headers 33 | return response_to_dict(request(method, url, params=params, data=data, 34 | headers=dict(headers, **DEFAULT_HEADERS))) 35 | -------------------------------------------------------------------------------- /cyme/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/celery/cyme/c49047318c56f6674bd6d6ea6673b48be4eb7886/cyme/tests/__init__.py -------------------------------------------------------------------------------- /cyme/tests/test_branch.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from celery.tests.utils import unittest 4 | from mock import Mock 5 | 6 | from cyme.branch import Branch 7 | from cyme.thread import gThread 8 | 9 | 10 | class test_Branch(unittest.TestCase): 11 | 12 | def setUp(self): 13 | self._gstart, gThread.start = gThread.start, Mock() 14 | 15 | def tearDown(self): 16 | gThread.start = self._gstart 17 | 18 | def test_run(self): 19 | branch = Branch() 20 | self.assertTrue(branch.id) 21 | self.assertTrue(branch.components) 22 | branch2 = Branch(without_httpd=True, without_amqp=True) 23 | self.assertNotEqual(len(branch2.components), len(branch.components)) 24 | branch.run() 25 | for component in branch.components: 26 | component.start.assert_called_with() 27 | -------------------------------------------------------------------------------- /cyme/tests/test_models.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from celery.tests.utils import unittest 4 | from mock import Mock 5 | 6 | from cyme.models import Instance, Queue 7 | 8 | 9 | class test_Queue(unittest.TestCase): 10 | 11 | def test__unicode__(self): 12 | self.assertTrue(unicode(Queue(name='foo'))) 13 | 14 | def test_enabled(self): 15 | q1 = Queue.objects.add('foo', options={'x': 1}) 16 | q2 = Queue.objects.add('bar') 17 | q2.is_enabled = False 18 | q2.save() 19 | 20 | x = Queue.objects.enabled() 21 | self.assertEqual(x[0], q1) 22 | 23 | 24 | class test_Instance(unittest.TestCase): 25 | 26 | def setUp(self): 27 | self._mt, Instance.MultiTool = Instance.MultiTool, Mock() 28 | self._query, Instance._query = Instance._query, Mock() 29 | 30 | def tearDown(self): 31 | Instance.MultiTool = self._mt 32 | Instance._query = self._query 33 | Queue.objects.all().delete() 34 | Instance.objects.all().delete() 35 | 36 | def test__unicode__(self): 37 | self.assertTrue(unicode(Instance(name='foo'))) 38 | 39 | def test_add(self): 40 | n1 = Instance.objects.add() 41 | self.assertTrue(n1.name) 42 | self.assertTrue(n1.default_args) 43 | self.assertTrue(n1.direct_queue) 44 | self.assertTrue(n1.multi) 45 | self.assertTrue(n1.argtuple) 46 | self.assertTrue(n1.argv) 47 | self.assertIsNone(n1.getpid()) 48 | 49 | n2 = Instance.objects.add('foo') 50 | self.assertEqual(n2.name, 'foo') 51 | 52 | n1.disable() 53 | self.assertFalse(Instance.objects.get(name=n1.name).is_enabled) 54 | n1.enable() 55 | self.assertTrue(Instance.objects.get(name=n1.name).is_enabled) 56 | 57 | def test_start_stop_restart(self): 58 | n = Instance.objects.add() 59 | n.start() 60 | n.stop() 61 | n.restart() 62 | 63 | def test_add_cancel_queue(self): 64 | n = Instance.objects.add() 65 | q = Queue.objects.create(name='xiziasd') 66 | q.save() 67 | 68 | n.add_queue(q) 69 | n._query.assert_called_with('add_consumer', dict(queue=q.name, 70 | exchange=q.name, routing_key=q.name, 71 | exchange_type=None)) 72 | 73 | n.cancel_queue(q) 74 | n._query.assert_called_with('cancel_consumer', dict(queue=q.name)) 75 | 76 | def test_objects(self): 77 | n = Instance.objects.add() 78 | Instance.objects.disable(n) 79 | Instance.objects.enable(n) 80 | Instance.objects.remove(n) 81 | 82 | def test_with_queues(self): 83 | Instance.objects.add(queues='foo,bar,baz') 84 | Instance.objects.add_queue('foo') 85 | Instance.objects.add_queue('xaz') 86 | 87 | Instance.objects.remove_queue('foo') 88 | Instance.objects.remove_queue('xaz') 89 | -------------------------------------------------------------------------------- /cyme/utils/__init__.py: -------------------------------------------------------------------------------- 1 | """cyme.utils""" 2 | from __future__ import absolute_import 3 | 4 | import sys 5 | 6 | from importlib import import_module 7 | 8 | from celery import current_app as celery 9 | from celery.utils import get_cls_by_name 10 | from vine import promise, maybe_promise # noqa 11 | from kombu.utils import uuid, cached_property # noqa 12 | from unipath import Path as _Path 13 | 14 | _pkg_cache = {} 15 | 16 | 17 | def force_list(obj): 18 | """Force object to be a list. 19 | 20 | If ``obj`` is a scalar value then a list with that value as 21 | sole element is returned, or 22 | if ``obj`` is a tuple then it is coerced into a list. 23 | 24 | """ 25 | if isinstance(obj, tuple): 26 | return list(obj) 27 | elif not isinstance(obj, list): 28 | return [obj] 29 | return obj 30 | 31 | 32 | def find_package(mod, _s=None): 33 | """Find the package a module belongs to. 34 | 35 | if you have structure:: 36 | 37 | package/__init__.py 38 | /foo.py 39 | /bar/__init__.py 40 | /bar/baz.py 41 | 42 | Then the following examples returns:: 43 | 44 | >>> find_package(import_module('package')) 45 | 'package' 46 | >>> find_package(import_module('package.foo')) 47 | 'package' 48 | >>> find_package(import_module('package.bar.baz')) 49 | >>> package.bar 50 | 51 | Note that this does not look at the file system, 52 | but rather uses the ``__package__`` attribute of a module. 53 | 54 | """ 55 | pkg = None 56 | _s = _s or mod 57 | if not mod: 58 | return 59 | if mod in _pkg_cache: 60 | pkg = _pkg_cache[_s] = _pkg_cache[mod] 61 | else: 62 | _mod = sys.modules[mod] 63 | if _mod.__package__: 64 | pkg = _pkg_cache[_s] = _mod.__package__ 65 | return pkg or find_package('.'.join(mod.split('.')[:-1]), _s) 66 | 67 | 68 | def find_symbol(origin, sym): 69 | """Find symbol in module relative by ``origin``. 70 | 71 | E.g. if ``origin`` is an object in the module ``package.foo``, 72 | then:: 73 | 74 | >>> find_symbol(origin, '.bar.my_symbol') 75 | 76 | will return the object ``my_symbol`` from module ``package.bar``. 77 | 78 | """ 79 | return get_cls_by_name(sym, 80 | package=find_package(getattr(origin, '__module__', None 81 | or origin.__class__.__module__))) 82 | 83 | 84 | def instantiate(origin, sym, *args, **kwargs): 85 | """Like :func:`find_symbol` but instantiates the class found 86 | using ``*args`` and ``**kwargs``.""" 87 | return find_symbol(origin, sym)(*args, **kwargs) 88 | 89 | 90 | class Path(_Path): 91 | """:class:`unipath.Path` version that can use the ``/`` operator 92 | to combine paths:: 93 | 94 | >>> p = Path('foo') 95 | >>> p / 'bar' / 'baz' 96 | Path('foo/bar/baz') 97 | """ 98 | 99 | def __div__(self, other): 100 | return Path(self, other) 101 | 102 | 103 | def imerge_settings(a, b): 104 | """Merge two django settings modules, 105 | keys in ``b`` have precedence.""" 106 | orig = import_module(a.SETTINGS_MODULE) 107 | for key, value in vars(b).iteritems(): 108 | if not hasattr(orig, key): 109 | setattr(a, key, value) 110 | 111 | 112 | def setup_logging(loglevel='INFO', logfile=None): 113 | """Setup logging using ``loglevel`` and ``logfile``. 114 | 115 | stderr will be used if not logfile provided. 116 | 117 | """ 118 | from celery.utils import LOG_LEVELS 119 | if isinstance(loglevel, basestring): 120 | loglevel = LOG_LEVELS[loglevel] 121 | return celery.log.setup_logging_subsystem(loglevel, logfile) 122 | 123 | 124 | def redirect_stdouts_to_logger(loglevel='INFO', logfile=None, 125 | redirect_level='WARNING', stdout=False, stderr=True): 126 | """See :meth:`celery.log.Log.redirect_stdouts_to_logger`.""" 127 | # XXX Currently unused. 128 | log = celery.log 129 | handled = setup_logging(loglevel, logfile) 130 | if not handled: 131 | return log.redirect_stdouts_to_logger( 132 | log.get_default_logger(name='cyme'), 133 | redirect_level, stdout=stdout, stderr=stderr) 134 | 135 | 136 | class LazyProgressBar(object): 137 | 138 | def __init__(self, size, description=None, endtext=None): 139 | self.size = size 140 | self.current = 0 141 | self.description = description 142 | self.endtext = endtext 143 | self._finished = False 144 | 145 | def step(self, i=1, **kwargs): 146 | if not self._finished: 147 | if not self.current: 148 | # implicit start 149 | self.current = 1 150 | if self.description: 151 | sys.stderr.write('\n\n%s\n' % (self.description, )) 152 | return self._bar ## noqa 153 | self.current += i 154 | if self.current >= self.size: 155 | self.size = self._bar.maxval = self.current 156 | self._bar.update(self.current) 157 | 158 | def finish(self, **kwargs): 159 | if not self._finished: 160 | self._bar.finish() 161 | self._finished = True 162 | if self.endtext: 163 | sys.stderr.write(self.endtext) 164 | 165 | @cached_property 166 | def _bar(self): 167 | from progressbar import ProgressBar 168 | return ProgressBar(maxval=self.size).start() 169 | -------------------------------------------------------------------------------- /cyme/utils/actors.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from celery.app import app_or_default 4 | 5 | import cell 6 | import cell.presence 7 | 8 | 9 | def construct(cls, instance, connection=None, *args, **kwargs): 10 | app = instance.app = app_or_default(kwargs.pop('app', None)) 11 | super(cls, instance).__init__(connection or app.broker_connection(), 12 | *args, **kwargs) 13 | 14 | 15 | class Actor(cell.Actor): 16 | 17 | def __init__(self, *args, **kwargs): 18 | construct(Actor, self, *args, **kwargs) 19 | 20 | 21 | class Agent(cell.Agent): 22 | 23 | def __init__(self, *args, **kwargs): 24 | construct(Agent, self, *args, **kwargs) 25 | 26 | 27 | class AwareAgent(cell.presence.AwareAgent): 28 | 29 | def __init__(self, *args, **kwargs): 30 | construct(AwareAgent, self, *args, **kwargs) 31 | -------------------------------------------------------------------------------- /cyme/utils/dictshield.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from dictshield.fields import BaseField 4 | 5 | 6 | class ListField(BaseField): 7 | 8 | def __init__(self, field, **kwargs): 9 | self.field = field 10 | kwargs.setdefault('default', list) 11 | super(ListField, self).__init__(**kwargs) 12 | 13 | def to_python(self, value): 14 | return [self.field.to_python(item) for item in value] 15 | 16 | def validate(self, value): 17 | [self.field.validate(item) for item in value] 18 | 19 | def lookup_member(self, name): 20 | return self.field.lookup_member(name) 21 | 22 | @property 23 | def owner_document(self): 24 | return self._owner_document 25 | 26 | @owner_document.setter # noqa 27 | def owner_document(self, d): # noqa 28 | self._owner_document = self.field.owner_document = d 29 | -------------------------------------------------------------------------------- /docs/.static/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/celery/cyme/c49047318c56f6674bd6d6ea6673b48be4eb7886/docs/.static/.keep -------------------------------------------------------------------------------- /docs/.templates/page.html: -------------------------------------------------------------------------------- 1 | {% extends "layout.html" %} 2 | {% block body %} 3 | {{ body }} 4 | {% endblock %} 5 | -------------------------------------------------------------------------------- /docs/.templates/sidebarintro.html: -------------------------------------------------------------------------------- 1 | 4 | 5 | -------------------------------------------------------------------------------- /docs/.templates/sidebarlogo.html: -------------------------------------------------------------------------------- 1 | 4 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | 9 | # Internal variables. 10 | PAPEROPT_a4 = -D latex_paper_size=a4 11 | PAPEROPT_letter = -D latex_paper_size=letter 12 | ALLSPHINXOPTS = -d .build/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 13 | 14 | .PHONY: help clean html web pickle htmlhelp latex changes linkcheck 15 | 16 | help: 17 | @echo "Please use \`make ' where is one of" 18 | @echo " html to make standalone HTML files" 19 | @echo " pickle to make pickle files" 20 | @echo " json to make JSON files" 21 | @echo " htmlhelp to make HTML files and a HTML help project" 22 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 23 | @echo " changes to make an overview over all changed/added/deprecated items" 24 | @echo " linkcheck to check all external links for integrity" 25 | 26 | clean: 27 | -rm -rf .build/* 28 | 29 | html: 30 | mkdir -p .build/html .build/doctrees 31 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) .build/html 32 | @echo 33 | @echo "Build finished. The HTML pages are in .build/html." 34 | 35 | coverage: 36 | mkdir -p .build/coverage .build/doctrees 37 | $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) .build/coverage 38 | @echo 39 | @echo "Build finished." 40 | 41 | pickle: 42 | mkdir -p .build/pickle .build/doctrees 43 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) .build/pickle 44 | @echo 45 | @echo "Build finished; now you can process the pickle files." 46 | 47 | web: pickle 48 | 49 | json: 50 | mkdir -p .build/json .build/doctrees 51 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) .build/json 52 | @echo 53 | @echo "Build finished; now you can process the JSON files." 54 | 55 | htmlhelp: 56 | mkdir -p .build/htmlhelp .build/doctrees 57 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) .build/htmlhelp 58 | @echo 59 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 60 | ".hhp project file in .build/htmlhelp." 61 | 62 | latex: 63 | mkdir -p .build/latex .build/doctrees 64 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) .build/latex 65 | @echo 66 | @echo "Build finished; the LaTeX files are in .build/latex." 67 | @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ 68 | "run these through (pdf)latex." 69 | 70 | changes: 71 | mkdir -p .build/changes .build/doctrees 72 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) .build/changes 73 | @echo 74 | @echo "The overview file is in .build/changes." 75 | 76 | linkcheck: 77 | mkdir -p .build/linkcheck .build/doctrees 78 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) .build/linkcheck 79 | @echo 80 | @echo "Link check complete; look for any errors in the above output " \ 81 | "or in .build/linkcheck/output.txt." 82 | -------------------------------------------------------------------------------- /docs/_ext/applyxrefs.py: -------------------------------------------------------------------------------- 1 | """Adds xref targets to the top of files.""" 2 | 3 | import sys 4 | import os 5 | 6 | testing = False 7 | 8 | DONT_TOUCH = ( 9 | './index.txt', 10 | ) 11 | 12 | 13 | def target_name(fn): 14 | if fn.endswith('.txt'): 15 | fn = fn[:-4] 16 | return '_' + fn.lstrip('./').replace('/', '-') 17 | 18 | 19 | def process_file(fn, lines): 20 | lines.insert(0, '\n') 21 | lines.insert(0, '.. %s:\n' % target_name(fn)) 22 | try: 23 | f = open(fn, 'w') 24 | except IOError: 25 | print("Can't open %s for writing. Not touching it." % fn) 26 | return 27 | try: 28 | f.writelines(lines) 29 | except IOError: 30 | print("Can't write to %s. Not touching it." % fn) 31 | finally: 32 | f.close() 33 | 34 | 35 | def has_target(fn): 36 | try: 37 | f = open(fn, 'r') 38 | except IOError: 39 | print("Can't open %s. Not touching it." % fn) 40 | return (True, None) 41 | readok = True 42 | try: 43 | lines = f.readlines() 44 | except IOError: 45 | print("Can't read %s. Not touching it." % fn) 46 | readok = False 47 | finally: 48 | f.close() 49 | if not readok: 50 | return (True, None) 51 | 52 | #print fn, len(lines) 53 | if len(lines) < 1: 54 | print("Not touching empty file %s." % fn) 55 | return (True, None) 56 | if lines[0].startswith('.. _'): 57 | return (True, None) 58 | return (False, lines) 59 | 60 | 61 | def main(argv=None): 62 | if argv is None: 63 | argv = sys.argv 64 | 65 | if len(argv) == 1: 66 | argv.extend('.') 67 | 68 | files = [] 69 | for root in argv[1:]: 70 | for (dirpath, dirnames, filenames) in os.walk(root): 71 | files.extend([(dirpath, f) for f in filenames]) 72 | files.sort() 73 | files = [os.path.join(p, fn) for p, fn in files if fn.endswith('.txt')] 74 | #print files 75 | 76 | for fn in files: 77 | if fn in DONT_TOUCH: 78 | print("Skipping blacklisted file %s." % fn) 79 | continue 80 | 81 | target_found, lines = has_target(fn) 82 | if not target_found: 83 | if testing: 84 | print '%s: %s' % (fn, lines[0]), 85 | else: 86 | print "Adding xref to %s" % fn 87 | process_file(fn, lines) 88 | else: 89 | print "Skipping %s: already has a xref" % fn 90 | 91 | if __name__ == '__main__': 92 | sys.exit(main()) 93 | -------------------------------------------------------------------------------- /docs/_ext/celerydocs.py: -------------------------------------------------------------------------------- 1 | def setup(app): 2 | app.add_crossref_type( 3 | directivename="setting", 4 | rolename="setting", 5 | indextemplate="pair: %s; setting", 6 | ) 7 | app.add_crossref_type( 8 | directivename="sig", 9 | rolename="sig", 10 | indextemplate="pair: %s; sig", 11 | ) 12 | app.add_crossref_type( 13 | directivename="state", 14 | rolename="state", 15 | indextemplate="pair: %s; state", 16 | ) 17 | app.add_crossref_type( 18 | directivename="control", 19 | rolename="control", 20 | indextemplate="pair: %s; control", 21 | ) 22 | app.add_crossref_type( 23 | directivename="signal", 24 | rolename="signal", 25 | indextemplate="pair: %s; signal", 26 | ) 27 | -------------------------------------------------------------------------------- /docs/_ext/literals_to_xrefs.py: -------------------------------------------------------------------------------- 1 | """ 2 | Runs through a reST file looking for old-style literals, and helps replace them 3 | with new-style references. 4 | """ 5 | 6 | import re 7 | import sys 8 | import shelve 9 | 10 | refre = re.compile(r'``([^`\s]+?)``') 11 | 12 | ROLES = ( 13 | 'attr', 14 | 'class', 15 | "djadmin", 16 | 'data', 17 | 'exc', 18 | 'file', 19 | 'func', 20 | 'lookup', 21 | 'meth', 22 | 'mod', 23 | "djadminopt", 24 | "ref", 25 | "setting", 26 | "term", 27 | "tfilter", 28 | "ttag", 29 | 30 | # special 31 | "skip", 32 | ) 33 | 34 | ALWAYS_SKIP = [ 35 | "NULL", 36 | "True", 37 | "False", 38 | ] 39 | 40 | 41 | def fixliterals(fname): 42 | data = open(fname).read() 43 | 44 | last = 0 45 | new = [] 46 | storage = shelve.open("/tmp/literals_to_xref.shelve") 47 | lastvalues = storage.get("lastvalues", {}) 48 | 49 | for m in refre.finditer(data): 50 | 51 | new.append(data[last:m.start()]) 52 | last = m.end() 53 | 54 | line_start = data.rfind("\n", 0, m.start()) 55 | line_end = data.find("\n", m.end()) 56 | prev_start = data.rfind("\n", 0, line_start) 57 | next_end = data.find("\n", line_end + 1) 58 | 59 | # Skip always-skip stuff 60 | if m.group(1) in ALWAYS_SKIP: 61 | new.append(m.group(0)) 62 | continue 63 | 64 | # skip when the next line is a title 65 | next_line = data[m.end():next_end].strip() 66 | if next_line[0] in "!-/:-@[-`{-~" and \ 67 | all(c == next_line[0] for c in next_line): 68 | new.append(m.group(0)) 69 | continue 70 | 71 | sys.stdout.write("\n" + "-" * 80 + "\n") 72 | sys.stdout.write(data[prev_start + 1:m.start()]) 73 | sys.stdout.write(colorize(m.group(0), fg="red")) 74 | sys.stdout.write(data[m.end():next_end]) 75 | sys.stdout.write("\n\n") 76 | 77 | replace_type = None 78 | while replace_type is None: 79 | replace_type = raw_input( 80 | colorize("Replace role: ", fg="yellow")).strip().lower() 81 | if replace_type and replace_type not in ROLES: 82 | replace_type = None 83 | 84 | if replace_type == "": 85 | new.append(m.group(0)) 86 | continue 87 | 88 | if replace_type == "skip": 89 | new.append(m.group(0)) 90 | ALWAYS_SKIP.append(m.group(1)) 91 | continue 92 | 93 | default = lastvalues.get(m.group(1), m.group(1)) 94 | if default.endswith("()") and \ 95 | replace_type in ("class", "func", "meth"): 96 | default = default[:-2] 97 | replace_value = raw_input( 98 | colorize("Text [", fg="yellow") + default + \ 99 | colorize("]: ", fg="yellow")).strip() 100 | if not replace_value: 101 | replace_value = default 102 | new.append(":%s:`%s`" % (replace_type, replace_value)) 103 | lastvalues[m.group(1)] = replace_value 104 | 105 | new.append(data[last:]) 106 | open(fname, "w").write("".join(new)) 107 | 108 | storage["lastvalues"] = lastvalues 109 | storage.close() 110 | 111 | 112 | def colorize(text='', opts=(), **kwargs): 113 | """ 114 | Returns your text, enclosed in ANSI graphics codes. 115 | 116 | Depends on the keyword arguments 'fg' and 'bg', and the contents of 117 | the opts tuple/list. 118 | 119 | Returns the RESET code if no parameters are given. 120 | 121 | Valid colors: 122 | 'black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan', 'white' 123 | 124 | Valid options: 125 | 'bold' 126 | 'underscore' 127 | 'blink' 128 | 'reverse' 129 | 'conceal' 130 | 'noreset' - string will not be auto-terminated with the RESET code 131 | 132 | Examples: 133 | colorize('hello', fg='red', bg='blue', opts=('blink',)) 134 | colorize() 135 | colorize('goodbye', opts=('underscore',)) 136 | print colorize('first line', fg='red', opts=('noreset',)) 137 | print 'this should be red too' 138 | print colorize('and so should this') 139 | print 'this should not be red' 140 | """ 141 | color_names = ('black', 'red', 'green', 'yellow', 142 | 'blue', 'magenta', 'cyan', 'white') 143 | foreground = dict([(color_names[x], '3%s' % x) for x in range(8)]) 144 | background = dict([(color_names[x], '4%s' % x) for x in range(8)]) 145 | 146 | RESET = '0' 147 | opt_dict = {'bold': '1', 148 | 'underscore': '4', 149 | 'blink': '5', 150 | 'reverse': '7', 151 | 'conceal': '8'} 152 | 153 | text = str(text) 154 | code_list = [] 155 | if text == '' and len(opts) == 1 and opts[0] == 'reset': 156 | return '\x1b[%sm' % RESET 157 | for k, v in kwargs.iteritems(): 158 | if k == 'fg': 159 | code_list.append(foreground[v]) 160 | elif k == 'bg': 161 | code_list.append(background[v]) 162 | for o in opts: 163 | if o in opt_dict: 164 | code_list.append(opt_dict[o]) 165 | if 'noreset' not in opts: 166 | text = text + '\x1b[%sm' % RESET 167 | return ('\x1b[%sm' % ';'.join(code_list)) + text 168 | 169 | if __name__ == '__main__': 170 | try: 171 | fixliterals(sys.argv[1]) 172 | except (KeyboardInterrupt, SystemExit): 173 | print 174 | -------------------------------------------------------------------------------- /docs/_theme/celery/static/celery.css_t: -------------------------------------------------------------------------------- 1 | /* 2 | * celery.css_t 3 | * ~~~~~~~~~~~~ 4 | * 5 | * :copyright: Copyright 2010 by Armin Ronacher. 6 | * :license: BSD, see LICENSE for details. 7 | */ 8 | 9 | {% set page_width = 940 %} 10 | {% set sidebar_width = 220 %} 11 | {% set body_font_stack = 'Optima, Segoe, "Segoe UI", Candara, Calibri, Arial, sans-serif' %} 12 | {% set headline_font_stack = 'Futura, "Trebuchet MS", Arial, sans-serif' %} 13 | {% set code_font_stack = "'Consolas', 'Menlo', 'Deja Vu Sans Mono', 'Bitstream Vera Sans Mono', monospace" %} 14 | 15 | @import url("basic.css"); 16 | 17 | /* -- page layout ----------------------------------------------------------- */ 18 | 19 | body { 20 | font-family: {{ body_font_stack }}; 21 | font-size: 17px; 22 | background-color: white; 23 | color: #000; 24 | margin: 30px 0 0 0; 25 | padding: 0; 26 | } 27 | 28 | div.document { 29 | width: {{ page_width }}px; 30 | margin: 0 auto; 31 | } 32 | 33 | div.deck { 34 | font-size: 18px; 35 | } 36 | 37 | p.developmentversion { 38 | color: red; 39 | } 40 | 41 | div.related { 42 | width: {{ page_width - 20 }}px; 43 | padding: 5px 10px; 44 | background: #F2FCEE; 45 | margin: 15px auto 15px auto; 46 | } 47 | 48 | div.documentwrapper { 49 | float: left; 50 | width: 100%; 51 | } 52 | 53 | div.bodywrapper { 54 | margin: 0 0 0 {{ sidebar_width }}px; 55 | } 56 | 57 | div.sphinxsidebar { 58 | width: {{ sidebar_width }}px; 59 | } 60 | 61 | hr { 62 | border: 1px solid #B1B4B6; 63 | } 64 | 65 | div.body { 66 | background-color: #ffffff; 67 | color: #3E4349; 68 | padding: 0 30px 0 30px; 69 | } 70 | 71 | img.celerylogo { 72 | padding: 0 0 10px 10px; 73 | float: right; 74 | } 75 | 76 | div.footer { 77 | width: {{ page_width - 15 }}px; 78 | margin: 10px auto 30px auto; 79 | padding-right: 15px; 80 | font-size: 14px; 81 | color: #888; 82 | text-align: right; 83 | } 84 | 85 | div.footer a { 86 | color: #888; 87 | } 88 | 89 | div.sphinxsidebar a { 90 | color: #444; 91 | text-decoration: none; 92 | border-bottom: 1px dashed #DCF0D5; 93 | } 94 | 95 | div.sphinxsidebar a:hover { 96 | border-bottom: 1px solid #999; 97 | } 98 | 99 | div.sphinxsidebar { 100 | font-size: 14px; 101 | line-height: 1.5; 102 | } 103 | 104 | div.sphinxsidebarwrapper { 105 | padding: 7px 10px; 106 | } 107 | 108 | div.sphinxsidebarwrapper p.logo { 109 | padding: 0 0 20px 0; 110 | margin: 0; 111 | } 112 | 113 | div.sphinxsidebar h3, 114 | div.sphinxsidebar h4 { 115 | font-family: {{ headline_font_stack }}; 116 | color: #444; 117 | font-size: 24px; 118 | font-weight: normal; 119 | margin: 0 0 5px 0; 120 | padding: 0; 121 | } 122 | 123 | div.sphinxsidebar h4 { 124 | font-size: 20px; 125 | } 126 | 127 | div.sphinxsidebar h3 a { 128 | color: #444; 129 | } 130 | 131 | div.sphinxsidebar p.logo a, 132 | div.sphinxsidebar h3 a, 133 | div.sphinxsidebar p.logo a:hover, 134 | div.sphinxsidebar h3 a:hover { 135 | border: none; 136 | } 137 | 138 | div.sphinxsidebar p { 139 | color: #555; 140 | margin: 10px 0; 141 | } 142 | 143 | div.sphinxsidebar ul { 144 | margin: 10px 0; 145 | padding: 0; 146 | color: #000; 147 | } 148 | 149 | div.sphinxsidebar input { 150 | border: 1px solid #ccc; 151 | font-family: {{ body_font_stack }}; 152 | font-size: 1em; 153 | } 154 | 155 | /* -- body styles ----------------------------------------------------------- */ 156 | 157 | a { 158 | color: #348613; 159 | text-decoration: underline; 160 | } 161 | 162 | a:hover { 163 | color: #59B833; 164 | text-decoration: underline; 165 | } 166 | 167 | div.body h1, 168 | div.body h2, 169 | div.body h3, 170 | div.body h4, 171 | div.body h5, 172 | div.body h6 { 173 | font-family: {{ headline_font_stack }}; 174 | font-weight: normal; 175 | margin: 30px 0px 10px 0px; 176 | padding: 0; 177 | } 178 | 179 | div.body h1 { margin-top: 0; padding-top: 0; font-size: 200%; } 180 | div.body h2 { font-size: 180%; } 181 | div.body h3 { font-size: 150%; } 182 | div.body h4 { font-size: 130%; } 183 | div.body h5 { font-size: 100%; } 184 | div.body h6 { font-size: 100%; } 185 | 186 | div.body h1 a.toc-backref, 187 | div.body h2 a.toc-backref, 188 | div.body h3 a.toc-backref, 189 | div.body h4 a.toc-backref, 190 | div.body h5 a.toc-backref, 191 | div.body h6 a.toc-backref { 192 | color: inherit!important; 193 | text-decoration: none; 194 | } 195 | 196 | a.headerlink { 197 | color: #ddd; 198 | padding: 0 4px; 199 | text-decoration: none; 200 | } 201 | 202 | a.headerlink:hover { 203 | color: #444; 204 | background: #eaeaea; 205 | } 206 | 207 | div.body p, div.body dd, div.body li { 208 | line-height: 1.4em; 209 | } 210 | 211 | div.admonition { 212 | background: #fafafa; 213 | margin: 20px -30px; 214 | padding: 10px 30px; 215 | border-top: 1px solid #ccc; 216 | border-bottom: 1px solid #ccc; 217 | } 218 | 219 | div.admonition p.admonition-title { 220 | font-family: {{ headline_font_stack }}; 221 | font-weight: normal; 222 | font-size: 24px; 223 | margin: 0 0 10px 0; 224 | padding: 0; 225 | line-height: 1; 226 | } 227 | 228 | div.admonition p.last { 229 | margin-bottom: 0; 230 | } 231 | 232 | div.highlight{ 233 | background-color: white; 234 | } 235 | 236 | dt:target, .highlight { 237 | background: #FAF3E8; 238 | } 239 | 240 | div.note { 241 | background-color: #eee; 242 | border: 1px solid #ccc; 243 | } 244 | 245 | div.seealso { 246 | background-color: #ffc; 247 | border: 1px solid #ff6; 248 | } 249 | 250 | div.topic { 251 | background-color: #eee; 252 | } 253 | 254 | div.warning { 255 | background-color: #ffe4e4; 256 | border: 1px solid #f66; 257 | } 258 | 259 | p.admonition-title { 260 | display: inline; 261 | } 262 | 263 | p.admonition-title:after { 264 | content: ":"; 265 | } 266 | 267 | pre, tt { 268 | font-family: {{ code_font_stack }}; 269 | font-size: 0.9em; 270 | } 271 | 272 | img.screenshot { 273 | } 274 | 275 | tt.descname, tt.descclassname { 276 | font-size: 0.95em; 277 | } 278 | 279 | tt.descname { 280 | padding-right: 0.08em; 281 | } 282 | 283 | img.screenshot { 284 | -moz-box-shadow: 2px 2px 4px #eee; 285 | -webkit-box-shadow: 2px 2px 4px #eee; 286 | box-shadow: 2px 2px 4px #eee; 287 | } 288 | 289 | table.docutils { 290 | border: 1px solid #888; 291 | -moz-box-shadow: 2px 2px 4px #eee; 292 | -webkit-box-shadow: 2px 2px 4px #eee; 293 | box-shadow: 2px 2px 4px #eee; 294 | } 295 | 296 | table.docutils td, table.docutils th { 297 | border: 1px solid #888; 298 | padding: 0.25em 0.7em; 299 | } 300 | 301 | table.field-list, table.footnote { 302 | border: none; 303 | -moz-box-shadow: none; 304 | -webkit-box-shadow: none; 305 | box-shadow: none; 306 | } 307 | 308 | table.footnote { 309 | margin: 15px 0; 310 | width: 100%; 311 | border: 1px solid #eee; 312 | background: #fdfdfd; 313 | font-size: 0.9em; 314 | } 315 | 316 | table.footnote + table.footnote { 317 | margin-top: -15px; 318 | border-top: none; 319 | } 320 | 321 | table.field-list th { 322 | padding: 0 0.8em 0 0; 323 | } 324 | 325 | table.field-list td { 326 | padding: 0; 327 | } 328 | 329 | table.footnote td.label { 330 | width: 0px; 331 | padding: 0.3em 0 0.3em 0.5em; 332 | } 333 | 334 | table.footnote td { 335 | padding: 0.3em 0.5em; 336 | } 337 | 338 | dl { 339 | margin: 0; 340 | padding: 0; 341 | } 342 | 343 | dl dd { 344 | margin-left: 30px; 345 | } 346 | 347 | blockquote { 348 | margin: 0 0 0 30px; 349 | padding: 0; 350 | } 351 | 352 | ul { 353 | margin: 10px 0 10px 30px; 354 | padding: 0; 355 | } 356 | 357 | pre { 358 | background: #F0FFEB; 359 | padding: 7px 10px; 360 | margin: 15px 0; 361 | border: 1px solid #C7ECB8; 362 | border-radius: 2px; 363 | -moz-border-radius: 2px; 364 | -webkit-border-radius: 2px; 365 | line-height: 1.3em; 366 | } 367 | 368 | tt { 369 | background: #F0FFEB; 370 | color: #222; 371 | /* padding: 1px 2px; */ 372 | } 373 | 374 | tt.xref, a tt { 375 | background: #F0FFEB; 376 | border-bottom: 1px solid white; 377 | } 378 | 379 | a.reference { 380 | text-decoration: none; 381 | border-bottom: 1px dashed #DCF0D5; 382 | } 383 | 384 | a.reference:hover { 385 | border-bottom: 1px solid #6D4100; 386 | } 387 | 388 | a.footnote-reference { 389 | text-decoration: none; 390 | font-size: 0.7em; 391 | vertical-align: top; 392 | border-bottom: 1px dashed #DCF0D5; 393 | } 394 | 395 | a.footnote-reference:hover { 396 | border-bottom: 1px solid #6D4100; 397 | } 398 | 399 | a:hover tt { 400 | background: #EEE; 401 | } 402 | -------------------------------------------------------------------------------- /docs/_theme/celery/theme.conf: -------------------------------------------------------------------------------- 1 | [theme] 2 | inherit = basic 3 | stylesheet = celery.css 4 | 5 | [options] 6 | -------------------------------------------------------------------------------- /docs/changelog.rst: -------------------------------------------------------------------------------- 1 | ../Changelog -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import sys 4 | import os 5 | 6 | this = os.path.dirname(os.path.abspath(__file__)) 7 | 8 | # If your extensions are in another directory, add it here. If the directory 9 | # is relative to the documentation root, use os.path.abspath to make it 10 | # absolute, like shown here. 11 | sys.path.append(os.path.join(os.pardir, "tests")) 12 | sys.path.append(os.path.join(this, "_ext")) 13 | settings = os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings") 14 | import django 15 | if django.VERSION < (1, 4): 16 | from django.core.management import setup_environ 17 | setup_environ(__import__(settings)) 18 | import cyme 19 | 20 | 21 | # use app loader 22 | from celery import Celery 23 | app = Celery(set_as_current=True) 24 | app.conf.update(BROKER_TRANSPORT="memory", 25 | CELERY_RESULT_BACKEND="cache", 26 | CELERY_CACHE_BACKEND="memory", 27 | CELERYD_HIJACK_ROOT_LOGGER=False, 28 | CELERYD_LOG_COLOR=False) 29 | 30 | # General configuration 31 | # --------------------- 32 | 33 | extensions = ['sphinx.ext.autodoc', 34 | 'sphinx.ext.coverage', 35 | 'sphinx.ext.pngmath', 36 | 'sphinx.ext.intersphinx', 37 | 'sphinxcontrib.issuetracker', 38 | 'celerydocs'] 39 | 40 | # Add any paths that contain templates here, relative to this directory. 41 | templates_path = ['.templates'] 42 | 43 | # The suffix of source filenames. 44 | source_suffix = '.rst' 45 | 46 | # The master toctree document. 47 | master_doc = 'index' 48 | 49 | # General information about the project. 50 | project = u'Cyme' 51 | copyright = u'2011-2013, Ask Solem. All Rights Reserved' 52 | 53 | # The version info for the project you're documenting, acts as replacement for 54 | # |version| and |release|, also used in various other places throughout the 55 | # built documents. 56 | # 57 | # The short X.Y version. 58 | version = ".".join(map(str, cyme.VERSION[0:2])) 59 | # The full version, including alpha/beta/rc tags. 60 | release = cyme.__version__ 61 | 62 | exclude_trees = ['.build'] 63 | 64 | # If true, '()' will be appended to :func: etc. cross-reference text. 65 | add_function_parentheses = True 66 | 67 | # The name of the Pygments (syntax highlighting) style to use. 68 | pygments_style = 'trac' 69 | 70 | # Add any paths that contain custom static files (such as style sheets) here, 71 | # relative to this directory. They are copied after the builtin static files, 72 | # so a file named "default.css" will overwrite the builtin "default.css". 73 | html_static_path = ['.static'] 74 | 75 | html_use_smartypants = True 76 | 77 | # If false, no module index is generated. 78 | html_use_modindex = True 79 | 80 | # If false, no index is generated. 81 | html_use_index = True 82 | 83 | latex_documents = [ 84 | ('index', 'cyme.tex', ur'Cyme Documentation', 85 | ur'Ask Solem', 'manual'), 86 | ] 87 | 88 | html_show_sphinx = False 89 | intersphinx_mapping = { 90 | "http://docs.python.org/dev": None, 91 | "https://kombu.readthedocs.io/en/latest/": None, 92 | "https://django-celery.readthedocs.io/en/latest": None, 93 | } 94 | 95 | html_theme = "celery" 96 | html_theme_path = ["_theme"] 97 | html_sidebars = { 98 | 'index': ['sidebarintro.html', 'sourcelink.html', 'searchbox.html'], 99 | '**': ['sidebarlogo.html', 'relations.html', 100 | 'sourcelink.html', 'searchbox.html'], 101 | } 102 | 103 | ### Issuetracker 104 | 105 | #issuetracker = "github" 106 | #issuetracker_user = "ask" 107 | #issuetracker_project = "celery" 108 | #issuetracker_issue_pattern = r'[Ii]ssue #(\d+)' 109 | -------------------------------------------------------------------------------- /docs/images/celery-icon-128.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/celery/cyme/c49047318c56f6674bd6d6ea6673b48be4eb7886/docs/images/celery-icon-128.png -------------------------------------------------------------------------------- /docs/images/celery-icon-32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/celery/cyme/c49047318c56f6674bd6d6ea6673b48be4eb7886/docs/images/celery-icon-32.png -------------------------------------------------------------------------------- /docs/images/celery-icon-64.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/celery/cyme/c49047318c56f6674bd6d6ea6673b48be4eb7886/docs/images/celery-icon-64.png -------------------------------------------------------------------------------- /docs/images/celery_favicon_128.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/celery/cyme/c49047318c56f6674bd6d6ea6673b48be4eb7886/docs/images/celery_favicon_128.png -------------------------------------------------------------------------------- /docs/images/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/celery/cyme/c49047318c56f6674bd6d6ea6673b48be4eb7886/docs/images/favicon.ico -------------------------------------------------------------------------------- /docs/images/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/celery/cyme/c49047318c56f6674bd6d6ea6673b48be4eb7886/docs/images/favicon.png -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | ================================ 2 | Cyme - Celery instance manager 3 | ================================ 4 | 5 | Contents: 6 | 7 | .. toctree:: 8 | :maxdepth: 1 9 | 10 | introduction 11 | changelog 12 | reference/index 13 | 14 | 15 | Indices and tables 16 | ================== 17 | 18 | * :ref:`genindex` 19 | * :ref:`modindex` 20 | * :ref:`search` 21 | 22 | -------------------------------------------------------------------------------- /docs/reference/cyme.api.views.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | cyme.api.views 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cyme.api.views 8 | 9 | .. automodule:: cyme.api.views 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cyme.api.web.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | cyme.api.web 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cyme.api.web 8 | 9 | .. automodule:: cyme.api.web 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cyme.bin.base.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | cyme.bin.base 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cyme.bin.base 8 | 9 | .. automodule:: cyme.bin.base 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cyme.bin.cyme.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | cyme.bin.cyme 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cyme.bin.cyme 8 | 9 | .. automodule:: cyme.bin.cyme 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cyme.bin.cyme_branch.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | cyme.bin.cyme_branch 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cyme.bin.cyme_branch 8 | 9 | .. automodule:: cyme.bin.cyme_branch 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cyme.branch.controller.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | cyme.branch.controller 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cyme.branch.controller 8 | 9 | .. automodule:: cyme.branch.controller 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cyme.branch.httpd.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | cyme.branch.httpd 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cyme.branch.httpd 8 | 9 | .. automodule:: cyme.branch.httpd 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cyme.branch.intsup.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | cyme.branch.intsup 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cyme.branch.intsup 8 | 9 | .. automodule:: cyme.branch.intsup 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cyme.branch.managers.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | cyme.branch.managers 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cyme.branch.managers 8 | 9 | .. automodule:: cyme.branch.managers 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cyme.branch.metrics.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | cyme.branch.metrics 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cyme.branch.metrics 8 | 9 | .. automodule:: cyme.branch.metrics 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cyme.branch.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | cyme.branch 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cyme.branch 8 | 9 | .. automodule:: cyme.branch 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cyme.branch.signals.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | cyme.branch.signals 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cyme.branch.signals 8 | 9 | .. automodule:: cyme.branch.signals 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cyme.branch.state.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | cyme.branch.state 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cyme.branch.state 8 | 9 | .. automodule:: cyme.branch.state 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cyme.branch.supervisor.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | cyme.branch.supervisor 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cyme.branch.supervisor 8 | 9 | .. automodule:: cyme.branch.supervisor 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cyme.branch.thread.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | cyme.branch.thread 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cyme.branch.thread 8 | 9 | .. automodule:: cyme.branch.thread 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cyme.client.base.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | cyme.client.base 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cyme.client.base 8 | 9 | .. automodule:: cyme.client.base 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cyme.client.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | cyme.client 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cyme.client 8 | 9 | .. automodule:: cyme.client 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cyme.management.commands.cyme.rst: -------------------------------------------------------------------------------- 1 | =================================== 2 | cyme.management.commands.cyme 3 | =================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cyme.management.commands.cyme 8 | 9 | .. automodule:: cyme.management.commands.cyme 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cyme.management.commands.cyme_branch.rst: -------------------------------------------------------------------------------- 1 | ====================================== 2 | cyme.management.commands.cyme_branch 3 | ====================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cyme.management.commands.cyme_branch 8 | 9 | .. automodule:: cyme.management.commands.cyme_branch 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cyme.models.managers.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | cyme.models.managers 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cyme.models.managers 8 | 9 | .. automodule:: cyme.models.managers 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cyme.models.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | cyme.models 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cyme.models 8 | 9 | .. automodule:: cyme.models 10 | 11 | .. autoclass:: Broker 12 | 13 | .. attribute:: url 14 | 15 | AMQP (kombu) url. 16 | 17 | .. method:: connection 18 | 19 | Return a new :class:`~kombu.Connection` to this broker. 20 | 21 | .. automethod:: as_dict 22 | 23 | .. attribute:: pool 24 | 25 | A connection pool with connections to this broker. 26 | 27 | .. attribute:: objects 28 | 29 | The manager for this model is :class:`~cyme.managers.BrokerManager`. 30 | 31 | 32 | .. autoclass:: Queue 33 | 34 | .. attribute:: name 35 | 36 | Queue name (unique, max length 128) 37 | 38 | .. attribute:: exchange 39 | 40 | Exchange name (max length 128) 41 | 42 | .. attribute:: exchange_type 43 | 44 | Exchange type (max length 128) 45 | 46 | .. attribute:: routing_key 47 | 48 | Routing key/binding key (max length 128). 49 | 50 | .. attribute:: options 51 | 52 | Additional JSON encoded queue/exchange/binding options. 53 | see :mod:`kombu.compat` for a list of options supported. 54 | 55 | .. attribute:: is_enabled 56 | 57 | Not in use. 58 | 59 | .. attribute:: created_at 60 | 61 | Timestamp created. 62 | 63 | .. automethod:: as_dict 64 | 65 | .. attribute:: objects 66 | 67 | The manager for this model is :class:`~cyme.managers.QueueManager`. 68 | 69 | 70 | .. autoclass:: Instance 71 | 72 | .. attribute:: name 73 | 74 | Name of the instance. 75 | 76 | .. attribute:: queues 77 | 78 | Queues this instance should consume from (many to many 79 | relation to :class:`Queue`). 80 | 81 | .. attribute:: max_concurrency 82 | 83 | Autoscale setting for max concurrency. 84 | (maximum number of processes/threads/green threads when the 85 | worker is active). 86 | 87 | .. attribute:: min_concurrency 88 | 89 | Autoscale setting for min concurrency. 90 | (minimum number of processes/threads/green threads when 91 | the worker is idle). 92 | 93 | .. attribute:: is_enabled 94 | 95 | Flag set if this instance should be running. 96 | 97 | .. attribute:: created_at 98 | 99 | Timestamp of when this instance was first created. 100 | 101 | .. attribute:: broker 102 | 103 | The broker this instance should connect to. 104 | (foreign key to :class:`Broker`). 105 | 106 | .. attribute:: Broker 107 | 108 | Broker model class used (default is :class:`Broker`) 109 | 110 | .. attribute:: Queue 111 | 112 | Queue model class used (default is :class:`Queue`) 113 | 114 | .. attribute:: MultiTool 115 | 116 | Class used to start/stop and restart celeryd instances. 117 | (Default is :class:`celery.bin.celeryd_multi.MultiTool`). 118 | 119 | .. attribute:: objects 120 | 121 | The manager used for this model is 122 | :class:`~cyme.managers.InstanceManager`. 123 | 124 | .. attribute:: cwd 125 | 126 | Working directory used by all instances. 127 | (Default is :file:`/var/run/cyme`). 128 | 129 | 130 | .. automethod:: as_dict 131 | 132 | .. automethod:: enable 133 | 134 | .. automethod:: disable 135 | 136 | .. automethod:: start 137 | 138 | .. automethod:: stop 139 | 140 | .. automethod:: restart 141 | 142 | .. automethod:: alive 143 | 144 | .. automethod:: stats 145 | 146 | .. automethod:: autoscale 147 | 148 | .. automethod:: responds_to_ping 149 | 150 | .. automethod:: responds_to_signal 151 | 152 | .. automethod:: consuming_from 153 | 154 | .. automethod:: add_queue 155 | 156 | .. automethod:: cancel_queue 157 | 158 | .. automethod:: getpid 159 | 160 | .. automethod:: _action 161 | 162 | .. automethod:: _query 163 | -------------------------------------------------------------------------------- /docs/reference/cyme.status.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | cyme.status 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cyme.status 8 | 9 | .. automodule:: cyme.status 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cyme.tasks.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | cyme.tasks 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cyme.tasks 8 | 9 | .. automodule:: cyme.tasks 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cyme.utils.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | cyme.utils 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cyme.utils 8 | 9 | .. automodule:: cyme.utils 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/index.rst: -------------------------------------------------------------------------------- 1 | =============== 2 | API Reference 3 | =============== 4 | 5 | :Release: |version| 6 | :Date: |today| 7 | 8 | .. toctree:: 9 | :maxdepth: 1 10 | 11 | cyme.client 12 | cyme.client.base 13 | cyme.branch 14 | cyme.branch.controller 15 | cyme.branch.managers 16 | cyme.branch.supervisor 17 | cyme.branch.httpd 18 | cyme.branch.signals 19 | cyme.branch.state 20 | cyme.branch.metrics 21 | cyme.branch.thread 22 | cyme.branch.intsup 23 | cyme.api.views 24 | cyme.api.web 25 | cyme.models 26 | cyme.models.managers 27 | cyme.status 28 | cyme.tasks 29 | cyme.management.commands.cyme 30 | cyme.management.commands.cyme_branch 31 | cyme.bin.base 32 | cyme.bin.cyme 33 | cyme.bin.cyme_branch 34 | cyme.utils 35 | -------------------------------------------------------------------------------- /funtests/setup.cfg: -------------------------------------------------------------------------------- 1 | [nosetests] 2 | verbosity = 1 3 | detailed-errors = 1 4 | where = suite 5 | -------------------------------------------------------------------------------- /funtests/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | try: 5 | from setuptools import setup 6 | from setuptools.command.install import install 7 | except ImportError: 8 | from ez_setup import use_setuptools 9 | use_setuptools() 10 | from setuptools import setup 11 | from setuptools.command.install import install 12 | 13 | import os 14 | import sys 15 | 16 | sys.path.insert(0, os.getcwd()) 17 | sys.path.insert(0, os.path.join(os.getcwd(), os.pardir)) 18 | import suite 19 | 20 | class no_install(install): 21 | 22 | def run(self, *args, **kwargs): 23 | import sys 24 | sys.stderr.write(""" 25 | ------------------------------------------------------ 26 | The Cyme functional test suite cannot be installed. 27 | ------------------------------------------------------ 28 | 29 | 30 | But you can execute the tests by running the command: 31 | 32 | $ python setup.py test 33 | 34 | 35 | """) 36 | 37 | 38 | setup( 39 | name='cyme-funtests', 40 | version="DEV", 41 | description="Functional test suite for Cyme", 42 | author="Ask Solem", 43 | author_email="ask@rabbitmq.com", 44 | url="--", 45 | platforms=["any"], 46 | packages=[], 47 | data_files=[], 48 | zip_safe=False, 49 | cmdclass={"install": no_install}, 50 | test_suite="nose.collector", 51 | tests_require=[ 52 | "unittest2>=0.4.0", 53 | "simplejson", 54 | "nose", 55 | ], 56 | classifiers=[ 57 | "Operating System :: OS Independent", 58 | "Programming Language :: Python", 59 | "License :: OSI Approved :: BSD License", 60 | "Intended Audience :: Developers", 61 | ], 62 | long_description="Do not install this package", 63 | ) 64 | -------------------------------------------------------------------------------- /funtests/suite/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | DB_NAME = os.environ["CYME_DB_NAME"] = "funtest.db" 5 | 6 | sys.path.insert(0, os.path.join(os.getcwd(), os.pardir)) 7 | 8 | 9 | def teardown(): 10 | if os.path.exists(DB_NAME): 11 | os.unlink(DB_NAME) 12 | -------------------------------------------------------------------------------- /funtests/suite/test_basic.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | # funtest config 5 | sys.path.insert(0, os.getcwd()) 6 | sys.path.insert(0, os.path.join(os.getcwd(), os.pardir)) 7 | 8 | from cyme.bin.base import app, Env 9 | from cyme.utils import cached_property, Path, uuid 10 | from eventlet.event import Event 11 | from nose import SkipTest 12 | 13 | from .utils import unittest 14 | 15 | CYME_PORT = int(os.environ.get("CYME_PORT") or 8013) 16 | CYME_URL = "http://127.0.0.1:%s" % (CYME_PORT, ) 17 | CYME_INSTANCE_DIR = Path("instances").absolute() 18 | 19 | _branch = [None] 20 | 21 | 22 | @app(needs_eventlet=True, instance_dir=CYME_INSTANCE_DIR) 23 | def start_branch(env, argv=None): 24 | env.syncdb(interactive=False) 25 | from cyme.branch import Branch 26 | ready_event = Event() 27 | CYME_INSTANCE_DIR.mkdir() 28 | instance = Branch("127.0.0.1:%s" % (CYME_PORT, ), numc=1, 29 | ready_event=ready_event) 30 | instance.start() 31 | ready_event.wait() 32 | return instance 33 | 34 | 35 | def destroy_branch(branch): 36 | branch.stop() 37 | 38 | 39 | def teardown(): 40 | if _branch[0] is not None: 41 | destroy_branch(_branch[0]) 42 | if CYME_INSTANCE_DIR.isdir(): 43 | CYME_INSTANCE_DIR.rmtree() 44 | 45 | 46 | class ClientTestCase(unittest.TestCase): 47 | 48 | @cached_property 49 | def Client(self): 50 | from cyme import Client 51 | return Client 52 | 53 | 54 | class BranchTestCase(unittest.TestCase): 55 | 56 | def setUp(self): 57 | if _branch[0] is None: 58 | _branch[0] = start_branch() 59 | 60 | 61 | class test_create_app(BranchTestCase, ClientTestCase): 62 | 63 | def test_create(self): 64 | client = self.Client(CYME_URL) 65 | app = client.add(uuid()) 66 | self.assertTrue(repr(app)) 67 | self.assertTrue(app) 68 | self.assertTrue(app.info) 69 | self.assertTrue(app.info.broker) 70 | self.assertIn(app.app, app.all()) 71 | app = client.get(app.app) 72 | self.assertTrue(app) 73 | app.delete() 74 | self.assertNotIn(app.app, app.all()) 75 | 76 | 77 | class test_basic(BranchTestCase, ClientTestCase): 78 | 79 | def setUp(self): 80 | BranchTestCase.setUp(self) 81 | self.app = self.Client(CYME_URL).add(uuid()) 82 | 83 | def tearDown(self): 84 | self.app.delete() 85 | BranchTestCase.tearDown(self) 86 | 87 | def test_basic(self): 88 | app = self.app 89 | instance = app.instances.add() 90 | self.assertTrue(repr(instance)) 91 | self.assertTrue(instance) 92 | instance = app.instances.get(instance.name) 93 | self.assertTrue(instance) 94 | self.assertIn(instance, app.instances) 95 | 96 | q = uuid() 97 | expected = dict(exchange=q, exchange_type="topic", routing_key=q) 98 | queue = app.queues.add(q, **expected) 99 | self.assertTrue(repr(queue)) 100 | queue = app.queues.get(queue.name) 101 | self.assertTrue(queue) 102 | for key, value in expected.items(): 103 | self.assertEqual(getattr(queue, key), value) 104 | self.assertIn(queue, app.queues) 105 | 106 | self.assertTrue(instance.consumers.add(queue)) 107 | self.assertTrue(instance.consumers.delete(queue)) 108 | 109 | queue.delete() 110 | self.assertNotIn(queue, app.queues) 111 | 112 | instance.delete() 113 | self.assertNotIn(instance, app.instances) 114 | 115 | 116 | if __name__ == "__main__": 117 | unittest.main() 118 | -------------------------------------------------------------------------------- /funtests/suite/utils.py: -------------------------------------------------------------------------------- 1 | try: 2 | import unittest 3 | unittest.skip 4 | except AttributeError: 5 | import unittest2 as unittest 6 | 7 | -------------------------------------------------------------------------------- /pavement.py: -------------------------------------------------------------------------------- 1 | from paver.easy import * # noqa 2 | from paver import doctools # noqa 3 | from paver.setuputils import setup # noqa 4 | 5 | options( 6 | sphinx=Bunch(builddir=".build"), 7 | ) 8 | 9 | 10 | def sphinx_builddir(options): 11 | return path("docs") / options.sphinx.builddir / "html" 12 | 13 | 14 | @task 15 | def clean_docs(options): 16 | sphinx_builddir(options).rmtree() 17 | 18 | 19 | @task 20 | @needs("clean_docs", "paver.doctools.html") 21 | def html(options): 22 | destdir = path("Documentation") 23 | destdir.rmtree() 24 | builtdocs = sphinx_builddir(options) 25 | builtdocs.move(destdir) 26 | 27 | 28 | @task 29 | @needs("paver.doctools.html") 30 | def qhtml(options): 31 | destdir = path("Documentation") 32 | builtdocs = sphinx_builddir(options) 33 | sh("rsync -az %s/ %s" % (builtdocs, destdir)) 34 | 35 | 36 | @task 37 | @needs("clean_docs", "paver.doctools.html") 38 | def ghdocs(options): 39 | builtdocs = sphinx_builddir(options) 40 | sh("git checkout gh-pages && \ 41 | cp -r %s/* . && \ 42 | git commit . -m 'Rendered documentation for Github Pages.' && \ 43 | git push origin gh-pages && \ 44 | git checkout master" % builtdocs) 45 | 46 | 47 | @task 48 | @needs("clean_docs", "paver.doctools.html") 49 | def upload_pypi_docs(options): 50 | builtdocs = path("docs") / options.builddir / "html" 51 | sh("python setup.py upload_sphinx --upload-dir='%s'" % (builtdocs)) 52 | 53 | 54 | @task 55 | @needs("upload_pypi_docs", "ghdocs") 56 | def upload_docs(options): 57 | pass 58 | 59 | 60 | @task 61 | def autodoc(options): 62 | sh("contrib/release/doc4allmods cyme") 63 | 64 | 65 | @task 66 | def verifyindex(options): 67 | sh("contrib/release/verify-reference-index.sh") 68 | 69 | 70 | @task 71 | def clean_readme(options): 72 | path("README").unlink() 73 | path("README.rst").unlink() 74 | 75 | 76 | @task 77 | @needs("clean_readme") 78 | def readme(options): 79 | sh("python contrib/release/sphinx-to-rst.py docs/templates/readme.txt \ 80 | > README.rst") 81 | sh("ln -sf README.rst README") 82 | 83 | 84 | @task 85 | def bump(options): 86 | sh("contrib/release/bump_version.py cyme/__init__.py README.rst") 87 | 88 | 89 | @task 90 | @cmdopts([ 91 | ("coverage", "c", "Enable coverage"), 92 | ("quick", "q", "Quick test"), 93 | ("verbose", "V", "Make more noise"), 94 | ]) 95 | def test(options): 96 | cmd = "nosetests" 97 | if getattr(options, "coverage", False): 98 | cmd += " --with-coverage3" 99 | if getattr(options, "quick", False): 100 | cmd = "QUICKTEST=1 SKIP_RLIMITS=1 %s" % cmd 101 | if getattr(options, "verbose", False): 102 | cmd += " --verbosity=2" 103 | sh(cmd) 104 | 105 | 106 | @task 107 | @cmdopts([ 108 | ("noerror", "E", "Ignore errors"), 109 | ]) 110 | def flake8(options): 111 | noerror = getattr(options, "noerror", False) 112 | sh("""flake8 cyme""", ignore_error=noerror) 113 | 114 | 115 | @task 116 | @cmdopts([ 117 | ("noerror", "E", "Ignore errors"), 118 | ]) 119 | def flakeplus(options): 120 | noerror = getattr(options, "noerror", False) 121 | sh("python contrib/release/flakeplus.py cyme", 122 | ignore_error=noerror) 123 | 124 | 125 | @task 126 | @cmdopts([ 127 | ("noerror", "E", "Ignore errors"), 128 | ]) 129 | def flakes(options): 130 | flake8(options) 131 | flakeplus(options) 132 | 133 | 134 | @task 135 | @cmdopts([ 136 | ("noerror", "E", "Ignore errors"), 137 | ]) 138 | def pep8(options): 139 | noerror = getattr(options, "noerror", False) 140 | return sh("""find cyme -name "*.py" | xargs pep8 | perl -nle'\ 141 | print; $a=1 if $_}{exit($a)'""", ignore_error=noerror) 142 | 143 | 144 | @task 145 | def removepyc(options): 146 | sh("find . -name '*.pyc' | xargs rm") 147 | 148 | 149 | @task 150 | @needs("removepyc") 151 | def gitclean(options): 152 | sh("git clean -xdn") 153 | 154 | 155 | @task 156 | @needs("removepyc") 157 | def gitcleanforce(options): 158 | sh("git clean -xdf") 159 | 160 | 161 | @task 162 | @needs("flakes", "autodoc", "verifyindex", "test", "gitclean") 163 | def releaseok(options): 164 | pass 165 | 166 | 167 | @task 168 | @needs("releaseok", "removepyc", "upload_docs") 169 | def release(options): 170 | pass 171 | -------------------------------------------------------------------------------- /requirements/default.txt: -------------------------------------------------------------------------------- 1 | cell 2 | eventlet 3 | dnspython 4 | django<1.6 5 | django-celery<3.1 6 | celery<4 7 | vine 8 | requests 9 | dictshield 10 | progressbar 11 | unipath 12 | -------------------------------------------------------------------------------- /requirements/docs.txt: -------------------------------------------------------------------------------- 1 | Sphinx 2 | sphinxcontrib-issuetracker 3 | -------------------------------------------------------------------------------- /requirements/test.txt: -------------------------------------------------------------------------------- 1 | unittest2>=0.4.0 2 | coverage>=3.0 3 | nose 4 | nose-cover3 5 | django-nose 6 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [build_sphinx] 2 | source-dir = docs/ 3 | build-dir = docs/.build 4 | all_files = 1 5 | 6 | [upload_sphinx] 7 | upload-dir = docs/.build/html 8 | 9 | [bdist_rpm] 10 | requires = cl 11 | eventlet 12 | dnspython 13 | django 14 | django-celery 15 | requests 16 | progressbar 17 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | import os 4 | import sys 5 | import codecs 6 | 7 | try: 8 | from setuptools import setup, Command 9 | except ImportError: 10 | from ez_setup import use_setuptools 11 | use_setuptools() 12 | from setuptools import setup, Command # noqa 13 | from distutils.command.install import INSTALL_SCHEMES 14 | 15 | os.environ["CYME_NO_EVAL"] = "yes" 16 | import cyme as distmeta 17 | os.environ.pop("CYME_NO_EVAL", None) 18 | sys.modules.pop("cyme", None) 19 | 20 | packages, data_files = [], [] 21 | root_dir = os.path.dirname(__file__) 22 | if root_dir != '': 23 | os.chdir(root_dir) 24 | src_dir = "cyme" 25 | 26 | 27 | def fullsplit(path, result=None): 28 | if result is None: 29 | result = [] 30 | head, tail = os.path.split(path) 31 | if head == '': 32 | return [tail] + result 33 | if head == path: 34 | return result 35 | return fullsplit(head, [tail] + result) 36 | 37 | 38 | for scheme in INSTALL_SCHEMES.values(): 39 | scheme['data'] = scheme['purelib'] 40 | 41 | SKIP_EXTENSIONS = [".pyc", ".pyo", ".swp", ".swo"] 42 | 43 | 44 | def is_unwanted_file(filename): 45 | for skip_ext in SKIP_EXTENSIONS: 46 | if filename.endswith(skip_ext): 47 | return True 48 | return False 49 | 50 | for dirpath, dirnames, filenames in os.walk(src_dir): 51 | # Ignore dirnames that start with '.' 52 | for i, dirname in enumerate(dirnames): 53 | if dirname.startswith("."): 54 | del dirnames[i] 55 | for filename in filenames: 56 | if filename.endswith(".py"): 57 | packages.append('.'.join(fullsplit(dirpath))) 58 | elif is_unwanted_file(filename): 59 | pass 60 | else: 61 | data_files.append([dirpath, [os.path.join(dirpath, f) for f in 62 | filenames]]) 63 | 64 | 65 | class RunTests(Command): 66 | description = "Run the django test suite from the tests dir." 67 | user_options = [] 68 | extra_env = {} 69 | extra_args = [] 70 | 71 | def run(self): 72 | for env_name, env_value in self.extra_env.items(): 73 | os.environ[env_name] = str(env_value) 74 | 75 | this_dir = os.getcwd() 76 | testproj_dir = os.path.join(this_dir, "tests") 77 | os.chdir(testproj_dir) 78 | sys.path.append(testproj_dir) 79 | from django.core.management import execute_manager 80 | os.environ["DJANGO_SETTINGS_MODULE"] = os.environ.get( 81 | "DJANGO_SETTINGS_MODULE", "settings") 82 | settings_file = os.environ["DJANGO_SETTINGS_MODULE"] 83 | settings_mod = __import__(settings_file, {}, {}, ['']) 84 | prev_argv = list(sys.argv) 85 | try: 86 | sys.argv = [__file__, "test"] + self.extra_args 87 | execute_manager(settings_mod, argv=sys.argv) 88 | finally: 89 | sys.argv = prev_argv 90 | 91 | def initialize_options(self): 92 | pass 93 | 94 | def finalize_options(self): 95 | pass 96 | 97 | 98 | class CIRunTests(RunTests): 99 | extra_args = ["--with-coverage3", "--with-xunit", 100 | "--cover3-xml", "--xunit-file=nosetests.xml", 101 | "--cover3-xml-file=coverage.xml"] 102 | 103 | 104 | if os.path.exists("README.rst"): 105 | long_description = codecs.open("README.rst", "r", "utf-8").read() 106 | else: 107 | long_description = "See http://github.com/celery/cyme" 108 | 109 | 110 | setup( 111 | name='cyme', 112 | version=distmeta.__version__, 113 | description=distmeta.__doc__, 114 | author=distmeta.__author__, 115 | author_email=distmeta.__contact__, 116 | url=distmeta.__homepage__, 117 | platforms=["any"], 118 | license="BSD", 119 | packages=packages, 120 | data_files=data_files, 121 | zip_safe=False, 122 | install_requires=[ 123 | "cell", 124 | "eventlet", 125 | "dnspython", 126 | "Django", 127 | "django-celery>=2.3.1", 128 | "requests", 129 | "dictshield", 130 | "progressbar", 131 | "unipath", 132 | ], 133 | cmdclass={"test": RunTests, 134 | "citest": CIRunTests}, 135 | classifiers=[ 136 | "Development Status :: 4 - Beta", 137 | "Framework :: Django", 138 | "Operating System :: OS Independent", 139 | "Programming Language :: Python", 140 | "Intended Audience :: Developers", 141 | "License :: OSI Approved :: BSD License", 142 | "Operating System :: POSIX", 143 | "Topic :: Communications", 144 | "Topic :: System :: Distributed Computing", 145 | "Topic :: Software Development :: Libraries :: Python Modules", 146 | ], 147 | entry_points={ 148 | "console_scripts": [ 149 | "cyme-branch = cyme.bin.cyme_branch:cyme_branch", 150 | "cyme = cyme.bin.cyme:cyme", 151 | "cyme-list-branches = cyme.bin.cyme_list_branches:main"], 152 | }, 153 | long_description=long_description, 154 | ) 155 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/celery/cyme/c49047318c56f6674bd6d6ea6673b48be4eb7886/tests/__init__.py -------------------------------------------------------------------------------- /tests/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from django.core.management import execute_manager 3 | try: 4 | import settings # Assumed to be in the same directory. 5 | except ImportError: 6 | import sys 7 | sys.stderr.write( 8 | "Error: Can't find the file 'settings.py' in the directory \ 9 | containing %r. It appears you've customized things.\n\ 10 | You'll have to run django-admin.py, passing it your settings\ 11 | module.\n(If the file settings.py does indeed exist, it's\ 12 | causing an ImportError somehow.)\n" % __file__) 13 | sys.exit(1) 14 | 15 | if __name__ == "__main__": 16 | execute_manager(settings) 17 | -------------------------------------------------------------------------------- /tests/settings.py: -------------------------------------------------------------------------------- 1 | # Django settings for testproj project. 2 | 3 | import os 4 | import sys 5 | # import source code dir 6 | sys.path.insert(0, os.getcwd()) 7 | sys.path.insert(0, os.path.join(os.getcwd(), os.pardir)) 8 | 9 | SITE_ID = 300 10 | 11 | DEBUG = True 12 | TEMPLATE_DEBUG = DEBUG 13 | 14 | ROOT_URLCONF = "tests.urls" 15 | 16 | ADMINS = ( 17 | # ('Your Name', 'your_email@domain.com'), 18 | ) 19 | 20 | TEST_RUNNER = "django_nose.run_tests" 21 | here = os.path.abspath(os.path.dirname(__file__)) 22 | COVERAGE_EXCLUDE_MODULES = ("cyme.tests.*", ) 23 | 24 | NOSE_ARGS = [os.path.join(here, os.pardir, "cyme", "tests"), 25 | os.environ.get("NOSE_VERBOSE") and "--verbose" or "", 26 | "--cover3-package=cyme", 27 | "--cover3-branch", 28 | "--cover3-exclude=%s" % ",".join(COVERAGE_EXCLUDE_MODULES)] 29 | 30 | BROKER_HOST = "localhost" 31 | BROKER_PORT = 5672 32 | BROKER_VHOST = "/" 33 | BROKER_USER = "guest" 34 | BROKER_PASSWORD = "guest" 35 | 36 | TT_HOST = "localhost" 37 | TT_PORT = 1978 38 | 39 | CELERY_DEFAULT_EXCHANGE = "test.cyme" 40 | CELERY_DEFAULT_ROUTING_KEY = "test.cyme" 41 | CELERY_DEFAULT_QUEUE = "test.cyme" 42 | 43 | CELERY_QUEUES = {"test.cyme": {"binding_key": "test.cyme"}} 44 | 45 | MANAGERS = ADMINS 46 | 47 | DATABASES = { 48 | "default": { 49 | "ENGINE": "django.db.backends.sqlite3", 50 | "NAME": "cyme-test.db", 51 | }, 52 | } 53 | 54 | INSTALLED_APPS = ( 55 | 'django.contrib.auth', 56 | 'django.contrib.contenttypes', 57 | 'django.contrib.sessions', 58 | 'django.contrib.sites', 59 | 'django_nose', 60 | 'cyme', 61 | 'cyme.api', 62 | ) 63 | 64 | CELERY_SEND_TASK_ERROR_EMAILS = False 65 | --------------------------------------------------------------------------------