├── push ├── __init__.py ├── hosts │ ├── mock.py │ ├── dns.py │ ├── zookeeper.py │ └── __init__.py ├── utils.py ├── syslog.py ├── irc.py ├── main.py ├── log.py ├── cli.py ├── ssh.py ├── deploy.py ├── config.py └── args.py ├── .gitignore ├── README.md ├── setup.py ├── etc └── push.ini.example └── LICENSE /push/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.swp 3 | etc/push.ini 4 | push.egg-info 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # This project has been obsoleted 2 | 3 | reddit uses [rollingpin](https://github.com/reddit/rollingpin) for deployment now. 4 | -------------------------------------------------------------------------------- /push/hosts/mock.py: -------------------------------------------------------------------------------- 1 | from push.hosts import HostSource 2 | 3 | 4 | class MockHostSource(HostSource): 5 | def __init__(self, config): 6 | self.host_count = config.hosts.mock.host_count 7 | 8 | def get_all_hosts(self): 9 | return ["app-%02d" % i for i in xrange(self.host_count)] 10 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | from setuptools import setup 4 | 5 | setup( 6 | name="push", 7 | version="", 8 | packages=["push"], 9 | install_requires=[ 10 | "wessex>=1.5", 11 | "paramiko", 12 | ], 13 | extras_require={ 14 | "DNS": [ 15 | "dnspython", 16 | ], 17 | "ZooKeeper": [ 18 | "kazoo", 19 | ], 20 | }, 21 | entry_points={ 22 | "console_scripts": [ 23 | "push = push.main:main", 24 | ] 25 | } 26 | ) 27 | -------------------------------------------------------------------------------- /etc/push.ini.example: -------------------------------------------------------------------------------- 1 | [ssh] 2 | user = super-deploy-user 3 | key_filename = /some/path/something.key 4 | strict_host_key_checking = true 5 | timeout = 10 6 | 7 | [deploy] 8 | build_host = localhost 9 | build_binary = /your/mother 10 | deploy_binary = /smells/of/elderberries 11 | 12 | [paths] 13 | log_root = /var/log/push/ 14 | 15 | [syslog] 16 | ident = deploy 17 | facility = LOCAL4 18 | priority = NOTICE 19 | 20 | [hosts] 21 | source = mock 22 | 23 | [hosts:mock] 24 | host_count = 30 25 | 26 | [aliases] 27 | apps = app-* 28 | something = @apps 29 | -------------------------------------------------------------------------------- /push/utils.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import os 3 | import random 4 | 5 | 6 | def get_random_word(config): 7 | file_size = os.path.getsize(config.paths.wordlist) 8 | word = "" 9 | 10 | with open(config.paths.wordlist, "r") as wordlist: 11 | while not word.isalpha() or not word.islower() or len(word) < 5: 12 | position = random.randint(1, file_size) 13 | wordlist.seek(position) 14 | wordlist.readline() 15 | word = unicode(wordlist.readline().rstrip("\n"), 'utf-8') 16 | 17 | return word 18 | 19 | 20 | def seeded_shuffle(seedword, list): 21 | list.sort(key=lambda h: hashlib.md5(seedword + h).hexdigest()) 22 | -------------------------------------------------------------------------------- /push/syslog.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import syslog 4 | import getpass 5 | 6 | 7 | def register(config, args, deployer, log): 8 | def write_syslog(message): 9 | syslog.syslog(config.syslog.priority, message.encode('utf-8')) 10 | 11 | syslog.openlog(ident=config.syslog.ident, facility=config.syslog.facility) 12 | 13 | @deployer.push_began 14 | def on_push_began(deployer): 15 | user = getpass.getuser() 16 | write_syslog('Push %s started by ' 17 | '%s with args "%s"' % (args.push_id, user, 18 | args.command_line)) 19 | 20 | @deployer.push_ended 21 | def on_push_ended(deployer): 22 | write_syslog("Push %s complete!" % args.push_id) 23 | 24 | @deployer.push_aborted 25 | def on_push_aborted(deployer, exception): 26 | write_syslog("Push %s aborted (%s)" % (args.push_id, exception)) 27 | -------------------------------------------------------------------------------- /push/hosts/dns.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import dns.name 4 | import dns.zone 5 | import dns.query 6 | import dns.exception 7 | import dns.resolver 8 | import dns.rdtypes 9 | 10 | from push.hosts import HostSource, HostLookupError 11 | 12 | 13 | class DnsHostSource(HostSource): 14 | def __init__(self, config): 15 | self.domain = config.hosts.dns.domain 16 | 17 | def get_all_hosts(self): 18 | """Pull all hosts from DNS by doing a zone transfer.""" 19 | 20 | try: 21 | soa_answer = dns.resolver.query(self.domain, "SOA", tcp=True) 22 | soa_host = soa_answer[0].mname 23 | 24 | master_answer = dns.resolver.query(soa_host, "A", tcp=True) 25 | master_addr = master_answer[0].address 26 | 27 | xfr_answer = dns.query.xfr(master_addr, self.domain) 28 | zone = dns.zone.from_xfr(xfr_answer) 29 | return [name.to_text() 30 | for name, ttl, rdata in zone.iterate_rdatas("A")] 31 | except dns.exception.DNSException, e: 32 | raise HostLookupError("host lookup by dns failed: %r" % e) 33 | -------------------------------------------------------------------------------- /push/irc.py: -------------------------------------------------------------------------------- 1 | import wessex 2 | import getpass 3 | 4 | 5 | def register(config, args, deployer, log): 6 | if not args.notify_irc: 7 | return 8 | 9 | harold = wessex.connect_harold() 10 | monitor = harold.get_deploy(args.push_id) 11 | 12 | def log_exception_and_continue(fn): 13 | def wrapper(*args, **kwargs): 14 | try: 15 | return fn(*args, **kwargs) 16 | except Exception, e: 17 | log.warning("Harold error: %s", e) 18 | return wrapper 19 | 20 | @deployer.push_began 21 | @log_exception_and_continue 22 | def on_push_began(deployer): 23 | monitor.begin(getpass.getuser(), args.command_line, 24 | log.log_path, len(args.hosts)) 25 | 26 | @deployer.process_host_ended 27 | @log_exception_and_continue 28 | def on_process_host_ended(deployer, host): 29 | index = args.hosts.index(host) + 1 30 | monitor.progress(host, index) 31 | 32 | @deployer.push_ended 33 | @log_exception_and_continue 34 | def on_push_ended(deployer): 35 | monitor.end() 36 | 37 | @deployer.push_aborted 38 | @log_exception_and_continue 39 | def on_push_aborted(deployer, e): 40 | monitor.abort(str(e)) 41 | 42 | @deployer.prompt_error_began 43 | @log_exception_and_continue 44 | def on_prompt_error_began(deployer, host, error): 45 | monitor.error("%s: %s" % (host, error)) 46 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012, reddit Inc. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | * Redistributions of source code must retain the above copyright 7 | notice, this list of conditions and the following disclaimer. 8 | * Redistributions in binary form must reproduce the above copyright 9 | notice, this list of conditions and the following disclaimer in the 10 | documentation and/or other materials provided with the distribution. 11 | * Neither the name of the nor the 12 | names of its contributors may be used to endorse or promote products 13 | derived from this software without specific prior written permission. 14 | 15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 16 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 17 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 | DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY 19 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 20 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 21 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 22 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 24 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | -------------------------------------------------------------------------------- /push/hosts/zookeeper.py: -------------------------------------------------------------------------------- 1 | from kazoo.client import KazooClient 2 | from kazoo.exceptions import KazooException, NoNodeException 3 | from kazoo.retry import KazooRetry 4 | 5 | from push.hosts import HostSource, HostLookupError 6 | 7 | 8 | class ZookeeperHostSource(HostSource): 9 | def __init__(self, config): 10 | self.zk = KazooClient(config.hosts.zookeeper.connection_string) 11 | self.zk.start() 12 | credentials = ":".join((config.hosts.zookeeper.username, 13 | config.hosts.zookeeper.password)) 14 | self.zk.add_auth("digest", credentials) 15 | self.retry = KazooRetry(max_tries=3) 16 | 17 | def get_all_hosts(self): 18 | try: 19 | return self.retry(self.zk.get_children, "/server") 20 | except KazooException as e: 21 | raise HostLookupError("zk host enumeration failed: %r", e) 22 | 23 | def should_host_be_alive(self, host_name): 24 | try: 25 | host_root = "/server/" + host_name 26 | 27 | state = self.retry(self.zk.get, host_root + "/state")[0] 28 | if state in ("kicking", "unhealthy"): 29 | return False 30 | 31 | is_autoscaled = self.retry(self.zk.exists, host_root + "/asg") 32 | is_running = self.retry(self.zk.exists, host_root + "/running") 33 | return not is_autoscaled or is_running 34 | except NoNodeException: 35 | return False 36 | except KazooException as e: 37 | raise HostLookupError("zk host aliveness check failed: %r", e) 38 | 39 | def shut_down(self): 40 | self.zk.stop() 41 | -------------------------------------------------------------------------------- /push/main.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os.path 3 | 4 | import push.config 5 | import push.args 6 | import push.log 7 | import push.deploy 8 | import push.syslog 9 | import push.irc 10 | import push.cli 11 | 12 | 13 | def main(): 14 | host_source = None 15 | 16 | try: 17 | # read in the various configs and arguments and get ready 18 | try: 19 | config = push.config.parse_config() 20 | host_source = push.hosts.make_host_source(config) 21 | args = push.args.parse_args(config, host_source) 22 | 23 | if args.list_hosts: 24 | for host in args.hosts: 25 | print host 26 | return 0 27 | 28 | log = push.log.Log(config, args) 29 | except (push.config.ConfigurationError, push.args.ArgumentError, 30 | push.hosts.HostOrAliasError, push.hosts.HostLookupError), e: 31 | print >> sys.stderr, "%s: %s" % (os.path.basename(sys.argv[0]), e) 32 | return 1 33 | else: 34 | deployer = push.deploy.Deployer(config, args, log, host_source) 35 | 36 | # set up listeners 37 | push.log.register(config, args, deployer, log) 38 | push.syslog.register(config, args, deployer, log) 39 | push.irc.register(config, args, deployer, log) 40 | push.cli.register(config, args, deployer, log) 41 | 42 | # go 43 | try: 44 | deployer.push() 45 | except push.deploy.PushAborted: 46 | pass 47 | except Exception, e: 48 | log.critical("Push failed: %s", e) 49 | return 1 50 | finally: 51 | log.close() 52 | 53 | return 0 54 | finally: 55 | if host_source: 56 | host_source.shut_down() 57 | 58 | if __name__ == "__main__": 59 | sys.exit(main()) 60 | -------------------------------------------------------------------------------- /push/log.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import codecs 4 | import getpass 5 | import datetime 6 | 7 | 8 | __all__ = ["Log", "register"] 9 | 10 | 11 | RED = 31 12 | GREEN = 32 13 | YELLOW = 33 14 | BLUE = 34 15 | MAGENTA = 35 16 | CYAN = 36 17 | WHITE = 37 18 | 19 | 20 | def colorize(text, color, bold): 21 | if color: 22 | boldizer = "1;" if bold else "" 23 | start_color = "\033[%s%dm" % (boldizer, color) 24 | end_color = "\033[0m" 25 | return "".join((start_color, text, end_color)) 26 | else: 27 | return text 28 | 29 | 30 | class Log(object): 31 | def __init__(self, config, args): 32 | self.args = args 33 | 34 | # generate a unique id for the push 35 | self.push_id = args.push_id 36 | 37 | # build the path for the logfile 38 | timestamp = datetime.datetime.utcnow().strftime("%Y-%m-%d_%H:%M:%S") 39 | log_name = "-".join((timestamp, self.push_id)) + ".log" 40 | self.log_path = os.path.join(config.paths.log_root, log_name) 41 | 42 | # open the logfile 43 | self.logfile = codecs.open(self.log_path, "w", "utf-8") 44 | 45 | def write(self, text, color=None, bold=False, newline=False, stdout=True): 46 | suffix = "\n" if newline else "" 47 | self.logfile.write(text + suffix) 48 | if stdout: 49 | sys.stdout.write(colorize(text, color, bold) + suffix) 50 | self.flush() 51 | 52 | def flush(self): 53 | self.logfile.flush() 54 | sys.stdout.flush() 55 | 56 | def debug(self, message, *args): 57 | self.write(message % args, 58 | newline=True, 59 | color=GREEN, 60 | stdout=not self.args.quiet) 61 | 62 | def info(self, message, *args): 63 | self.write(message % args, 64 | newline=True, 65 | stdout=not self.args.quiet) 66 | 67 | def notice(self, message, *args): 68 | self.write(message % args, 69 | newline=True, 70 | color=BLUE, 71 | bold=True, 72 | stdout=not self.args.quiet) 73 | 74 | def warning(self, message, *args): 75 | self.write(message % args, 76 | newline=True, 77 | color=YELLOW, 78 | bold=True) 79 | 80 | def critical(self, message, *args): 81 | self.write(message % args, 82 | newline=True, 83 | color=RED, 84 | bold=True) 85 | 86 | def close(self): 87 | self.logfile.close() 88 | 89 | 90 | def register(config, args, deployer, log): 91 | @deployer.push_began 92 | def on_push_began(deployer): 93 | user = getpass.getuser() 94 | time = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M") 95 | log.write("Push started by %s at %s " 96 | "UTC with args: %s" % (user, time, args.command_line), 97 | newline=True, stdout=False) 98 | -------------------------------------------------------------------------------- /push/hosts/__init__.py: -------------------------------------------------------------------------------- 1 | import fnmatch 2 | import importlib 3 | import re 4 | 5 | 6 | MAX_NESTED_ALIASES = 10 7 | 8 | 9 | def _sorted_nicely(iter): 10 | """Sorts strings with embedded numbers in them the way humans would expect. 11 | 12 | http://nedbatchelder.com/blog/200712/human_sorting.html#comments""" 13 | 14 | def tryint(s): 15 | try: 16 | return int(s) 17 | except ValueError: 18 | return s 19 | 20 | def alphanum_key(s): 21 | return [tryint(c) for c in re.split('([0-9]+)', s)] 22 | 23 | return sorted(iter, key=alphanum_key) 24 | 25 | 26 | class HostLookupError(Exception): 27 | def __init__(self, message): 28 | self.message = message 29 | 30 | def __str__(self): 31 | return self.message 32 | 33 | 34 | class HostOrAliasError(Exception): 35 | def __init__(self, alias, fmt, *args): 36 | self.alias = alias 37 | self.fmt = fmt 38 | self.args = args 39 | 40 | def __str__(self): 41 | return ('alias "%s":' % self.alias) + " " + (self.fmt % self.args) 42 | 43 | 44 | class HostSource(object): 45 | def get_all_hosts(self): 46 | raise NotImplementedError 47 | 48 | def should_host_be_alive(self, host): 49 | return True 50 | 51 | def shut_down(self): 52 | pass 53 | 54 | 55 | def make_host_source(config): 56 | source_name = config.hosts.source 57 | source_module = importlib.import_module("push.hosts." + source_name) 58 | source_cls = getattr(source_module, source_name.title() + "HostSource") 59 | return source_cls(config) 60 | 61 | 62 | def get_hosts_and_aliases(config, host_source): 63 | """Fetches hosts from DNS then aliases them by globs specified in 64 | the config file. Returns a tuple of (all_hosts:list, aliases:dict).""" 65 | 66 | all_hosts = _sorted_nicely(host_source.get_all_hosts()) 67 | aliases = {} 68 | 69 | def dereference_alias(alias_name, globs, depth=0): 70 | if depth > MAX_NESTED_ALIASES: 71 | raise HostOrAliasError(alias_name, 72 | "exceeded maximum recursion depth. " 73 | "circular reference?") 74 | 75 | hosts = [] 76 | for glob in globs: 77 | if glob.startswith("@"): 78 | # recursive alias reference 79 | subalias_name = glob[1:] 80 | if subalias_name not in config.aliases: 81 | raise HostOrAliasError(alias_name, 82 | 'referenced undefined alias "%s"', 83 | subalias_name) 84 | subhosts = dereference_alias(subalias_name, 85 | config.aliases[subalias_name], 86 | depth=depth + 1) 87 | hosts.extend(subhosts) 88 | else: 89 | globbed = fnmatch.filter(all_hosts, glob) 90 | if not globbed: 91 | raise HostOrAliasError(alias_name, 'unmatched glob "%s"', 92 | glob) 93 | hosts.extend(globbed) 94 | return hosts 95 | 96 | for alias, globs in config.aliases.iteritems(): 97 | aliases[alias] = dereference_alias(alias, globs) 98 | 99 | return all_hosts, aliases 100 | -------------------------------------------------------------------------------- /push/cli.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import tty 3 | import time 4 | import termios 5 | import signal 6 | 7 | import push.deploy 8 | 9 | 10 | SIGNAL_MESSAGES = {signal.SIGINT: "received SIGINT", 11 | signal.SIGHUP: "received SIGHUP. tsk tsk."} 12 | 13 | 14 | def read_character(): 15 | "Read a single character from the terminal without echoing it." 16 | fd = sys.stdin.fileno() 17 | old_settings = termios.tcgetattr(fd) 18 | try: 19 | tty.setcbreak(fd) 20 | ch = sys.stdin.read(1) 21 | finally: 22 | termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) 23 | return ch 24 | 25 | 26 | def wait_for_input(log, deployer): 27 | """Wait for the user's choice of whether or not to continue the push. 28 | Return how many hosts to push to before asking again (0 for all).""" 29 | 30 | print >> log, ('Press "x" to abort, "c" to go to the next host, a ' 31 | 'number from 1-9 to push to that many hosts before ' 32 | 'pausing again, or "a" to continue automatically.') 33 | 34 | while True: 35 | c = read_character() 36 | if c == "a": 37 | print >> log, "Continuing automatically. Press ^C to abort." 38 | return 0 39 | elif c == "x": 40 | deployer.cancel_push('"x" pressed') 41 | elif c == "c": 42 | return 1 43 | else: 44 | # see if they entered a 1-9 number 45 | try: 46 | num_hosts = int(c) 47 | except ValueError: 48 | continue 49 | if num_hosts > 0: 50 | return num_hosts 51 | 52 | 53 | def sleep_with_countdown(log, sleeptime): 54 | if sleeptime == 0: 55 | return 56 | 57 | print >> log, "Sleeping...", 58 | log.flush() 59 | 60 | for i in xrange(sleeptime, 0, -1): 61 | print >> log, " %d..." % i, 62 | log.flush() 63 | time.sleep(1) 64 | 65 | print >> log, "" 66 | 67 | 68 | def register(config, args, deployer, log): 69 | def sighandler(sig, stack): 70 | reason = SIGNAL_MESSAGES[sig] 71 | deployer.cancel_push(reason) 72 | 73 | @deployer.push_began 74 | def on_push_began(deployer): 75 | signal.signal(signal.SIGINT, sighandler) 76 | signal.signal(signal.SIGHUP, sighandler) 77 | 78 | if args.testing: 79 | log.warning("*** Testing mode. No commands will be run. ***") 80 | 81 | log.notice("*** Beginning push. ***") 82 | log.notice("Log available at %s", log.log_path) 83 | 84 | @deployer.synchronize_began 85 | def on_sync_began(deployer): 86 | log.notice("Synchronizing build repos with GitHub...") 87 | 88 | @deployer.resolve_refs_began 89 | def on_resolve_refs_began(deployer): 90 | log.notice("Resolving refs...") 91 | 92 | @deployer.deploy_to_build_host_began 93 | def on_deploy_to_build_host_began(deployer): 94 | log.notice("Deploying to build host...") 95 | 96 | @deployer.build_static_began 97 | def on_build_static_began(deployer): 98 | log.notice("Building static files...") 99 | 100 | @deployer.process_host_began 101 | def on_process_host_began(deployer, host): 102 | log.notice('Starting host "%s"...', host) 103 | 104 | @deployer.process_host_ended 105 | def on_process_host_ended(deployer, host): 106 | host_index = args.hosts.index(host) + 1 107 | host_count = len(args.hosts) 108 | percentage = int((float(host_index) / host_count) * 100) 109 | log.notice('Host "%s" done (%d of %d -- %d%% done).', 110 | host, host_index, host_count, percentage) 111 | 112 | if args.hosts[-1] == host: 113 | pass 114 | elif args.hosts_before_pause == 1: 115 | args.hosts_before_pause = wait_for_input(log, deployer) 116 | else: 117 | args.hosts_before_pause -= 1 118 | sleep_with_countdown(log, args.sleeptime) 119 | 120 | @deployer.push_ended 121 | def on_push_ended(deployer): 122 | log.notice("*** Push complete! ***") 123 | 124 | @deployer.push_aborted 125 | def on_push_aborted(deployer, exception): 126 | if isinstance(exception, push.deploy.PushAborted): 127 | log.critical("\n*** Push cancelled (%s) ***", exception) 128 | 129 | def host_error_prompt(host, exception): 130 | log.critical("Encountered error on %s: %s", host, exception) 131 | print >> log, ('Press "x" to abort, "r" to retry this host, ' 132 | 'or "c" to skip to the next host') 133 | 134 | while True: 135 | c = read_character() 136 | if c == "x": 137 | return push.deploy.Deployer.ABORT 138 | elif c == "c": 139 | return push.deploy.Deployer.CONTINUE 140 | elif c == "r": 141 | return push.deploy.Deployer.RETRY 142 | deployer.host_error_prompt = host_error_prompt 143 | -------------------------------------------------------------------------------- /push/ssh.py: -------------------------------------------------------------------------------- 1 | import select 2 | import getpass 3 | import paramiko 4 | 5 | 6 | # hack to add paramiko support for AES encrypted private keys 7 | if "AES-128-CBC" not in paramiko.PKey._CIPHER_TABLE: 8 | from Crypto.Cipher import AES 9 | paramiko.PKey._CIPHER_TABLE["AES-128-CBC"] = dict(cipher=AES, keysize=16, blocksize=16, mode=AES.MODE_CBC) 10 | 11 | 12 | class SshError(Exception): 13 | def __init__(self, code): 14 | self.code = code 15 | 16 | def __str__(self): 17 | return "remote command exited with code %d" % self.code 18 | 19 | 20 | class SshConnection(object): 21 | def __init__(self, config, log, host): 22 | self.config = config 23 | self.log = log 24 | self.host = host 25 | 26 | self.client = paramiko.SSHClient() 27 | if not config.ssh.strict_host_key_checking: 28 | self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) 29 | self.client.connect(host, 30 | username=config.ssh.user, 31 | timeout=config.ssh.timeout, 32 | pkey=config.ssh.pkey) 33 | 34 | def execute_command(self, command, display_output=False): 35 | transport = self.client.get_transport() 36 | channel = transport.open_session() 37 | channel.settimeout(self.config.ssh.timeout) 38 | channel.set_combine_stderr(True) 39 | channel.exec_command(command) 40 | channel.shutdown_write() 41 | 42 | output = [] 43 | while True: 44 | readable = select.select([channel], [], [])[0] 45 | 46 | if not readable: 47 | continue 48 | 49 | received = channel.recv(1024) 50 | if not received: 51 | break 52 | 53 | received = unicode(received, "utf-8") 54 | 55 | output.append(received) 56 | 57 | if display_output: 58 | self.log.write(received, newline=False) 59 | 60 | status_code = channel.recv_exit_status() 61 | if status_code != 0: 62 | raise SshError(status_code) 63 | 64 | return "".join(output) 65 | 66 | def close(self): 67 | self.client.close() 68 | 69 | 70 | class SshDeployer(object): 71 | """Executes deploy commands on remote systems using SSH. If multiple 72 | commands are run on the same host in succession, the same connection is 73 | reused for each.""" 74 | 75 | def __init__(self, config, args, log): 76 | self.config = config 77 | self.args = args 78 | self.log = log 79 | self.current_connection = None 80 | 81 | config.ssh.pkey = None 82 | if not config.ssh.key_filename: 83 | return 84 | 85 | key_classes = (paramiko.RSAKey, paramiko.DSSKey) 86 | for key_class in key_classes: 87 | try: 88 | config.ssh.pkey = key_class.from_private_key_file( 89 | config.ssh.key_filename) 90 | except paramiko.PasswordRequiredException: 91 | need_password = True 92 | break 93 | except paramiko.SSHException: 94 | continue 95 | else: 96 | need_password = False 97 | break 98 | else: 99 | raise SshError("invalid key file %s" % config.ssh.key_filename) 100 | 101 | tries_remaining = 3 102 | while need_password and tries_remaining: 103 | password = getpass.getpass("password for %s: " % 104 | config.ssh.key_filename) 105 | 106 | try: 107 | config.ssh.pkey = key_class.from_private_key_file( 108 | config.ssh.key_filename, 109 | password=password) 110 | need_password = False 111 | except paramiko.SSHException: 112 | tries_remaining -= 1 113 | 114 | if need_password and not tries_remaining: 115 | raise SshError("invalid password.") 116 | 117 | def shutdown(self): 118 | if self.current_connection: 119 | self.current_connection.close() 120 | self.current_connection = None 121 | 122 | def _get_connection(self, host): 123 | if self.current_connection and self.current_connection.host != host: 124 | self.current_connection.close() 125 | self.current_connection = None 126 | if not self.current_connection: 127 | self.current_connection = SshConnection(self.config, self.log, host) 128 | return self.current_connection 129 | 130 | def _run_command(self, host, binary, *args, **kwargs): 131 | command = " ".join(("/usr/bin/sudo", binary) + args) 132 | self.log.debug(command) 133 | 134 | if not self.args.testing: 135 | conn = self._get_connection(host) 136 | display_output = kwargs.get("display_output", True) 137 | return conn.execute_command(command, display_output=display_output) 138 | else: 139 | return "TESTING" 140 | 141 | def run_build_command(self, *args, **kwargs): 142 | return self._run_command(self.config.deploy.build_host, 143 | self.config.deploy.build_binary, 144 | *args, **kwargs) 145 | 146 | def run_deploy_command(self, host, *args, **kwargs): 147 | return self._run_command(host, 148 | self.config.deploy.deploy_binary, 149 | *args, **kwargs) 150 | -------------------------------------------------------------------------------- /push/deploy.py: -------------------------------------------------------------------------------- 1 | import push.ssh 2 | 3 | auto_events = [] 4 | 5 | 6 | class PushAborted(Exception): 7 | "Raised when the deploy is cancelled." 8 | def __init__(self, reason): 9 | self.reason = reason 10 | 11 | def __str__(self): 12 | return self.reason 13 | 14 | 15 | class Event(object): 16 | """An event that can have an arbitrary number of listeners that get called 17 | when the event fires.""" 18 | def __init__(self, parent): 19 | self.parent = parent 20 | self.listeners = set() 21 | 22 | def register_listener(self, callable): 23 | self.listeners.add(callable) 24 | return callable 25 | 26 | def fire(self, *args, **kwargs): 27 | for listener in self.listeners: 28 | listener(self.parent, *args, **kwargs) 29 | 30 | __call__ = register_listener 31 | 32 | 33 | def event_wrapped(fn): 34 | """Wraps a function "fn" and fires the "fn_began" event before entering 35 | the function, "fn_ended" after succesfully returning, and "fn_aborted" 36 | on exception.""" 37 | began_name = fn.__name__ + "_began" 38 | ended_name = fn.__name__ + "_ended" 39 | aborted_name = fn.__name__ + "_aborted" 40 | auto_events.extend((began_name, ended_name, aborted_name)) 41 | 42 | def proxy(self, *args, **kwargs): 43 | getattr(self, began_name).fire(*args, **kwargs) 44 | try: 45 | result = fn(self, *args, **kwargs) 46 | except Exception, e: 47 | getattr(self, aborted_name).fire(e) 48 | raise 49 | else: 50 | getattr(self, ended_name).fire(*args, **kwargs) 51 | return result 52 | 53 | return proxy 54 | 55 | 56 | class Deployer(object): 57 | def __init__(self, config, args, log, host_source): 58 | self.config = config 59 | self.args = args 60 | self.log = log 61 | self.host_source = host_source 62 | self.deployer = push.ssh.SshDeployer(config, args, log) 63 | 64 | for event_name in auto_events: 65 | setattr(self, event_name, Event(self)) 66 | 67 | def _run_fetch_on_host(self, host, origin="origin"): 68 | for repo in self.args.fetches: 69 | self.deployer.run_deploy_command(host, "fetch", repo, origin) 70 | 71 | def _deploy_to_host(self, host): 72 | for repo in self.args.deploys: 73 | self.deployer.run_deploy_command(host, "deploy", repo, 74 | self.args.revisions[repo]) 75 | 76 | @event_wrapped 77 | def synchronize(self): 78 | for repo in self.args.fetches: 79 | self.deployer.run_build_command("synchronize", repo) 80 | 81 | self._run_fetch_on_host(self.config.deploy.build_host) 82 | 83 | @event_wrapped 84 | def resolve_refs(self): 85 | for repo in self.args.deploys: 86 | default_ref = self.config.default_refs.get(repo, "origin/master") 87 | ref_to_deploy = self.args.revisions.get(repo, default_ref) 88 | revision = self.deployer.run_build_command("get-revision", repo, 89 | ref_to_deploy, 90 | display_output=False) 91 | self.args.revisions[repo] = revision.strip() 92 | 93 | @event_wrapped 94 | def build_static(self): 95 | self.deployer.run_build_command("build-static") 96 | 97 | @event_wrapped 98 | def deploy_to_build_host(self): 99 | self._deploy_to_host(self.config.deploy.build_host) 100 | 101 | @event_wrapped 102 | def process_host(self, host): 103 | self._run_fetch_on_host(host) 104 | self._deploy_to_host(host) 105 | 106 | for command in self.args.deploy_commands: 107 | self.deployer.run_deploy_command(host, *command) 108 | 109 | def needs_static_build(self, repo): 110 | try: 111 | self.deployer.run_build_command("needs-static-build", repo, 112 | display_output=False) 113 | except push.ssh.SshError: 114 | return False 115 | else: 116 | return True 117 | 118 | @event_wrapped 119 | def push(self): 120 | try: 121 | self._push() 122 | finally: 123 | self.deployer.shutdown() 124 | 125 | 126 | ABORT = "abort" 127 | RETRY = "retry" 128 | CONTINUE = "continue" 129 | 130 | def host_error_prompt(self, host, error): 131 | return self.ABORT 132 | 133 | @event_wrapped 134 | def prompt_error(self, host, error): 135 | return self.host_error_prompt(host, error) 136 | 137 | def _push(self): 138 | if self.args.fetches: 139 | self.synchronize() 140 | 141 | if self.args.deploys: 142 | self.resolve_refs() 143 | self.deploy_to_build_host() 144 | 145 | if self.args.build_static: 146 | build_static = False 147 | for repo in self.args.deploys: 148 | if repo == "public" or self.needs_static_build(repo): 149 | build_static = True 150 | break 151 | 152 | if build_static: 153 | self.build_static() 154 | self.args.deploy_commands.append(["fetch-names"]) 155 | 156 | i = 0 157 | while i < len(self.args.hosts): 158 | host = self.args.hosts[i] 159 | i += 1 160 | 161 | try: 162 | self.process_host(host) 163 | except (push.ssh.SshError, IOError) as e: 164 | if self.host_source.should_host_be_alive(host): 165 | response = self.prompt_error(host, e) 166 | if response == self.ABORT: 167 | raise 168 | elif response == self.CONTINUE: 169 | continue 170 | elif response == self.RETRY: 171 | # rewind one host and try again 172 | i -= 1 173 | continue 174 | else: 175 | self.log.warning("Host %r appears to have been terminated." 176 | " ignoring errors and continuing." % host) 177 | 178 | def cancel_push(self, reason): 179 | raise PushAborted(reason) 180 | -------------------------------------------------------------------------------- /push/config.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import os 4 | import collections 5 | import ConfigParser 6 | 7 | 8 | NoDefault = object() 9 | SECTIONS = collections.OrderedDict() 10 | 11 | 12 | class attrdict(dict): 13 | "A dict whose keys can be accessed as attributes." 14 | def __init__(self, *args, **kwargs): 15 | dict.__init__(self, *args, **kwargs) 16 | self.__dict__ = self 17 | 18 | 19 | class ConfigurationError(Exception): 20 | "Exception to raise when there's a problem with the configuration file." 21 | def __init__(self, section, name, message): 22 | self.section = section 23 | self.name = name 24 | self.message = message 25 | 26 | def __str__(self): 27 | return 'section "%s": option "%s": %s' % (self.section, 28 | self.name, 29 | self.message) 30 | 31 | 32 | def boolean(input): 33 | """Converter that takes a string and tries to divine if it means 34 | true or false""" 35 | if input.lower() in ("true", "on"): 36 | return True 37 | elif input.lower() in ("false", "off"): 38 | return False 39 | else: 40 | raise ValueError('"%s" not boolean' % input) 41 | 42 | 43 | class Option(object): 44 | "Declarative explanation of a configuration option." 45 | def __init__(self, convert, default=NoDefault, validator=None): 46 | self.convert = convert 47 | self.default = default 48 | self.validator = validator 49 | 50 | 51 | def _make_extractor(cls, prefix="", required=True): 52 | section_name = cls.__name__[:-len("config")].lower() 53 | if prefix: 54 | section_name = prefix + ":" + section_name 55 | 56 | def config_extractor(parser): 57 | section = attrdict() 58 | for name, option_def in vars(cls).iteritems(): 59 | if not isinstance(option_def, Option): 60 | continue 61 | 62 | try: 63 | value = parser.get(section_name, name) 64 | except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): 65 | if option_def.default is NoDefault: 66 | raise ConfigurationError(section_name, name, 67 | "required but not present") 68 | value = option_def.default 69 | else: 70 | try: 71 | value = option_def.convert(value) 72 | except Exception, e: 73 | raise ConfigurationError(section_name, name, e) 74 | 75 | section[name] = value 76 | return section 77 | 78 | config_extractor.required = required 79 | config_extractor.prefix = prefix 80 | SECTIONS[section_name] = config_extractor 81 | 82 | 83 | def config_section(*args, **kwargs): 84 | if len(args) == 1 and not kwargs: 85 | # bare decorator "@config_section" style 86 | return _make_extractor(args[0]) 87 | 88 | def config_decorator(cls): 89 | return _make_extractor(cls, **kwargs) 90 | return config_decorator 91 | 92 | 93 | @config_section 94 | class SshConfig(object): 95 | user = Option(str) 96 | key_filename = Option(str, default=None) 97 | strict_host_key_checking = Option(boolean, default=True) 98 | timeout = Option(int, default=30) 99 | 100 | 101 | @config_section 102 | class DeployConfig(object): 103 | build_host = Option(str) 104 | deploy_binary = Option(str) 105 | build_binary = Option(str) 106 | 107 | 108 | @config_section 109 | class PathsConfig(object): 110 | log_root = Option(str) 111 | wordlist = Option(str, default="/usr/share/dict/words") 112 | 113 | 114 | @config_section 115 | class SyslogConfig(object): 116 | def syslog_enum(value): 117 | import syslog 118 | value = "LOG_" + value 119 | return getattr(syslog, value) 120 | 121 | ident = Option(str, default="deploy") 122 | facility = Option(syslog_enum) 123 | priority = Option(syslog_enum) 124 | 125 | 126 | @config_section 127 | class HostsConfig(object): 128 | def valid_host_source(value): 129 | try: 130 | section = SECTIONS["hosts:" + value] 131 | except KeyError: 132 | raise ValueError("invalid host source: %r" % value) 133 | section.required = True 134 | return value 135 | source = Option(valid_host_source) 136 | 137 | 138 | @config_section(prefix="hosts", required=False) 139 | class DnsConfig(object): 140 | domain = Option(str) 141 | 142 | 143 | @config_section(prefix="hosts", required=False) 144 | class MockConfig(object): 145 | host_count = Option(int) 146 | 147 | 148 | @config_section(prefix="hosts", required=False) 149 | class ZooKeeperConfig(object): 150 | connection_string = Option(str) 151 | username = Option(str) 152 | password = Option(str) 153 | 154 | 155 | @config_section 156 | class DefaultsConfig(object): 157 | sleeptime = Option(int, default=0) 158 | shuffle = Option(boolean, default=False) 159 | 160 | 161 | def alias_parser(parser): 162 | aliases = {} 163 | if parser.has_section("aliases"): 164 | for key, value in parser.items("aliases"): 165 | aliases[key] = [glob.strip() for glob in value.split(' ')] 166 | return aliases 167 | SECTIONS["aliases"] = alias_parser 168 | 169 | 170 | def default_ref_parser(parser): 171 | default_refs = {} 172 | if parser.has_section("default_refs"): 173 | default_refs.update(parser.items("default_refs")) 174 | return default_refs 175 | SECTIONS["default_refs"] = default_ref_parser 176 | 177 | 178 | def parse_config(): 179 | """Loads the configuration files and parses them according to the 180 | section parsers in SECTIONS.""" 181 | parser = ConfigParser.RawConfigParser() 182 | parser.read(["/opt/push/etc/push.ini", os.path.expanduser("~/.push.ini")]) 183 | 184 | config = attrdict() 185 | for name, section_parser in SECTIONS.iteritems(): 186 | is_required = getattr(section_parser, "required", True) 187 | if is_required or parser.has_section(name): 188 | prefix = getattr(section_parser, "prefix", None) 189 | parsed = section_parser(parser) 190 | if not prefix: 191 | config[name] = parsed 192 | else: 193 | unprefixed = name[len(prefix) + 1:] 194 | config.setdefault(prefix, attrdict())[unprefixed] = parsed 195 | 196 | return config 197 | -------------------------------------------------------------------------------- /push/args.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import argparse 3 | import itertools 4 | import collections 5 | 6 | import push.hosts 7 | import push.utils 8 | 9 | 10 | __all__ = ["parse_args", "ArgumentError"] 11 | 12 | 13 | class MutatingAction(argparse.Action): 14 | def __init__(self, *args, **kwargs): 15 | self.type_to_mutate = kwargs.pop("type_to_mutate") 16 | argparse.Action.__init__(self, *args, **kwargs) 17 | 18 | def get_attr_to_mutate(self, namespace): 19 | o = getattr(namespace, self.dest, None) 20 | if not o: 21 | o = self.type_to_mutate() 22 | setattr(namespace, self.dest, o) 23 | return o 24 | 25 | 26 | class SetAddConst(MutatingAction): 27 | "Action that adds a constant to a set." 28 | def __init__(self, *args, **kwargs): 29 | kwargs["nargs"] = 0 30 | MutatingAction.__init__(self, *args, 31 | type_to_mutate=collections.OrderedDict, 32 | **kwargs) 33 | 34 | def __call__(self, parser, namespace, values, option_string=None): 35 | s = self.get_attr_to_mutate(namespace) 36 | 37 | if hasattr(self.const, "__iter__"): 38 | for x in self.const: 39 | s[x] = "" 40 | else: 41 | s[self.const] = "" 42 | 43 | 44 | class SetAddValues(MutatingAction): 45 | "Action that adds values to a set." 46 | def __init__(self, *args, **kwargs): 47 | MutatingAction.__init__(self, *args, 48 | type_to_mutate=collections.OrderedDict, 49 | **kwargs) 50 | 51 | def __call__(self, parser, namespace, values, option_string=None): 52 | s = self.get_attr_to_mutate(namespace) 53 | 54 | for x in values: 55 | s[x] = "" 56 | 57 | 58 | class DictAdd(MutatingAction): 59 | "Action that adds an argument to a dict with a constant key." 60 | def __init__(self, *args, **kwargs): 61 | MutatingAction.__init__(self, *args, type_to_mutate=dict, **kwargs) 62 | 63 | def __call__(self, parser, namespace, values, option_string=None): 64 | d = self.get_attr_to_mutate(namespace) 65 | key, value = values 66 | d[key] = value 67 | 68 | 69 | class RestartCommand(MutatingAction): 70 | """Makes a deploy command out of -r (graceful restart) options.""" 71 | 72 | def __init__(self, *args, **kwargs): 73 | MutatingAction.__init__(self, *args, type_to_mutate=list, **kwargs) 74 | 75 | def __call__(self, parser, namespace, values, option_string=None): 76 | command_list = self.get_attr_to_mutate(namespace) 77 | command_list.append(["restart", values[0]]) 78 | 79 | 80 | class KillCommand(MutatingAction): 81 | """Makes a deploy command out of -k (kill) options.""" 82 | 83 | def __init__(self, *args, **kwargs): 84 | MutatingAction.__init__(self, *args, type_to_mutate=list, **kwargs) 85 | 86 | def __call__(self, parser, namespace, values, option_string=None): 87 | command_list = self.get_attr_to_mutate(namespace) 88 | command_list.append(["kill", values[0]]) 89 | 90 | 91 | class StoreIfHost(argparse.Action): 92 | "Stores value if it is a known host." 93 | def __init__(self, *args, **kwargs): 94 | self.all_hosts = kwargs.pop("all_hosts") 95 | argparse.Action.__init__(self, *args, **kwargs) 96 | 97 | def __call__(self, parser, namespace, value, option_string=None): 98 | if value not in self.all_hosts: 99 | raise argparse.ArgumentError(self, 'unknown host "%s"' % value) 100 | setattr(namespace, self.dest, value) 101 | 102 | 103 | class ArgumentError(Exception): 104 | "Exception raised when there's something wrong with the arguments." 105 | def __init__(self, message): 106 | self.message = message 107 | 108 | def __str__(self): 109 | return self.message 110 | 111 | 112 | class ArgumentParser(argparse.ArgumentParser): 113 | """Custom argument parser that raises an exception rather than exiting 114 | the program""" 115 | 116 | def error(self, message): 117 | raise ArgumentError(message) 118 | 119 | 120 | def _parse_args(config): 121 | parser = ArgumentParser(description="Deploy stuff to servers.", 122 | epilog="To deploy all code: push -h apps " 123 | "-pc -dc -r all", 124 | add_help=False) 125 | 126 | parser.add_argument("-h", dest="host_refs", metavar="HOST", required=True, 127 | action="append", nargs="+", 128 | help="hosts or groups to execute commands on") 129 | parser.add_argument("--sleeptime", dest="sleeptime", nargs="?", 130 | type=int, default=config.defaults.sleeptime, 131 | metavar="SECONDS", 132 | help="time in seconds to sleep between hosts") 133 | parser.add_argument("--startat", dest="start_at", 134 | action="store", nargs='?', metavar="HOST", 135 | help="skip to this position in the host list") 136 | parser.add_argument("--stopbefore", dest="stop_before", 137 | action="store", nargs="?", metavar="HOST", 138 | help="end the push on the host before this one") 139 | parser.add_argument("--pauseafter", dest="hosts_before_pause", nargs="?", 140 | type=int, metavar="NUMBER", default=1, 141 | help="push to NUMBER hosts before pausing") 142 | parser.add_argument("--seed", dest="seed", action="store", 143 | nargs="?", metavar="WORD", default=None, 144 | help="name of push to copy the shuffle-order of") 145 | parser.add_argument("--shuffle", dest="shuffle", 146 | default=config.defaults.shuffle, 147 | action="store_true", help="shuffle host list") 148 | parser.add_argument("--no-shuffle", dest="shuffle", 149 | action="store_false", 150 | help="don't shuffle host list") 151 | parser.add_argument("--list", dest="list_hosts", 152 | action="store_true", default=False, 153 | help="print the host list to stdout and exit") 154 | 155 | flags_group = parser.add_argument_group("flags") 156 | flags_group.add_argument("-t", dest="testing", action="store_true", 157 | help="testing: print but don't execute") 158 | flags_group.add_argument("-q", dest="quiet", action="store_true", 159 | help="quiet: no output except errors. implies " 160 | "--no-input") 161 | flags_group.add_argument("--no-irc", dest="notify_irc", 162 | action="store_false", 163 | help="don't announce actions in irc") 164 | flags_group.add_argument("--no-static", dest="build_static", 165 | action="store_false", 166 | help="don't build static files") 167 | flags_group.add_argument("--no-input", dest="auto_continue", 168 | action="store_true", 169 | help="don't wait for input after deploy") 170 | 171 | parser.add_argument("--help", action="help", help="display this help") 172 | 173 | deploy_group = parser.add_argument_group("deploy") 174 | deploy_group.add_argument("-p", dest="fetches", default=set(), 175 | action=SetAddValues, nargs="+", 176 | metavar="REPO", 177 | help="git-fetch the specified repo(s)") 178 | deploy_group.add_argument("-pc", dest="fetches", 179 | action=SetAddConst, const=["public", "private"], 180 | help="short for -p public private") 181 | deploy_group.add_argument("-ppr", dest="fetches", 182 | action=SetAddConst, const=["private"], 183 | help="short for -p private") 184 | 185 | deploy_group.add_argument("-d", dest="deploys", default=set(), 186 | action=SetAddValues, nargs="+", 187 | metavar="REPO", 188 | help="deploy the specified repo(s)") 189 | deploy_group.add_argument("-dc", dest="deploys", 190 | action=SetAddConst, const=["public", "private"], 191 | help="short for -d public private") 192 | deploy_group.add_argument("-dpr", dest="deploys", 193 | action=SetAddConst, const=["private"], 194 | help="short for -d private") 195 | deploy_group.add_argument("-rev", dest="revisions", default={}, 196 | metavar=("REPO", "REF"), action=DictAdd, 197 | nargs=2, 198 | help="revision to deploy for specified repo") 199 | 200 | parser.add_argument("-c", dest="deploy_commands", nargs="+", 201 | metavar=("COMMAND", "ARG"), action="append", 202 | help="deploy command to run on the host", 203 | default=[]) 204 | parser.add_argument("-r", dest="deploy_commands", nargs=1, 205 | metavar="COMMAND", action=RestartCommand, 206 | help="whom to (gracefully) restart on the host") 207 | parser.add_argument("-k", dest="deploy_commands", nargs=1, 208 | action=KillCommand, choices=["all", "apps"], 209 | help="whom to kill on the host") 210 | 211 | if len(sys.argv) == 1: 212 | parser.print_help() 213 | 214 | return parser.parse_args() 215 | 216 | 217 | def build_command_line(config, args): 218 | "Given a configured environment, build a canonical command line for it." 219 | components = [] 220 | 221 | components.append("-h") 222 | components.extend(itertools.chain.from_iterable(args.host_refs)) 223 | 224 | if args.start_at: 225 | components.append("--startat=%s" % args.start_at) 226 | 227 | if args.stop_before: 228 | components.append("--stopbefore=%s" % args.stop_before) 229 | 230 | if args.hosts_before_pause > 1: 231 | components.append("--pauseafter=%s" % args.hosts_before_pause) 232 | 233 | if args.fetches: 234 | components.append("-p") 235 | components.extend(args.fetches) 236 | 237 | if args.deploys: 238 | components.append("-d") 239 | components.extend(args.deploys) 240 | 241 | commands = dict(restart="-r", 242 | kill="-k") 243 | for command in args.deploy_commands: 244 | special_command = commands.get(command[0]) 245 | if special_command: 246 | components.append(special_command) 247 | command = command[1:] 248 | else: 249 | components.append("-c") 250 | 251 | components.extend(command) 252 | 253 | for repo, rev in args.revisions.iteritems(): 254 | components.extend(("-rev", repo, rev)) 255 | 256 | if not args.build_static: 257 | components.append("--no-static") 258 | 259 | if args.auto_continue: 260 | components.append("--no-input") 261 | 262 | if not args.notify_irc: 263 | components.append("--no-irc") 264 | 265 | if args.quiet: 266 | components.append("--quiet") 267 | 268 | if args.testing: 269 | components.append("-t") 270 | 271 | if args.shuffle: 272 | components.append("--shuffle") 273 | 274 | if args.seed: 275 | components.append("--seed=%s" % args.seed) 276 | 277 | components.append("--sleeptime=%d" % args.sleeptime) 278 | 279 | return " ".join(components) 280 | 281 | 282 | def parse_args(config, host_source): 283 | args = _parse_args(config) 284 | 285 | # give the push a unique name 286 | args.push_id = push.utils.get_random_word(config) 287 | 288 | # quiet implies autocontinue 289 | if args.quiet or args.auto_continue: 290 | args.hosts_before_pause = 0 291 | 292 | # dereference the host lists 293 | all_hosts, aliases = push.hosts.get_hosts_and_aliases(config, host_source) 294 | args.hosts = [] 295 | queue = collections.deque(args.host_refs) 296 | while queue: 297 | host_or_alias = queue.popleft() 298 | 299 | # individual instances of -h append a list to the list. flatten 300 | if hasattr(host_or_alias, "__iter__"): 301 | queue.extend(host_or_alias) 302 | continue 303 | 304 | # backwards compatibility with perl version 305 | if " " in host_or_alias: 306 | queue.extend(x.strip() for x in host_or_alias.split()) 307 | continue 308 | 309 | if host_or_alias in all_hosts: 310 | args.hosts.append(host_or_alias) 311 | elif host_or_alias in aliases: 312 | args.hosts.extend(aliases[host_or_alias]) 313 | else: 314 | raise ArgumentError('-h: unknown host or alias "%s"' % 315 | host_or_alias) 316 | 317 | # make sure the startat is in the dereferenced host list 318 | if args.start_at and args.start_at not in args.hosts: 319 | raise ArgumentError('--startat: host "%s" not in host list.' % 320 | args.start_at) 321 | 322 | # it really doesn't make sense to start-at while shufflin' w/o a seed 323 | if args.start_at and args.shuffle and not args.seed: 324 | raise ArgumentError("--startat: doesn't make sense " 325 | "while shuffling without a seed") 326 | 327 | # make sure the stopbefore is in the dereferenced host list 328 | if args.stop_before and args.stop_before not in args.hosts: 329 | raise ArgumentError('--stopbefore: host "%s" not in host list.' % 330 | args.stop_before) 331 | 332 | # it really doesn't make sense to stop-at while shufflin' w/o a seed 333 | if args.stop_before and args.shuffle and not args.seed: 334 | raise ArgumentError("--stopbefore: doesn't make sense " 335 | "while shuffling without a seed") 336 | 337 | # restrict the host list if start_at or stop_before were defined 338 | if args.start_at or args.stop_before: 339 | if args.stop_before: 340 | args.hosts = itertools.takewhile( 341 | lambda host: host != args.stop_before, args.hosts) 342 | if args.start_at: 343 | args.hosts = itertools.dropwhile( 344 | lambda host: host != args.start_at, args.hosts) 345 | args.hosts = list(args.hosts) 346 | 347 | # do the shuffle! 348 | if args.shuffle: 349 | seed = args.seed or args.push_id 350 | push.utils.seeded_shuffle(seed, args.hosts) 351 | 352 | # build a psuedo-commandline out of args and defaults 353 | args.command_line = build_command_line(config, args) 354 | 355 | return args 356 | --------------------------------------------------------------------------------