├── MANIFEST.in ├── .gitignore ├── .coveragerc ├── .travis.yml ├── setup.cfg ├── CHANGES.rst ├── juju_scaleway ├── __init__.py ├── constraints.py ├── exceptions.py ├── ssh.py ├── runner.py ├── config.py ├── provider.py ├── ops.py ├── client.py ├── cli.py ├── env.py └── commands.py ├── LICENSE.rst ├── setup.py └── README.rst /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include CHANGES.rst 2 | include LICENSE.rst 3 | include MANIFEST.in 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.egg 2 | *.egg-info 3 | *.pyc 4 | bin 5 | build 6 | dist 7 | include 8 | lib 9 | local 10 | man 11 | *.swp 12 | .*.swp 13 | *~ 14 | *# 15 | .#* 16 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | # http://nedbatchelder.com/code/coverage/config.html#config 2 | 3 | [run] 4 | source = juju_scaleway 5 | branch = True 6 | omit = */tests/* 7 | 8 | [report] 9 | omit = */tests/* 10 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "2.7" 4 | - "3.4" 5 | install: 6 | - pip install -e . 7 | - pip install codecov 8 | script: 9 | - coverage run setup.py test 10 | - coverage report -m 11 | after_script: 12 | codecov 13 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 1.0.4 3 | files = juju_scaleway/__init__.py CHANGES.rst 4 | allow_dirty = True 5 | commit = False 6 | tag = False 7 | tag_name = v{new_version} 8 | 9 | [nosetests] 10 | match = ^test 11 | cover-package = juju_scaleway 12 | with-coverage = 1 13 | cover-erase = 1 14 | cover-branches = 1 15 | cover-min-percentage = 94 16 | 17 | [build_sphinx] 18 | all_files = 1 19 | build-dir = doc/_build 20 | source-dir = doc/ 21 | -------------------------------------------------------------------------------- /CHANGES.rst: -------------------------------------------------------------------------------- 1 | ChangeLog 2 | ========= 3 | 4 | 1.0.4 (unreleased) 5 | ------------------ 6 | 7 | * No changes yet. 8 | 9 | 1.0.3 (2015-11-23) 10 | ------------------ 11 | 12 | * Add travis config. 13 | * Add bumpversion config. 14 | * Switch from coveralls.io to codecov.io. 15 | 16 | 1.0.2 (2015-08-17) 17 | ------------------ 18 | 19 | * Add missing license. Closes #2. 20 | * Rewrite documentation from Markdown to RST to please PyPi. 21 | * Add badges. 22 | 23 | 1.0.1 (2015-05-12) 24 | ------------------ 25 | 26 | * Fix Python3 compatibility. 27 | 28 | 1.0.0 (2015-04-13) 29 | ------------------ 30 | 31 | * Initial release. 32 | -------------------------------------------------------------------------------- /juju_scaleway/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (c) 2014-2015 Online SAS and Contributors. All Rights Reserved. 4 | # Edouard Bonlieu 5 | # Julien Castets 6 | # Manfred Touron 7 | # Kevin Deldycke 8 | # 9 | # Licensed under the BSD 2-Clause License (the "License"); you may not use this 10 | # file except in compliance with the License. You may obtain a copy of the 11 | # License at http://opensource.org/licenses/BSD-2-Clause 12 | 13 | __version__ = '1.0.4' 14 | -------------------------------------------------------------------------------- /juju_scaleway/constraints.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (c) 2014-2015 Online SAS and Contributors. All Rights Reserved. 4 | # Edouard Bonlieu 5 | # Julien Castets 6 | # Manfred Touron 7 | # Kevin Deldycke 8 | # 9 | # Licensed under the BSD 2-Clause License (the "License"); you may not use this 10 | # file except in compliance with the License. You may obtain a copy of the 11 | # License at http://opensource.org/licenses/BSD-2-Clause 12 | 13 | SERIES_MAP = { 14 | 'Ubuntu Utopic (14.10)': 'utopic', 15 | 'Ubuntu Trusty (14.04 LTS)': 'trusty', 16 | } 17 | 18 | 19 | def get_images(client): 20 | images = {} 21 | for i in client.get_images(): 22 | if not i.public: 23 | continue 24 | 25 | for serie in SERIES_MAP: 26 | if ("%s" % serie) == i.name: 27 | images[SERIES_MAP[serie]] = i.id 28 | 29 | return images 30 | -------------------------------------------------------------------------------- /LICENSE.rst: -------------------------------------------------------------------------------- 1 | BSD 2-Clause License 2 | -------------------- 3 | 4 | Copyright (c) 2014-2015, `Scaleway `_ and individual 5 | contributors. 6 | All rights reserved. 7 | 8 | Redistribution and use in source and binary forms, with or without 9 | modification, are permitted provided that the following conditions are met: 10 | 11 | * Redistributions of source code must retain the above copyright notice, this 12 | list of conditions and the following disclaimer. 13 | 14 | * Redistributions in binary form must reproduce the above copyright notice, 15 | this list of conditions and the following disclaimer in the documentation 16 | and/or other materials provided with the distribution. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /juju_scaleway/exceptions.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (c) 2014-2015 Online SAS and Contributors. All Rights Reserved. 4 | # Edouard Bonlieu 5 | # Julien Castets 6 | # Manfred Touron 7 | # Kevin Deldycke 8 | # 9 | # Licensed under the BSD 2-Clause License (the "License"); you may not use this 10 | # file except in compliance with the License. You may obtain a copy of the 11 | # License at http://opensource.org/licenses/BSD-2-Clause 12 | 13 | 14 | class ConfigError(ValueError): 15 | """ Environments.yaml configuration error. 16 | """ 17 | 18 | 19 | class PrecheckError(ValueError): 20 | """ A precondition check failed. 21 | """ 22 | 23 | 24 | class ConstraintError(ValueError): 25 | """ Specificed constraint is invalid. 26 | """ 27 | 28 | 29 | class TimeoutError(ValueError): 30 | """ Instance could not be provisioned before timeout. 31 | """ 32 | 33 | 34 | class ProviderError(Exception): 35 | """Instance could not be provisioned. 36 | """ 37 | 38 | 39 | class ProviderAPIError(Exception): 40 | 41 | def __init__(self, response, message): 42 | self.response = response 43 | self.message = message 44 | 45 | def __str__(self): 46 | return "" % ( 47 | self.message or "Unknown", 48 | self.response.status_code) 49 | -------------------------------------------------------------------------------- /juju_scaleway/ssh.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (c) 2014-2015 Online SAS and Contributors. All Rights Reserved. 4 | # Edouard Bonlieu 5 | # Julien Castets 6 | # Manfred Touron 7 | # Kevin Deldycke 8 | # 9 | # Licensed under the BSD 2-Clause License (the "License"); you may not use this 10 | # file except in compliance with the License. You may obtain a copy of the 11 | # License at http://opensource.org/licenses/BSD-2-Clause 12 | 13 | import subprocess 14 | 15 | # juju-core will defer to either ssh or go.crypto/ssh impl 16 | # these options are only for the ssh ops below (availability 17 | # check and apt-get update on precise instances). 18 | SSH_CMD = ("/usr/bin/ssh", 19 | "-o", "StrictHostKeyChecking=no", 20 | "-o", "UserKnownHostsFile=/dev/null") 21 | 22 | 23 | def check_ssh(host, user="root"): 24 | cmd = list(SSH_CMD) + ["%s@%s" % (user, host), "ls"] 25 | process = subprocess.Popen( 26 | args=cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) 27 | 28 | output, err = process.communicate() 29 | retcode = process.poll() 30 | 31 | if retcode: 32 | raise subprocess.CalledProcessError( 33 | retcode, cmd, '%s%s' % (output, err or '') 34 | ) 35 | return True 36 | 37 | 38 | def update_instance(host, user="root"): 39 | base = list(SSH_CMD) + ["%s@%s" % (user, host)] 40 | subprocess.check_output( 41 | base + ["apt-get", "update"], stderr=subprocess.STDOUT 42 | ) 43 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2014-2015 Online SAS and Contributors. All Rights Reserved. 5 | # Julien Castets 6 | # Kevin Deldycke 7 | # 8 | # Licensed under the BSD 2-Clause License (the "License"); you may not use this 9 | # file except in compliance with the License. You may obtain a copy of the 10 | # License at http://opensource.org/licenses/BSD-2-Clause 11 | 12 | import os 13 | import re 14 | 15 | from setuptools import setup, find_packages 16 | 17 | 18 | MODULE_NAME = 'juju_scaleway' 19 | 20 | DEPENDENCIES = [ 21 | 'PyYAML', 22 | 'requests', 23 | 'ndg-httpsclient >=0.3.3', 24 | ] 25 | 26 | TEST_DEPENDENCIES = [ 27 | 'mock' 28 | ] 29 | 30 | EXTRA_DEPENDENCIES = { 31 | 'dev': ['PyYAML', 'requests', 'nose', 'mock'] 32 | } 33 | 34 | 35 | def get_version(): 36 | """ Reads package version number from package's __init__.py. """ 37 | with open(os.path.join( 38 | os.path.dirname(__file__), MODULE_NAME, '__init__.py' 39 | )) as init: 40 | for line in init.readlines(): 41 | res = re.match(r'^__version__ = [\'"](.*)[\'"]$', line) 42 | if res: 43 | return res.group(1) 44 | 45 | 46 | def get_long_description(): 47 | """ Read description from README and CHANGES. """ 48 | with open( 49 | os.path.join(os.path.dirname(__file__), 'README.rst') 50 | ) as readme, open( 51 | os.path.join(os.path.dirname(__file__), 'CHANGES.rst') 52 | ) as changes: 53 | return readme.read() + '\n' + changes.read() 54 | 55 | 56 | setup( 57 | name='juju-scaleway', 58 | version=get_version(), 59 | author='Scaleway', 60 | author_email='opensource@scaleway.com', 61 | description='Scaleway integration with juju', 62 | url='https://pypi.python.org/pypi/juju-scaleway', 63 | license='BSD', 64 | packages=find_packages(), 65 | install_requires=DEPENDENCIES, 66 | tests_require=DEPENDENCIES + TEST_DEPENDENCIES, 67 | classifiers=[ 68 | 'Development Status :: 5 - Production/Stable', 69 | 'Environment :: Web Environment', 70 | 'Intended Audience :: Developers', 71 | 'License :: OSI Approved :: BSD License', 72 | 'Operating System :: OS Independent', 73 | 'Programming Language :: Python', 74 | 'Programming Language :: Python :: 2', 75 | 'Programming Language :: Python :: 2.7', 76 | 'Topic :: Software Development :: Libraries :: Python Modules', 77 | 'Topic :: Internet', 78 | 'Topic :: System :: Distributed Computing', 79 | ], 80 | entry_points={ 81 | 'console_scripts': [ 82 | 'juju-scaleway = juju_scaleway.cli:main' 83 | ] 84 | }, 85 | extras_require=EXTRA_DEPENDENCIES 86 | ) 87 | -------------------------------------------------------------------------------- /juju_scaleway/runner.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (c) 2014-2015 Online SAS and Contributors. All Rights Reserved. 4 | # Edouard Bonlieu 5 | # Julien Castets 6 | # Manfred Touron 7 | # Kevin Deldycke 8 | # 9 | # Licensed under the BSD 2-Clause License (the "License"); you may not use this 10 | # file except in compliance with the License. You may obtain a copy of the 11 | # License at http://opensource.org/licenses/BSD-2-Clause 12 | 13 | """ 14 | Thread based concurrency around bulk ops. scaleway api is sync 15 | """ 16 | 17 | import logging 18 | 19 | try: 20 | from Queue import Queue, Empty 21 | except ImportError: # Python3 22 | from queue import Queue, Empty 23 | 24 | import threading 25 | 26 | 27 | logger = logging.getLogger("juju.scaleway") 28 | 29 | 30 | class Runner(object): 31 | 32 | DEFAULT_NUM_RUNNER = 4 33 | 34 | def __init__(self): 35 | self.jobs = Queue() 36 | self.results = Queue() 37 | self.job_count = 0 38 | self.runners = [] 39 | self.started = False 40 | 41 | def queue_op(self, operation): 42 | self.jobs.put(operation) 43 | self.job_count += 1 44 | 45 | def iter_results(self): 46 | auto = not self.started 47 | 48 | if auto: 49 | self.start(min(self.DEFAULT_NUM_RUNNER, self.job_count)) 50 | 51 | for _ in range(self.job_count): 52 | self.job_count -= 1 53 | result = self.gather_result() 54 | if isinstance(result, Exception): 55 | continue 56 | yield result 57 | 58 | if auto: 59 | self.stop() 60 | 61 | def gather_result(self): 62 | return self.results.get() 63 | 64 | def start(self, count): 65 | for _ in range(count): 66 | runner = OpRunner(self.jobs, self.results) 67 | runner.daemon = True 68 | self.runners.append(runner) 69 | runner.start() 70 | self.started = True 71 | 72 | def stop(self): 73 | for runner in self.runners: 74 | runner.join() 75 | self.started = False 76 | 77 | 78 | class OpRunner(threading.Thread): 79 | 80 | def __init__(self, ops, results): 81 | self.ops = ops 82 | self.results = results 83 | super(OpRunner, self).__init__() 84 | 85 | def run(self): 86 | while 1: 87 | try: 88 | operation = self.ops.get(block=False) 89 | except Empty: 90 | return 91 | 92 | try: 93 | result = operation.run() 94 | except Exception as exc: 95 | logger.exception("Error while processing op %s", operation) 96 | result = exc 97 | 98 | self.results.put(result) 99 | -------------------------------------------------------------------------------- /juju_scaleway/config.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (c) 2014-2015 Online SAS and Contributors. All Rights Reserved. 4 | # Edouard Bonlieu 5 | # Julien Castets 6 | # Manfred Touron 7 | # Kevin Deldycke 8 | # 9 | # Licensed under the BSD 2-Clause License (the "License"); you may not use this 10 | # file except in compliance with the License. You may obtain a copy of the 11 | # License at http://opensource.org/licenses/BSD-2-Clause 12 | 13 | import os 14 | import yaml 15 | import sys 16 | 17 | from juju_scaleway.env import Environment 18 | from juju_scaleway.exceptions import ConfigError 19 | from juju_scaleway import provider 20 | 21 | 22 | class Config(object): 23 | 24 | def __init__(self, options): 25 | self.options = options 26 | 27 | def connect_provider(self): 28 | """Connect to Scaleway. 29 | """ 30 | return provider.factory() 31 | 32 | def connect_environment(self): 33 | """Return a websocket connection to the environment. 34 | """ 35 | return Environment(self) 36 | 37 | def validate(self): 38 | provider.validate() 39 | self.get_env_name() 40 | 41 | @property 42 | def verbose(self): 43 | return self.options.verbose 44 | 45 | @property 46 | def constraints(self): 47 | return self.options.constraints 48 | 49 | @property 50 | def series(self): 51 | return self.options.series 52 | 53 | @property 54 | def upload_tools(self): 55 | return getattr(self.options, 'upload_tools', False) 56 | 57 | @property 58 | def num_machines(self): 59 | return getattr(self.options, 'num_machines', 0) 60 | 61 | @property 62 | def juju_home(self): 63 | jhome = os.environ.get("JUJU_HOME") 64 | if jhome is not None: 65 | return os.path.expanduser(jhome) 66 | if sys.platform == "win32": 67 | return os.path.join( 68 | os.path.join('APPDATA'), "Juju") 69 | return os.path.expanduser("~/.juju") 70 | 71 | def get_env_name(self): 72 | """Get the environment name. 73 | """ 74 | if self.options.environment: 75 | return self.options.environment 76 | elif os.environ.get("JUJU_ENV"): 77 | return os.environ['JUJU_ENV'] 78 | 79 | env_ptr = os.path.join(self.juju_home, "current-environment") 80 | if os.path.exists(env_ptr): 81 | with open(env_ptr) as handle: 82 | return handle.read().strip() 83 | 84 | with open(self.get_env_conf()) as handle: 85 | conf = yaml.safe_load(handle.read()) 86 | if 'default' not in conf: 87 | raise ConfigError("No Environment specified") 88 | return conf['default'] 89 | 90 | def get_env_conf(self): 91 | """Get the environment config file. 92 | """ 93 | conf = os.path.join(self.juju_home, 'environments.yaml') 94 | if not os.path.exists(conf): 95 | raise ConfigError("Juju environments.yaml not found %s" % conf) 96 | return conf 97 | -------------------------------------------------------------------------------- /juju_scaleway/provider.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (c) 2014-2015 Online SAS and Contributors. All Rights Reserved. 4 | # Edouard Bonlieu 5 | # Julien Castets 6 | # Manfred Touron 7 | # Kevin Deldycke 8 | # 9 | # Licensed under the BSD 2-Clause License (the "License"); you may not use this 10 | # file except in compliance with the License. You may obtain a copy of the 11 | # License at http://opensource.org/licenses/BSD-2-Clause 12 | 13 | import logging 14 | import os 15 | import time 16 | import itertools 17 | 18 | from juju_scaleway.exceptions import ConfigError, ProviderError 19 | from juju_scaleway.client import Client 20 | 21 | logger = logging.getLogger("juju.scaleway") 22 | 23 | 24 | def factory(): 25 | cfg = Scaleway.get_config() 26 | return Scaleway(cfg) 27 | 28 | 29 | def validate(): 30 | Scaleway.get_config() 31 | 32 | 33 | class Scaleway(object): 34 | 35 | def __init__(self, config, client=None): 36 | self.config = config 37 | if client is None: 38 | self.client = Client( 39 | config['access_key'], 40 | config['secret_key']) 41 | 42 | @classmethod 43 | def get_config(cls): 44 | provider_conf = {} 45 | 46 | access_key = os.environ.get('SCALEWAY_ACCESS_KEY') 47 | if access_key: 48 | provider_conf['access_key'] = access_key 49 | 50 | secret_key = os.environ.get('SCALEWAY_SECRET_KEY') 51 | if secret_key: 52 | provider_conf['secret_key'] = secret_key 53 | 54 | if 'access_key' not in provider_conf or \ 55 | 'secret_key' not in provider_conf: 56 | raise ConfigError("Missing Scaleway api credentials") 57 | return provider_conf 58 | 59 | def get_servers(self): 60 | return self.client.get_servers() 61 | 62 | def get_server(self, server_id): 63 | return self.client.get_server(server_id) 64 | 65 | def launch_server(self, params): 66 | return self.client.create_server(**params) 67 | 68 | def terminate_server(self, server_id): 69 | self.client.destroy_server(server_id) 70 | 71 | def wait_on(self, server): 72 | # Wait up to 5 minutes, in 30 sec increments 73 | print(server.name) 74 | result = self._wait_on_server(server, 30, 10) 75 | if not result: 76 | raise ProviderError("Could not provision server before timeout") 77 | return result 78 | 79 | def _wait_on_server(self, server, limit, delay=10): 80 | # Redo cci.wait to give user feedback in verbose mode. 81 | for count, new_server in enumerate(itertools.repeat(server.id)): 82 | server = self.get_server(new_server) 83 | if server.state == 'running': 84 | return True 85 | if count >= limit: 86 | return False 87 | if count and count % 3 == 0: 88 | logger.debug( 89 | "Waiting for server:%s ip:%s waited:%ds", 90 | server.name, 91 | server.public_ip['address'] if server.public_ip else None, 92 | count * delay 93 | ) 94 | time.sleep(delay) 95 | -------------------------------------------------------------------------------- /juju_scaleway/ops.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (c) 2014-2015 Online SAS and Contributors. All Rights Reserved. 4 | # Edouard Bonlieu 5 | # Julien Castets 6 | # Manfred Touron 7 | # Kevin Deldycke 8 | # 9 | # Licensed under the BSD 2-Clause License (the "License"); you may not use this 10 | # file except in compliance with the License. You may obtain a copy of the 11 | # License at http://opensource.org/licenses/BSD-2-Clause 12 | 13 | import logging 14 | import time 15 | import subprocess 16 | 17 | from juju_scaleway.exceptions import TimeoutError 18 | from juju_scaleway import ssh 19 | 20 | 21 | logger = logging.getLogger("juju.scaleway") 22 | 23 | 24 | class MachineOp(object): 25 | 26 | def __init__(self, provider, env, params, **options): 27 | self.provider = provider 28 | self.env = env 29 | self.params = params 30 | self.created = time.time() 31 | self.options = options 32 | 33 | def run(self): 34 | raise NotImplementedError() 35 | 36 | 37 | class MachineAdd(MachineOp): 38 | 39 | timeout = 360 40 | delay = 8 41 | 42 | def run(self): 43 | server = self.provider.launch_server(self.params) 44 | self.provider.wait_on(server) 45 | server = self.provider.get_server(server.id) 46 | self.verify_ssh(server) 47 | return server 48 | 49 | def verify_ssh(self, server): 50 | """Workaround for manual provisioning and ssh availability. 51 | Manual provider bails immediately upon failure to connect on 52 | ssh, we loop to allow the server time to start ssh. 53 | """ 54 | max_time = self.timeout + time.time() 55 | running = False 56 | while max_time > time.time(): 57 | try: 58 | if ssh.check_ssh(server.public_ip['address']): 59 | running = True 60 | break 61 | except subprocess.CalledProcessError as exc: 62 | if ("Connection refused" in exc.output or 63 | "Connection timed out" in exc.output or 64 | "Connection closed" in exc.output or 65 | "Connection reset by peer" in exc.output): 66 | logger.debug( 67 | "Waiting for ssh on id:%s ip:%s name:%s remaining:%d", 68 | server.id, server.public_ip['address'], server.name, 69 | int(max_time-time.time())) 70 | time.sleep(self.delay) 71 | else: 72 | logger.error( 73 | "Could not ssh to server name: %s id: %s ip: %s\n%s", 74 | server.name, server.id, server.public_ip['address'], 75 | exc.output) 76 | raise 77 | 78 | if running is False: 79 | raise TimeoutError( 80 | "Could not provision id:%s name:%s ip:%s before timeout" % ( 81 | server.id, server.name, server.public_ip['address'])) 82 | 83 | 84 | class MachineRegister(MachineAdd): 85 | 86 | def run(self): 87 | server = super(MachineRegister, self).run() 88 | try: 89 | machine_id = self.env.add_machine( 90 | "ssh:root@%s" % server.public_ip['address'], 91 | key=self.options.get('key')) 92 | except: 93 | self.provider.terminate_server(server.id) 94 | raise 95 | return server, machine_id 96 | 97 | 98 | class MachineDestroy(MachineOp): 99 | 100 | def run(self): 101 | if not self.options.get('iaas_only'): 102 | self.env.terminate_machines([self.params['machine_id']]) 103 | if self.options.get('env_only'): 104 | return 105 | logger.debug("Destroying server %s", self.params['server_id']) 106 | self.provider.terminate_server(self.params['server_id']) 107 | -------------------------------------------------------------------------------- /juju_scaleway/client.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (c) 2014-2015 Online SAS and Contributors. All Rights Reserved. 4 | # Edouard Bonlieu 5 | # Julien Castets 6 | # Manfred Touron 7 | # Kevin Deldycke 8 | # 9 | # Licensed under the BSD 2-Clause License (the "License"); you may not use this 10 | # file except in compliance with the License. You may obtain a copy of the 11 | # License at http://opensource.org/licenses/BSD-2-Clause 12 | 13 | import os 14 | 15 | from juju_scaleway.exceptions import ProviderAPIError 16 | 17 | import json 18 | import requests 19 | 20 | 21 | class Entity(object): 22 | 23 | @classmethod 24 | def from_dict(cls, data): 25 | i = cls() 26 | i.__dict__.update(data) 27 | return i 28 | 29 | 30 | class Server(Entity): 31 | """ 32 | Attributes: id, name, image, state, public_ip, creation_date 33 | """ 34 | 35 | 36 | class Image(Entity): 37 | """ 38 | Attributes:, id, name, arch, public 39 | """ 40 | 41 | 42 | class Client(object): 43 | 44 | def __init__(self, access_key, secret_key): 45 | self.access_key = access_key 46 | self.secret_key = secret_key 47 | self.api_url_base = 'https://api.scaleway.com' 48 | 49 | def get_images(self): 50 | data = self.request("/images") 51 | return [Image.from_dict(image) for image in data.get("images", [])] 52 | 53 | def get_url(self, target): 54 | return "%s%s" % (self.api_url_base, target) 55 | 56 | def get_servers(self): 57 | data = self.request("/servers") 58 | return [Server.from_dict(server) for server in data.get('servers', [])] 59 | 60 | def get_server(self, server_id): 61 | data = self.request("/servers/%s" % (server_id)) 62 | return Server.from_dict(data.get('server', {})) 63 | 64 | def create_server(self, name, image): 65 | params = dict( 66 | name=name, 67 | image=image, 68 | organization=self.access_key) 69 | 70 | data = self.request('/servers', method='POST', params=params) 71 | server = Server.from_dict(data.get('server', {})) 72 | # Execute poweron action 73 | self.request('/servers/%s/action' % (server.id), 74 | method='POST', params={'action': 'poweron'}) 75 | 76 | return server 77 | 78 | def destroy_server(self, server_id): 79 | data = self.request('/servers/%s/action' % (server_id), 80 | method='POST', params={'action': 'terminate'}) 81 | return data.get('task') 82 | 83 | def request(self, target, method='GET', params=None): 84 | params = params and dict(params) or {} 85 | 86 | headers = {'User-Agent': 'juju/client'} 87 | headers = {'X-Auth-Token': self.secret_key} 88 | url = self.get_url(target) 89 | 90 | if method == 'POST': 91 | headers['Content-Type'] = "application/json" 92 | response = requests.post( 93 | url, headers=headers, data=json.dumps(params) 94 | ) 95 | else: 96 | response = requests.get(url, headers=headers, params=params) 97 | 98 | data = response.json() 99 | if not data: 100 | raise ProviderAPIError(response, 'No json result found') 101 | if response.status_code >= 400: 102 | raise ProviderAPIError(response, data['message']) 103 | 104 | return data 105 | 106 | @classmethod 107 | def connect(cls): 108 | access_key = os.environ.get('SCALEWAY_ACCESS_KEY') 109 | secret_key = os.environ.get('SCALEWAY_SECRET_KEY') 110 | if not access_key or not secret_key: 111 | raise KeyError("Missing api credentials") 112 | return cls(access_key, secret_key) 113 | 114 | 115 | def main(): 116 | import code 117 | client = Client.connect() 118 | code.interact(local={'client': client}) 119 | 120 | 121 | if __name__ == '__main__': 122 | main() 123 | -------------------------------------------------------------------------------- /juju_scaleway/cli.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (c) 2014-2015 Online SAS and Contributors. All Rights Reserved. 4 | # Edouard Bonlieu 5 | # Julien Castets 6 | # Manfred Touron 7 | # Kevin Deldycke 8 | # 9 | # Licensed under the BSD 2-Clause License (the "License"); you may not use this 10 | # file except in compliance with the License. You may obtain a copy of the 11 | # License at http://opensource.org/licenses/BSD-2-Clause 12 | 13 | import argparse 14 | import logging 15 | import sys 16 | 17 | from juju_scaleway.config import Config 18 | from juju_scaleway.constraints import SERIES_MAP 19 | from juju_scaleway.exceptions import ( 20 | ConfigError, PrecheckError, ProviderAPIError) 21 | from juju_scaleway import commands 22 | 23 | 24 | def _default_opts(parser): 25 | parser.add_argument( 26 | "-e", "--environment", help="Juju environment to operate on" 27 | ) 28 | parser.add_argument( 29 | "-v", "--verbose", action="store_true", help="Verbose output" 30 | ) 31 | 32 | 33 | def _machine_opts(parser): 34 | parser.add_argument( 35 | "--constraints", default="", help="Machine allocation criteria" 36 | ) 37 | parser.add_argument( 38 | "--series", default="trusty", choices=SERIES_MAP.values(), 39 | help="OS Release for machine." 40 | ) 41 | 42 | 43 | PLUGIN_DESCRIPTION = "Juju Scaleway client-side provider" 44 | 45 | 46 | def setup_parser(): 47 | if '--description' in sys.argv: 48 | print(PLUGIN_DESCRIPTION) 49 | sys.exit(0) 50 | 51 | parser = argparse.ArgumentParser(description=PLUGIN_DESCRIPTION) 52 | subparsers = parser.add_subparsers() 53 | bootstrap = subparsers.add_parser( 54 | 'bootstrap', 55 | help="Bootstrap an environment") 56 | _default_opts(bootstrap) 57 | _machine_opts(bootstrap) 58 | bootstrap.add_argument( 59 | "--upload-tools", 60 | action="store_true", default=False, 61 | help="upload local version of tools before bootstrapping") 62 | bootstrap.set_defaults(command=commands.Bootstrap) 63 | 64 | add_machine = subparsers.add_parser( 65 | 'add-machine', 66 | help="Add machines to an environment") 67 | add_machine.add_argument( 68 | "-n", "--num-machines", type=int, default=1, 69 | help="Number of machines to allocate") 70 | _default_opts(add_machine) 71 | _machine_opts(add_machine) 72 | add_machine.add_argument( 73 | "-k", "--ssh-key", default="", 74 | help="Use specified key when adding machines") 75 | add_machine.set_defaults(command=commands.AddMachine) 76 | 77 | list_machines = subparsers.add_parser( 78 | 'list-machines', 79 | help="List machines allocated to an environment.") 80 | _default_opts(list_machines) 81 | list_machines.add_argument( 82 | "-a", "--all", action="store_true", default=False, 83 | help="Display all servers in Scaleway.") 84 | list_machines.set_defaults(command=commands.ListMachines) 85 | 86 | terminate_machine = subparsers.add_parser( 87 | "terminate-machine", 88 | help="Terminate machine") 89 | terminate_machine.add_argument("machines", nargs="+") 90 | _default_opts(terminate_machine) 91 | terminate_machine.set_defaults(command=commands.TerminateMachine) 92 | 93 | destroy_environment = subparsers.add_parser( 94 | 'destroy-environment', 95 | help="Destroy all machines in juju environment") 96 | _default_opts(destroy_environment) 97 | destroy_environment.add_argument( 98 | "--force", action="store_true", default=False, 99 | help="Irrespective of environment state, destroy all env machines") 100 | destroy_environment.set_defaults(command=commands.DestroyEnvironment) 101 | 102 | return parser 103 | 104 | 105 | def main(): 106 | parser = setup_parser() 107 | options = parser.parse_args() 108 | config = Config(options) 109 | 110 | if config.verbose: 111 | level = logging.DEBUG 112 | else: 113 | level = logging.INFO 114 | logging.basicConfig( 115 | level=level, 116 | datefmt="%Y/%m/%d %H:%M.%S", 117 | format="%(asctime)s:%(levelname)s %(message)s") 118 | logging.getLogger('requests').setLevel(level=logging.WARNING) 119 | 120 | try: 121 | config.validate() 122 | except ConfigError as exc: 123 | print("Configuration error: %s" % str(exc)) 124 | sys.exit(1) 125 | 126 | cmd = options.command( 127 | config, 128 | config.connect_provider(), 129 | config.connect_environment()) 130 | try: 131 | cmd.run() 132 | except ProviderAPIError as exc: 133 | print("Provider interaction error: %s" % str(exc)) 134 | except ConfigError as exc: 135 | print("Configuration error: %s" % str(exc)) 136 | sys.exit(1) 137 | except PrecheckError as exc: 138 | print("Precheck error: %s" % str(exc)) 139 | sys.exit(1) 140 | 141 | if __name__ == '__main__': 142 | main() 143 | -------------------------------------------------------------------------------- /juju_scaleway/env.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (c) 2014-2015 Online SAS and Contributors. All Rights Reserved. 4 | # Edouard Bonlieu 5 | # Julien Castets 6 | # Manfred Touron 7 | # Kevin Deldycke 8 | # 9 | # Licensed under the BSD 2-Clause License (the "License"); you may not use this 10 | # file except in compliance with the License. You may obtain a copy of the 11 | # License at http://opensource.org/licenses/BSD-2-Clause 12 | 13 | try: 14 | import httplib 15 | except ImportError: # Python3 16 | import http.client as httplib 17 | 18 | import logging 19 | import shutil 20 | import subprocess 21 | import socket 22 | 23 | import os 24 | import yaml 25 | 26 | from juju_scaleway.constraints import SERIES_MAP 27 | 28 | 29 | logger = logging.getLogger("juju.scaleway") 30 | 31 | 32 | class Environment(object): 33 | 34 | def __init__(self, config): 35 | self.config = config 36 | 37 | def _run(self, command, env=None, capture_err=False): 38 | if env is None: 39 | env = dict(os.environ) 40 | env["JUJU_ENV"] = self.config.get_env_name() 41 | args = ['juju'] 42 | args.extend(command) 43 | logger.debug("Running juju command: %s", " ".join(args)) 44 | try: 45 | if capture_err: 46 | return subprocess.check_call( 47 | args, env=env, stderr=subprocess.STDOUT) 48 | return subprocess.check_output( 49 | args, env=env, stderr=subprocess.STDOUT) 50 | except subprocess.CalledProcessError as exc: 51 | logger.error( 52 | "Failed to run command %s\n%s", 53 | ' '.join(args), exc.output 54 | ) 55 | raise 56 | 57 | def status(self): 58 | return yaml.safe_load(self._run(['status'])) 59 | 60 | def is_running(self): 61 | """Try to connect the api server websocket to see if env is running. 62 | """ 63 | name = self.config.get_env_name() 64 | jenv = os.path.join( 65 | self.config.juju_home, "environments", "%s.jenv" % name) 66 | if not os.path.exists(jenv): 67 | return False 68 | with open(jenv) as handle: 69 | data = yaml.safe_load(handle.read()) 70 | if not data: 71 | return False 72 | conf = data.get('bootstrap-config') 73 | if not conf['type'] in ('manual', 'null'): 74 | return False 75 | conn = httplib.HTTPSConnection( 76 | conf['bootstrap-host'], port=17070, timeout=1.2) 77 | try: 78 | conn.request("GET", "/") 79 | return True 80 | except socket.error: 81 | return False 82 | 83 | def add_machine(self, location, key=None, debug=False): 84 | ops = ['add-machine', location] 85 | if key: 86 | ops.extend(['--ssh-key', key]) 87 | if debug: 88 | ops.append('--debug') 89 | 90 | return self._run(ops, capture_err=debug) 91 | 92 | def terminate_machines(self, machines): 93 | cmd = ['terminate-machine', '--force'] 94 | cmd.extend(machines) 95 | return self._run(cmd) 96 | 97 | def destroy_environment(self): 98 | cmd = [ 99 | 'destroy-environment', "-y", self.config.get_env_name()] 100 | return self._run(cmd) 101 | 102 | def destroy_environment_jenv(self): 103 | """Force remove client cache of environment by deleting jenv. 104 | Specifically this is for when we force/fast destroy an environment 105 | by deleting all the underlying iaas resources. Juju cli with --force 106 | will work, but will wait for a timeout to connect to the state server 107 | before doing the same. 108 | """ 109 | env_name = self.config.get_env_name() 110 | jenv_path = os.path.join( 111 | self.config.juju_home, "environments", "%s.jenv" % env_name) 112 | if os.path.exists(jenv_path): 113 | os.remove(jenv_path) 114 | 115 | def bootstrap(self): 116 | return self._run(['bootstrap', '-v']) 117 | 118 | def bootstrap_jenv(self, host): 119 | """Bootstrap an environment in a sandbox. 120 | Manual provider config keeps transient state in the form of 121 | bootstrap-host for its config. 122 | A temporary JUJU_HOME is used to modify environments.yaml 123 | """ 124 | env_name = self.config.get_env_name() 125 | 126 | # Prep a new juju home 127 | boot_home = os.path.join( 128 | self.config.juju_home, "boot-%s" % env_name) 129 | 130 | if not os.path.exists(boot_home): 131 | os.makedirs(os.path.join(boot_home, 'environments')) 132 | 133 | # Check that this installation has been used before. 134 | jenv_dir = os.path.join(self.config.juju_home, 'environments') 135 | if not os.path.exists(jenv_dir): 136 | os.mkdir(jenv_dir) 137 | 138 | ssh_key_dir = os.path.join(self.config.juju_home, 'ssh') 139 | 140 | # If no keys, create juju ssh keys via side effect. 141 | if not os.path.exists(ssh_key_dir): 142 | self._run(["switch"]) 143 | 144 | # Use existing juju ssh keys when bootstrapping 145 | shutil.copytree( 146 | ssh_key_dir, 147 | os.path.join(boot_home, 'ssh')) 148 | 149 | # Updated env config with the bootstrap host. 150 | with open(self.config.get_env_conf()) as handle: 151 | data = yaml.safe_load(handle.read()) 152 | env_conf = data['environments'].get(env_name) 153 | env_conf['bootstrap-host'] = host 154 | 155 | with open(os.path.join(boot_home, 'environments.yaml'), 'w') as handle: 156 | handle.write(yaml.safe_dump({ 157 | 'environments': {env_name: env_conf} 158 | })) 159 | 160 | # Change JUJU_ENV 161 | env = dict(os.environ) 162 | env['JUJU_HOME'] = boot_home 163 | env['JUJU_LOGGING'] = "=DEBUG" 164 | cmd = ['bootstrap', '--debug'] 165 | if self.config.upload_tools: 166 | cmd.append("--upload-tools") 167 | cmd.append('--series') 168 | cmd.append("%s" % (",".join(sorted(SERIES_MAP.values())))) 169 | 170 | capture_err = self.config.verbose and True or False 171 | try: 172 | self._run(cmd, env=env, capture_err=capture_err) 173 | # Copy over the jenv 174 | shutil.copy( 175 | os.path.join( 176 | boot_home, "environments", "%s.jenv" % env_name), 177 | os.path.join( 178 | self.config.juju_home, 179 | "environments", "%s.jenv" % env_name)) 180 | finally: 181 | shutil.rmtree(boot_home) 182 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Juju Scaleway provider 2 | ====================== 3 | 4 | Stable release: |release| |license| |dependencies| |popularity| 5 | 6 | Development: |build| |quality| |coverage| 7 | 8 | .. |release| image:: https://img.shields.io/pypi/v/juju-scaleway.svg?style=flat 9 | :target: https://pypi.python.org/pypi/juju-scaleway 10 | :alt: Last release 11 | .. |license| image:: https://img.shields.io/pypi/l/juju-scaleway.svg?style=flat 12 | :target: http://opensource.org/licenses/BSD-2-Clause 13 | :alt: Software license 14 | .. |popularity| image:: https://img.shields.io/pypi/dm/juju-scaleway.svg?style=flat 15 | :target: https://pypi.python.org/pypi/juju-scaleway#downloads 16 | :alt: Popularity 17 | .. |dependencies| image:: https://img.shields.io/requires/github/scaleway/juju-scaleway/master.svg?style=flat 18 | :target: https://requires.io/github/scaleway/juju-scaleway/requirements/?branch=master 19 | :alt: Requirements freshness 20 | .. |build| image:: https://img.shields.io/travis/scaleway/juju-scaleway/develop.svg?style=flat 21 | :target: https://travis-ci.org/scaleway/juju-scaleway 22 | :alt: Unit-tests status 23 | .. |coverage| image:: https://codecov.io/github/scaleway/juju-scaleway/coverage.svg?branch=develop 24 | :target: https://codecov.io/github/scaleway/juju-scaleway?branch=develop 25 | :alt: Coverage Status 26 | .. |quality| image:: https://img.shields.io/scrutinizer/g/scaleway/juju-scaleway.svg?style=flat 27 | :target: https://scrutinizer-ci.com/g/scaleway/juju-scaleway/?branch=develop 28 | :alt: Code Quality 29 | 30 | This package provides a CLI plugin for `Juju `_ to 31 | provision physical servers on `Scaleway `_, the first 32 | platform to offer dedicated ARM servers in the cloud. 33 | 34 | Juju provides for workloads management and orchestration using a collection of 35 | workloads definitions (charms) that can be assembled lego fashion at runtime 36 | into complex application topologies. 37 | 38 | This plugin is highly inspired by `@kapilt `_ Juju 39 | plugins. 40 | 41 | 42 | Installation 43 | ============ 44 | 45 | Linux 46 | ----- 47 | 48 | A usable version of Juju is available out of the box in Ubuntu 14.04 and later 49 | versions. For earlier versions of Ubuntu, please use the stable PPA: 50 | 51 | .. code-block:: bash 52 | 53 | $ sudo add-apt-repository ppa:juju/stable 54 | $ apt-get update && apt-get install juju 55 | 56 | 57 | Mac OS X 58 | -------- 59 | 60 | Juju is in Homebrew. To install Juju it is required to have `homebrew 61 | `_ installed. To install Juju run the following command: 62 | 63 | .. code-block:: bash 64 | 65 | $ brew install juju 66 | 67 | 68 | Plugin install (any OS) 69 | ----------------------- 70 | 71 | Plugin installation is done via ``pip`` which is the python language package 72 | managers, its available by default on Ubuntu. Also recommended is 73 | ``virtualenv`` to sandbox this install from your system packages: 74 | 75 | .. code-block:: bash 76 | 77 | $ pip install -U juju-scaleway 78 | 79 | 80 | Setup 81 | ===== 82 | 83 | **Requirements**: 84 | 85 | - You have an account and are logged into `scaleway.com 86 | `_; 87 | - You have configured your `SSH Key 88 | `_. 89 | 90 | 91 | Scaleway API keys 92 | ----------------- 93 | 94 | Provide the credentials required by the plugin using environment variables: 95 | 96 | .. code-block:: bash 97 | 98 | $ export SCALEWAY_ACCESS_KEY= 99 | $ export SCALEWAY_SECRET_KEY= 100 | 101 | 102 | Juju configuration 103 | ------------------ 104 | 105 | To configure a Juju environment for Scaleway, add the following in your 106 | ``~/.juju/environments.yaml``: 107 | 108 | .. code-block:: yaml 109 | 110 | environments: 111 | scaleway: 112 | type: manual 113 | bootstrap-host: null 114 | bootstrap-user: root 115 | 116 | 117 | Usage 118 | ===== 119 | 120 | You have to tell Juju which environment to use. One way to do this is to use 121 | the following command: 122 | 123 | .. code-block:: bash 124 | 125 | $ juju switch scaleway 126 | $ export JUJU_ENV=scaleway 127 | 128 | Now you can bootstrap your Scaleway environment: 129 | 130 | .. code-block:: bash 131 | 132 | $ juju scaleway bootstrap 133 | 134 | All machines created by this plugin will have the Juju environment name as a 135 | prefix for their servers name. 136 | 137 | After your environment is bootstrapped you can add additional machines to it 138 | via the the add-machine command, for instance the following will add 2 139 | additional machines: 140 | 141 | .. code-block:: bash 142 | 143 | $ juju scaleway add-machine -n 2 144 | $ juju status 145 | 146 | You can now use standard Juju commands for deploying service workloads aka 147 | charms: 148 | 149 | .. code-block:: bash 150 | 151 | $ juju deploy wordpress 152 | 153 | Without specifying the machine to place the workload on, the machine will 154 | automatically go to an unused machine within the environment. 155 | 156 | There are hundreds of available charms ready to be used, you can find out more 157 | about what's out there from at `jujucharms.com `_. Or 158 | alternatively the `'plain' html version 159 | `_. 160 | 161 | You can use manual placement to deploy target particular machines: 162 | 163 | .. code-block:: bash 164 | 165 | $ juju deploy mysql --to=2 166 | 167 | And of course the real magic of Juju comes in its ability to assemble these 168 | workloads together via relations like lego blocks: 169 | 170 | .. code-block:: bash 171 | 172 | $ juju add-relation wordpress mysql 173 | 174 | You can list all machines in Scaleway that are part of the Juju environment 175 | with the list-machines command. This directly queries the Scaleway API and does 176 | not interact with Juju API. 177 | 178 | .. code-block:: bash 179 | 180 | $ juju scaleway list-machines 181 | 182 | Id Name Status Created Address 183 | 6222349 scaleway-0 active 2014-11-25 212.47.239.232 184 | 6342360 scaleway-ef19ad5cc... active 2014-11-25 212.47.228.28 185 | 2224321 scaleway-145bf7a80... active 2014-11-25 212.47.228.79 186 | 187 | You can terminate allocated machines by their machine ID. By default with the 188 | Scaleway plugin, machines are forcibly terminated which will also terminate any 189 | service units on those machines: 190 | 191 | .. code-block:: bash 192 | 193 | $ juju scaleway terminate-machine 1 2 194 | 195 | And you can destroy the entire environment via: 196 | 197 | .. code-block:: bash 198 | 199 | $ juju scaleway destroy-environment 200 | 201 | ``destroy-environment`` also takes a ``--force`` option which only uses the 202 | Scaleway API. Its helpful if state server or other machines are killed 203 | independently of Juju. 204 | 205 | All commands have builtin help facilities and accept a ``-v`` option which will 206 | print verbose output while running. 207 | 208 | You can find out more about using from `Juju docs 209 | `_. 210 | 211 | 212 | License 213 | ======= 214 | 215 | This software is licensed under a `BSD 2-Clause License 216 | `_. 217 | -------------------------------------------------------------------------------- /juju_scaleway/commands.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (c) 2014-2015 Online SAS and Contributors. All Rights Reserved. 4 | # Edouard Bonlieu 5 | # Julien Castets 6 | # Manfred Touron 7 | # Kevin Deldycke 8 | # 9 | # Licensed under the BSD 2-Clause License (the "License"); you may not use this 10 | # file except in compliance with the License. You may obtain a copy of the 11 | # License at http://opensource.org/licenses/BSD-2-Clause 12 | 13 | import logging 14 | import time 15 | import uuid 16 | import yaml 17 | 18 | from juju_scaleway import constraints 19 | from juju_scaleway.exceptions import ConfigError, PrecheckError 20 | from juju_scaleway import ops 21 | from juju_scaleway.runner import Runner 22 | 23 | 24 | logger = logging.getLogger("juju.scaleway") 25 | 26 | 27 | class BaseCommand(object): 28 | 29 | def __init__(self, config, provider, environment): 30 | self.config = config 31 | self.provider = provider 32 | self.env = environment 33 | self.runner = Runner() 34 | 35 | def solve_constraints(self): 36 | start_time = time.time() 37 | image_map = constraints.get_images(self.provider.client) 38 | logger.debug("Looked up scaleway images in %0.2f seconds", 39 | time.time() - start_time) 40 | return image_map[self.config.series] 41 | 42 | def check_preconditions(self): 43 | """Check for provider and configured environments.yaml. 44 | """ 45 | env_name = self.config.get_env_name() 46 | with open(self.config.get_env_conf()) as handle: 47 | conf = yaml.safe_load(handle.read()) 48 | if 'environments' not in conf: 49 | raise ConfigError( 50 | "Invalid environments.yaml, no 'environments' section") 51 | if env_name not in conf['environments']: 52 | raise ConfigError( 53 | "Environment %r not in environments.yaml" % env_name) 54 | env = conf['environments'][env_name] 55 | if not env['type'] in ('null', 'manual'): 56 | raise ConfigError( 57 | "Environment %r provider type is %r must be 'null'" % ( 58 | env_name, env['type'])) 59 | if env['bootstrap-host']: 60 | raise ConfigError( 61 | "Environment %r already has a bootstrap-host" % ( 62 | env_name)) 63 | 64 | 65 | class Bootstrap(BaseCommand): 66 | """ 67 | Actions: 68 | - Launch an server 69 | - Wait for it to reach running state 70 | - Update environment in environments.yaml with bootstrap-host address. 71 | - Bootstrap juju environment 72 | Preconditions: 73 | - named environment found in environments.yaml 74 | - environment provider type is null 75 | - bootstrap-host must be null 76 | - ? existing scaleway with matching env name does not exist. 77 | """ 78 | def run(self): 79 | self.check_preconditions() 80 | image = self.solve_constraints() 81 | logger.info("Launching bootstrap host (eta 5m)...") 82 | params = dict( 83 | name="%s-0" % self.config.get_env_name(), image=image) 84 | 85 | machine = ops.MachineAdd( 86 | self.provider, self.env, params, series=self.config.series 87 | ) 88 | server = machine.run() 89 | 90 | logger.info("Bootstrapping environment...") 91 | try: 92 | self.env.bootstrap_jenv(server.public_ip['address']) 93 | except: 94 | self.provider.terminate_server(server.id) 95 | raise 96 | logger.info("Bootstrap complete.") 97 | 98 | def check_preconditions(self): 99 | result = super(Bootstrap, self).check_preconditions() 100 | if self.env.is_running(): 101 | raise PrecheckError( 102 | "Environment %s is already bootstrapped" % ( 103 | self.config.get_env_name())) 104 | return result 105 | 106 | 107 | class ListMachines(BaseCommand): 108 | 109 | def run(self): 110 | env_name = self.config.get_env_name() 111 | header = "{:<8} {:<18} {:<8} {:<12} {:<10}".format( 112 | "Id", "Name", "Status", "Created", "Address") 113 | 114 | allmachines = self.config.options.all 115 | for server in self.provider.get_servers(): 116 | name = server.name 117 | 118 | if not allmachines and not name.startswith('%s-' % env_name): 119 | continue 120 | 121 | if header: 122 | print(header) 123 | header = None 124 | 125 | if len(name) > 18: 126 | name = name[:15] + "..." 127 | 128 | print("{:<8} {:<18} {:<8} {:<12} {:<10}".format( 129 | server.id, 130 | name, 131 | server.state, 132 | server.creation_date[:-10], 133 | server.public_ip['address'] if server.public_ip else 'none' 134 | ).strip()) 135 | 136 | 137 | class AddMachine(BaseCommand): 138 | 139 | def run(self): 140 | self.check_preconditions() 141 | image = self.solve_constraints() 142 | logger.info("Launching %d servers...", self.config.num_machines) 143 | 144 | template = dict( 145 | image=image) 146 | 147 | for _ in range(self.config.num_machines): 148 | params = dict(template) 149 | params['name'] = "%s-%s" % ( 150 | self.config.get_env_name(), uuid.uuid4().hex) 151 | self.runner.queue_op( 152 | ops.MachineRegister( 153 | self.provider, self.env, params, series=self.config.series 154 | ) 155 | ) 156 | 157 | for (server, _) in self.runner.iter_results(): 158 | logger.info( 159 | "Registered id:%s name:%s ip:%s as juju machine", 160 | server.id, server.name, 161 | server.public_ip['address'] if server.public_ip else None 162 | ) 163 | 164 | 165 | class TerminateMachine(BaseCommand): 166 | 167 | def run(self): 168 | """Terminate machine in environment. 169 | """ 170 | self.check_preconditions() 171 | self._terminate_machines(lambda x: x in self.config.options.machines) 172 | 173 | def _terminate_machines(self, machine_filter): 174 | logger.debug("Checking for machines to terminate") 175 | status = self.env.status() 176 | machines = status.get('machines', {}) 177 | 178 | # Using the api server-id can be the provider id, but 179 | # else it defaults to ip, and we have to disambiguate. 180 | remove = [] 181 | for machine in machines: 182 | if machine_filter(machine): 183 | remove.append({ 184 | 'address': machines[machine]['dns-name'], 185 | 'server_id': machines[machine]['instance-id'], 186 | 'machine_id': machine 187 | }) 188 | 189 | address_map = dict([ 190 | (d.public_ip['address'] if d.public_ip else None, d) 191 | for d in self.provider.get_servers() 192 | ]) 193 | if not remove: 194 | return status, address_map 195 | 196 | logger.info( 197 | "Terminating machines %s", 198 | " ".join([machine['machine_id'] for machine in remove]) 199 | ) 200 | 201 | for machine in remove: 202 | server = address_map.get(machine['address']) 203 | env_only = False # Remove from only env or also provider. 204 | if server is None: 205 | logger.warning( 206 | "Couldn't resolve machine %s's address %s to server", 207 | machine['machine_id'], machine['address'] 208 | ) 209 | # We have a machine in juju state that we couldn't 210 | # find in provider. Remove it from state so destroy 211 | # can proceed. 212 | env_only = True 213 | server_id = None 214 | else: 215 | server_id = server.id 216 | self.runner.queue_op( 217 | ops.MachineDestroy( 218 | self.provider, self.env, { 219 | 'machine_id': machine['machine_id'], 220 | 'server_id': server_id 221 | }, 222 | env_only=env_only 223 | ) 224 | ) 225 | for _ in self.runner.iter_results(): 226 | pass 227 | 228 | return status, address_map 229 | 230 | 231 | class DestroyEnvironment(TerminateMachine): 232 | 233 | def run(self): 234 | """Destroy environment. 235 | """ 236 | self.check_preconditions() 237 | force = self.config.options.force 238 | 239 | # Manual provider needs machines removed prior to env destroy. 240 | def state_service_filter(machine): 241 | if machine == "0": 242 | return False 243 | return True 244 | 245 | if force: 246 | return self.force_environment_destroy() 247 | 248 | env_status, server_map = self._terminate_machines( 249 | state_service_filter 250 | ) 251 | 252 | # sadness, machines are marked dead, but juju is async to 253 | # reality. either sleep (racy) or retry loop, 10s seems to 254 | # plenty of time. 255 | time.sleep(10) 256 | 257 | logger.info("Destroying environment") 258 | self.env.destroy_environment() 259 | 260 | # Remove the state server. 261 | bootstrap_host = env_status.get( 262 | 'machines', {}).get('0', {}).get('dns-name') 263 | server = server_map.get(bootstrap_host) 264 | if server: 265 | logger.info("Terminating state server") 266 | self.provider.terminate_server(server.id) 267 | logger.info("Environment Destroyed") 268 | 269 | def force_environment_destroy(self): 270 | env_name = self.config.get_env_name() 271 | env_machines = [m for m in self.provider.get_servers() 272 | if m.name.startswith("%s-" % env_name)] 273 | 274 | logger.info("Destroying environment") 275 | for machine in env_machines: 276 | self.runner.queue_op( 277 | ops.MachineDestroy( 278 | self.provider, self.env, {'server_id': machine.id}, 279 | iaas_only=True 280 | ) 281 | ) 282 | 283 | for _ in self.runner.iter_results(): 284 | pass 285 | 286 | # Fast destroy the client cache by removing the jenv file. 287 | self.env.destroy_environment_jenv() 288 | logger.info("Environment Destroyed") 289 | --------------------------------------------------------------------------------