├── .dockerignore ├── .gitignore ├── .travis.yml ├── Dockerfile ├── HISTORY.rst ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.rst ├── django-cloudlaunch ├── cloudlaunch │ ├── __init__.py │ ├── admin.py │ ├── apps.py │ ├── authentication.py │ ├── backend_plugins │ │ ├── __init__.py │ │ ├── app_plugin.py │ │ ├── base_vm_app.py │ │ ├── cl_integration_test_app.py │ │ ├── cloudman2 │ │ │ ├── __init__.py │ │ │ ├── cloudman2_app.py │ │ │ ├── rancher2_aws_iam_policy.json │ │ │ ├── rancher2_aws_iam_trust_policy-cn.json │ │ │ └── rancher2_aws_iam_trust_policy.json │ │ ├── cloudman2_app.py │ │ ├── cloudman_app.py │ │ ├── docker_app.py │ │ ├── gvl_app.py │ │ ├── pulsar_app.py │ │ └── simple_web_app.py │ ├── configurers.py │ ├── forms.py │ ├── management │ │ └── commands │ │ │ ├── export_app_data.py │ │ │ ├── import_app_data.py │ │ │ └── serializers.py │ ├── migrations │ │ ├── 0001_initial.py │ │ └── __init__.py │ ├── models.py │ ├── serializers.py │ ├── signals.py │ ├── tasks.py │ ├── templates │ │ ├── admin │ │ │ └── import_data.html │ │ └── rest_framework │ │ │ └── api.html │ ├── tests │ │ ├── __init__.py │ │ ├── data │ │ │ ├── apps_new.yaml │ │ │ └── apps_update.yaml │ │ ├── test_api.py │ │ ├── test_launch.py │ │ └── test_mgmt_commands.py │ ├── urls.py │ ├── util.py │ ├── view_helpers.py │ └── views.py ├── cloudlaunchserver │ ├── __init__.py │ ├── apps.py │ ├── celery.py │ ├── celeryconfig.py │ ├── celeryconfig_test.py │ ├── runner │ │ ├── __init__.py │ │ ├── __main__.py │ │ ├── commands │ │ │ ├── __init__.py │ │ │ ├── django.py │ │ │ └── help.py │ │ └── decorators.py │ ├── settings.py │ ├── settings_local.py.sample │ ├── settings_prod.py │ ├── settings_test.py │ ├── urls.py │ └── wsgi.py ├── manage.py └── public_appliances │ ├── __init__.py │ ├── admin.py │ ├── apps.py │ ├── migrations │ ├── 0001_initial.py │ └── __init__.py │ ├── models.py │ ├── serializers.py │ ├── tests.py │ ├── urls.py │ └── views.py ├── docs ├── Makefile ├── conf.py ├── images │ ├── add-social-app-sm.png │ ├── add-social-app.png │ ├── github-oauth-app.png │ ├── github-ouath-app-sm.png │ ├── twitter-oauth-app-sm.png │ └── twitter-oauth-app.png ├── index.rst └── topics │ ├── configuration.rst │ ├── development_server_installation.rst │ ├── overview.rst │ ├── production_server_mgmt.rst │ └── social_auth.rst ├── requirements.txt ├── requirements_dev.txt ├── requirements_test.txt ├── setup.cfg ├── setup.py ├── tests ├── fixtures │ └── initial_test_data.json └── run_cli_integration_tests.sh └── tox.ini /.dockerignore: -------------------------------------------------------------------------------- 1 | # this file uses slightly different syntax than .gitignore, 2 | # e.g. ".tox/" will not ignore .tox directory 3 | 4 | # well, official docker build should be done on clean git checkout 5 | # anyway, so .tox should be empty... But I'm sure people will try to 6 | # test docker on their git working directories. 7 | 8 | .git 9 | .tox 10 | venv 11 | venv3 12 | docs 13 | redis-stable 14 | Dockerfile 15 | dist 16 | *.sqlite3 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | bin 2 | build 3 | include 4 | lib 5 | man 6 | local 7 | .Python 8 | *.pyc 9 | 10 | django-cloudlaunch/cloudlaunchserver/settings_local.py 11 | db.sqlite3* 12 | cloudlaunch*.log 13 | cloudlaunch-django.log 14 | django-cloudlaunch/.coverage 15 | django-cloudlaunch/cloudlaunch_server.egg-info 16 | 17 | .idea 18 | 19 | docs/_build/ 20 | 21 | *.DS_Store 22 | /venv/ 23 | /.tox/ 24 | 25 | codecloud.html 26 | dump.rdb 27 | .vscode/ -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: xenial 2 | sudo: false 3 | language: python 4 | cache: pip 5 | python: 3.6 6 | os: 7 | - linux 8 | # - osx 9 | env: 10 | - TOX_ENV=py36 11 | - TOX_ENV=cli_integration 12 | matrix: 13 | fast_finish: true 14 | allow_failures: 15 | - os: osx 16 | install: 17 | - pip install tox 18 | - pip install coveralls 19 | script: 20 | - tox -e $TOX_ENV 21 | after_success: 22 | - coveralls 23 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:20.04 as stage1 2 | 3 | ARG DEBIAN_FRONTEND=noninteractive 4 | ENV PYTHONUNBUFFERED 1 5 | 6 | RUN set -xe; \ 7 | apt-get -qq update && apt-get install -y --no-install-recommends \ 8 | apt-transport-https \ 9 | git-core \ 10 | make \ 11 | software-properties-common \ 12 | gcc \ 13 | python3-dev \ 14 | libffi-dev \ 15 | python3-pip \ 16 | python3-setuptools \ 17 | && apt-get autoremove -y && apt-get clean \ 18 | && rm -rf /var/lib/apt/lists/* /tmp/* \ 19 | && mkdir -p /app \ 20 | && pip3 install virtualenv \ 21 | && virtualenv -p python3 --prompt "(cloudlaunch)" /app/venv 22 | 23 | # Set working directory to /app/ 24 | WORKDIR /app/ 25 | 26 | # Only add files required for installation to improve build caching 27 | ADD requirements.txt /app 28 | ADD setup.py /app 29 | ADD README.rst /app 30 | ADD HISTORY.rst /app 31 | ADD django-cloudlaunch/cloudlaunchserver/__init__.py /app/django-cloudlaunch/cloudlaunchserver/__init__.py 32 | 33 | # Install requirements. Move this above ADD as 'pip install cloudlaunch-server' 34 | # asap so caching works 35 | RUN /app/venv/bin/pip3 install -U pip && /app/venv/bin/pip3 install --no-cache-dir -r requirements.txt 36 | 37 | # Stage-2 38 | FROM ubuntu:20.04 39 | 40 | ARG DEBIAN_FRONTEND=noninteractive 41 | ENV PYTHONUNBUFFERED 1 42 | ENV LC_ALL=en_US.UTF-8 43 | ENV LANG=en_US.UTF-8 44 | 45 | # Create cloudlaunch user environment 46 | RUN useradd -ms /bin/bash cloudlaunch \ 47 | && mkdir -p /app \ 48 | && chown cloudlaunch:cloudlaunch /app -R \ 49 | && apt-get -qq update && apt-get install -y --no-install-recommends \ 50 | git-core \ 51 | python3-pip \ 52 | python3-setuptools \ 53 | locales \ 54 | && locale-gen $LANG && update-locale LANG=$LANG \ 55 | && apt-get autoremove -y && apt-get clean \ 56 | && rm -rf /var/lib/apt/lists/* /tmp/* \ 57 | 58 | WORKDIR /app/ 59 | 60 | # Copy cloudlaunch files to final image 61 | COPY --chown=cloudlaunch:cloudlaunch --from=stage1 /app /app 62 | 63 | # Add the source files last to minimize layer cache invalidation 64 | ADD --chown=cloudlaunch:cloudlaunch . /app 65 | 66 | # Switch to new, lower-privilege user 67 | USER cloudlaunch 68 | 69 | WORKDIR /app/django-cloudlaunch/ 70 | 71 | RUN /app/venv/bin/python manage.py collectstatic --no-input 72 | 73 | # gunicorn will listen on this port 74 | EXPOSE 8000 75 | 76 | CMD /bin/bash -c "source /app/venv/bin/activate && /app/venv/bin/gunicorn -k gevent -b :8000 --access-logfile - --error-logfile - --log-level info cloudlaunchserver.wsgi" 77 | -------------------------------------------------------------------------------- /HISTORY.rst: -------------------------------------------------------------------------------- 1 | .. :changelog: 2 | 3 | History 4 | ------- 5 | 6 | 2.0.0 (2017-01-28) 7 | ++++++++++++++++++ 8 | 9 | * First release of the rewritten CloudLaunch on PyPI. 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Galaxy Project 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include django-cloudlaunch/cloudlaunch/templates * 2 | recursive-include django-cloudlaunch/cloudlaunch/management * -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: clean-pyc clean-build docs help 2 | .DEFAULT_GOAL := help 3 | define BROWSER_PYSCRIPT 4 | import os, webbrowser, sys 5 | try: 6 | from urllib import pathname2url 7 | except: 8 | from urllib.request import pathname2url 9 | 10 | webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1]))) 11 | endef 12 | export BROWSER_PYSCRIPT 13 | BROWSER := python -c "$$BROWSER_PYSCRIPT" 14 | 15 | help: 16 | @perl -nle'print $& if m{^[a-zA-Z_-]+:.*?## .*$$}' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-25s\033[0m %s\n", $$1, $$2}' 17 | 18 | clean: clean-build clean-pyc 19 | 20 | clean-build: ## remove build artifacts 21 | rm -fr build/ 22 | rm -fr dist/ 23 | rm -fr *.egg-info 24 | 25 | clean-pyc: ## remove Python file artifacts 26 | find . -name '*.pyc' -exec rm -f {} + 27 | find . -name '*.pyo' -exec rm -f {} + 28 | find . -name '*~' -exec rm -f {} + 29 | 30 | lint: ## check style with flake8 31 | flake8 django-cloudlaunch tests 32 | 33 | test: ## run tests quickly with the default Python 34 | python runtests.py tests 35 | 36 | test-all: ## run tests on every Python version with tox 37 | tox 38 | 39 | coverage: ## check code coverage quickly with the default Python 40 | coverage run --source django-cloudlaunch setup.py test 41 | coverage report -m 42 | coverage html 43 | open htmlcov/index.html 44 | 45 | docs: ## generate Sphinx HTML documentation, including API docs 46 | rm -f docs/django-cloudlaunch.rst 47 | rm -f docs/modules.rst 48 | sphinx-apidoc -o docs/ django-cloudlaunch 49 | $(MAKE) -C docs clean 50 | $(MAKE) -C docs html 51 | $(BROWSER) docs/_build/html/index.html 52 | 53 | release: clean ## package and upload a release 54 | python setup.py sdist upload 55 | python setup.py bdist_wheel upload 56 | 57 | sdist: clean ## package 58 | python setup.py sdist 59 | ls -l dist 60 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | .. image:: https://readthedocs.org/projects/cloudlaunch/badge/?version=latest 2 | :target: http://cloudlaunch.readthedocs.io/en/latest/?badge=latest 3 | :alt: Documentation Status 4 | 5 | =========== 6 | CloudLaunch 7 | =========== 8 | 9 | CloudLaunch is a ReSTful, extensible Django app for discovering and launching 10 | applications on cloud, container, or local infrastructure. A live version is 11 | available at https://launch.usegalaxy.org/. 12 | 13 | CloudLaunch can be extended with your own plug-ins, which can provide custom 14 | launch logic for arbitrary applications. Visit the live site to see 15 | currently available applications in the Catalog. CloudLaunch is also tightly 16 | integrated with `CloudBridge `_, 17 | which makes CloudLaunch natively multi-cloud. If you would like to have an 18 | additional cloud provider added as an available option for a given appliance, 19 | please create an issue in this repo. 20 | 21 | CloudLaunch has a web and commandline front-end. The Web UI is maintained in the 22 | `CloudLaunch-UI `_ repository. 23 | The commandline client is maintained in the 24 | `cloudlaunch-cli `_ repository. 25 | 26 | Installation 27 | ------------ 28 | 29 | On Kuberneets, via Helm 30 | *********************** 31 | The recommended way to install CloudLaunch is via the CloudLaunch Helm Chart: 32 | https://github.com/cloudve/cloudlaunch-helm 33 | 34 | 35 | Locally, via commandline 36 | ************************ 37 | 38 | 1. Install the CloudLaunch Django server 39 | 40 | .. code-block:: bash 41 | 42 | $ pip install cloudlaunch-server 43 | 44 | Once installed, you can run Django admin commands as follows: 45 | 46 | .. code-block:: bash 47 | 48 | $ cloudlaunch-server django 49 | 50 | 2. Copy ``cloudlaunchserver/settings_local.py.sample`` to 51 | ``cloudlaunchserver/settings_local.py`` and make any desired configuration 52 | changes. **Make sure to change** the value for ``FERNET_KEYS`` variable 53 | because it is used to encrypt sensitive database fields. 54 | 55 | 3. Prepare the database with: 56 | 57 | .. code-block:: bash 58 | 59 | $ cloudlaunch-server django migrate 60 | $ cloudlaunch-server django createsuperuser 61 | 62 | 4. Start the development server and celery task queue (along with a Redis 63 | server as the message broker), each process in its own tab. 64 | 65 | .. code-block:: bash 66 | 67 | $ python manage.py runserver 68 | $ redis-server & celery -A cloudlaunchserver worker -l info --beat 69 | 70 | 5. Visit http://127.0.0.1:8000/cloudlaunch/admin/ to define your application and 71 | infrastructure properties. 72 | 73 | 6 . Install the UI for the server by following instructions from 74 | https://github.com/galaxyproject/cloudlaunch-ui. 75 | 76 | 77 | Install Development Version 78 | --------------------------- 79 | 80 | CloudLaunch is based on Python 3.6 and although it may work on older Python 81 | versions, 3.6 is the only supported version. Use of Conda or virtualenv is also 82 | highly advised. 83 | 84 | 1. Checkout CloudLaunch and create an isolated environment 85 | 86 | .. code-block:: bash 87 | 88 | $ conda create --name cl --yes python=3.6 89 | $ conda activate cl 90 | $ git clone https://github.com/galaxyproject/cloudlaunch.git 91 | $ cd cloudlaunch 92 | $ pip install -r requirements_dev.txt 93 | $ cd django-cloudlaunch 94 | 95 | 2. Copy ``cloudlaunchserver/settings_local.py.sample`` to 96 | ``cloudlaunchserver/settings_local.py`` and make any desired configuration changes. 97 | 98 | 3. Run the migrations and create a superuser: 99 | 100 | .. code-block:: bash 101 | 102 | $ python manage.py migrate 103 | $ python manage.py createsuperuser 104 | 105 | 4. Start the web server and Celery in separate tabs 106 | 107 | .. code-block:: bash 108 | 109 | $ python manage.py runserver 110 | $ redis-server & celery -A cloudlaunchserver worker -l info --beat 111 | 112 | 5. Visit http://127.0.0.1:8000/cloudlaunch/admin/ to define appliances and 113 | add cloud providers. 114 | 115 | 6. Visit http://127.0.0.1:8000/cloudlaunch/api/v1/ to explore the API. 116 | 117 | 7 . Install the UI for the server by following instructions from 118 | https://github.com/galaxyproject/cloudlaunch-ui. 119 | 120 | 121 | Contributing 122 | ------------ 123 | 124 | Every PR should also bump the version or build number. Do this by running one 125 | of the following commands as part of the PR, which will create a commit: 126 | 127 | - For updating a dev version: ``bumpversion [major | minor | patch]`` 128 | eg, with current version 4.0.0, running ``bumpversion patch`` will result in 129 | *4.0.1-dev0* 130 | 131 | - For updating a build version: ``bumpversion build`` will result in 132 | *4.0.1-dev1* 133 | 134 | - For production version: ``bumpversion --tag release`` will result 135 | in *4.0.1*, with a git tag 136 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunch/__init__.py: -------------------------------------------------------------------------------- 1 | default_app_config = 'cloudlaunch.apps.CloudLaunchConfig' 2 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunch/admin.py: -------------------------------------------------------------------------------- 1 | """Models exposed via Django Admin.""" 2 | import ast 3 | from django.http import HttpResponse 4 | from django.core.management import call_command 5 | from django.conf import settings 6 | from django.contrib import admin 7 | from django.shortcuts import render 8 | from django.contrib import messages 9 | from django.utils.translation import gettext as _ 10 | import nested_admin 11 | 12 | import djcloudbridge 13 | 14 | from polymorphic.admin import PolymorphicChildModelAdmin 15 | from polymorphic.admin import PolymorphicParentModelAdmin 16 | from polymorphic.admin import PolymorphicChildModelFilter 17 | 18 | from . import forms 19 | from . import models 20 | 21 | 22 | class AppVersionCloudConfigInline(nested_admin.NestedTabularInline): 23 | model = models.ApplicationVersionCloudConfig 24 | extra = 1 25 | form = forms.ApplicationVersionCloudConfigForm 26 | 27 | 28 | class AppVersionInline(nested_admin.NestedStackedInline): 29 | model = models.ApplicationVersion 30 | extra = 0 31 | inlines = [AppVersionCloudConfigInline] 32 | form = forms.ApplicationVersionForm 33 | 34 | 35 | class AppAdmin(nested_admin.NestedModelAdmin): 36 | prepopulated_fields = {"slug": ("name",)} 37 | inlines = [AppVersionInline] 38 | form = forms.ApplicationForm 39 | list_display = ('name', 'default_version') 40 | ordering = ('display_order',) 41 | 42 | 43 | class AppCategoryAdmin(admin.ModelAdmin): 44 | model = models.AppCategory 45 | 46 | 47 | class CloudImageAdmin(admin.ModelAdmin): 48 | model = models.Image 49 | list_display = ('name', 'region', 'image_id') 50 | list_filter = ('region__cloud', 'region', 'name') 51 | ordering = ('region',) 52 | 53 | 54 | # Utility class for read-only fields 55 | class ReadOnlyTabularInline(admin.TabularInline): 56 | extra = 0 57 | can_delete = False 58 | editable_fields = [] 59 | readonly_fields = [] 60 | exclude = [] 61 | 62 | def get_readonly_fields(self, request, obj=None): 63 | return list(self.readonly_fields) + \ 64 | [field.name for field in self.model._meta.fields 65 | if field.name not in self.editable_fields and 66 | field.name not in self.exclude] 67 | 68 | 69 | class AppDeployTaskAdmin(ReadOnlyTabularInline): 70 | model = models.ApplicationDeploymentTask 71 | ordering = ('added',) 72 | 73 | 74 | class AppDeploymentsAdmin(admin.ModelAdmin): 75 | models = models.ApplicationDeployment 76 | list_display = ('name', 'archived', 'owner') 77 | list_filter = ('archived', 'owner__username') 78 | inlines = [AppDeployTaskAdmin] 79 | 80 | 81 | @admin.register(models.CloudDeploymentTarget) 82 | class AWSCloudAdmin(PolymorphicChildModelAdmin): 83 | base_model = models.CloudDeploymentTarget 84 | 85 | 86 | @admin.register(models.HostDeploymentTarget) 87 | class HostCloudAdmin(PolymorphicChildModelAdmin): 88 | base_model = models.HostDeploymentTarget 89 | 90 | 91 | @admin.register(models.KubernetesDeploymentTarget) 92 | class K8sCloudAdmin(PolymorphicChildModelAdmin): 93 | base_model = models.KubernetesDeploymentTarget 94 | 95 | 96 | @admin.register(models.DeploymentTarget) 97 | class DeploymentTargetAdmin(PolymorphicParentModelAdmin): 98 | base_model = models.DeploymentTarget 99 | child_models = (models.CloudDeploymentTarget, models.HostDeploymentTarget, 100 | models.KubernetesDeploymentTarget) 101 | list_display = ('id', 'custom_column') 102 | list_filter = (PolymorphicChildModelFilter,) 103 | 104 | def custom_column(self, obj): 105 | return models.DeploymentTarget.objects.get(pk=obj.id).__str__() 106 | custom_column.short_description = ("Deployment Target") 107 | 108 | 109 | class UsageAdmin(admin.ModelAdmin): 110 | models = models.Usage 111 | 112 | def deployment_target(self, obj): 113 | if obj.app_deployment: 114 | return obj.app_deployment.deployment_target 115 | return None 116 | deployment_target.short_description = 'Deployment Target' 117 | 118 | def application(self, obj): 119 | if obj.app_deployment: 120 | return obj.app_deployment.application_version.application.name 121 | return None 122 | 123 | def instance_type(self, obj): 124 | app_config = ast.literal_eval(obj.app_config) 125 | return app_config.get('config_cloudlaunch', {}).get('instanceType') 126 | 127 | # Enable column-based display&filtering of entries 128 | list_display = ('added', 'deployment_target', 'instance_type', 'application', 129 | 'user') 130 | # Enable filtering of displayed entries 131 | list_filter = ('added', 'app_deployment__deployment_target', 'user', 132 | 'app_deployment__application_version__application__name') 133 | # Enable hierarchical navigation by date 134 | date_hierarchy = 'added' 135 | ordering = ('-added',) 136 | # Add search 137 | search_fields = ['user'] 138 | 139 | 140 | class PublicKeyInline(admin.StackedInline): 141 | model = models.PublicKey 142 | extra = 1 143 | 144 | 145 | class UserProfileAdmin(djcloudbridge.admin.UserProfileAdmin): 146 | inlines = djcloudbridge.admin.UserProfileAdmin.inlines + [PublicKeyInline] 147 | 148 | 149 | admin.site.register(models.Application, AppAdmin) 150 | admin.site.register(models.AppCategory, AppCategoryAdmin) 151 | admin.site.register(models.ApplicationDeployment, AppDeploymentsAdmin) 152 | admin.site.register(models.Image, CloudImageAdmin) 153 | admin.site.register(models.Usage, UsageAdmin) 154 | 155 | # Add public key to existing UserProfile 156 | admin.site.unregister(djcloudbridge.models.UserProfile) 157 | admin.site.register(djcloudbridge.models.UserProfile, UserProfileAdmin) 158 | 159 | 160 | # Django Site Admin import/export actions 161 | 162 | def import_app_data(modeladmin, request, queryset): 163 | # All requests here will actually be of type POST 164 | # so we will need to check for our special key 'apply' 165 | # rather than the actual request type 166 | if request.POST.get('post'): 167 | # The user clicked submit on the intermediate form. 168 | # Perform our update action: 169 | app_registry_url = request.POST['app_registry_url'] 170 | call_command('import_app_data', '-u', app_registry_url) 171 | 172 | modeladmin.message_user( 173 | request, 174 | _("Successfully imported registry from url: %(app_registry_url)s") % { 175 | "app_registry_url": app_registry_url}, messages.SUCCESS) 176 | return None 177 | 178 | return render(request, 'admin/import_data.html', 179 | context={'app_registry_url': settings.CLOUDLAUNCH_APP_REGISTRY_URL, 180 | 'rows': queryset}) 181 | 182 | 183 | import_app_data.short_description = "Import app data from url" 184 | 185 | 186 | def export_app_data(modeladmin, request, queryset): 187 | response = HttpResponse(content_type="application/yaml") 188 | response['Content-Disposition'] = 'attachment; filename="app-registry.yaml"' 189 | response.write(call_command('export_app_data')) 190 | return response 191 | 192 | 193 | export_app_data.short_description = "Export app data to file" 194 | 195 | admin.site.add_action(import_app_data) 196 | admin.site.add_action(export_app_data) -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunch/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | class CloudLaunchConfig(AppConfig): 4 | name = 'cloudlaunch' 5 | 6 | def ready(self): 7 | import cloudlaunch.signals # noqa 8 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunch/authentication.py: -------------------------------------------------------------------------------- 1 | import rest_framework.authentication 2 | 3 | from .models import AuthToken 4 | 5 | 6 | # Override drf.authtoken's default create token and add a default token name. 7 | # This requires the following setting: 8 | # REST_AUTH_TOKEN_CREATOR = 'cloudlaunch.authentication.default_create_token' 9 | def default_create_token(token_model, user, serializer): 10 | token, _ = token_model.objects.get_or_create(user=user, name="default") 11 | return token 12 | 13 | 14 | # This is linked in from settings.py through DEFAULT_AUTHENTICATION_CLASSES 15 | # and will override DRF's default token auth. 16 | # Also requires setting REST_AUTH_TOKEN_MODEL = 'cloudlaunch.models.AuthToken' 17 | class TokenAuthentication(rest_framework.authentication.TokenAuthentication): 18 | model = AuthToken 19 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunch/backend_plugins/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/galaxyproject/cloudlaunch/e96cb45d1c2b19be9b14ac3202df0c708f017425/django-cloudlaunch/cloudlaunch/backend_plugins/__init__.py -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunch/backend_plugins/app_plugin.py: -------------------------------------------------------------------------------- 1 | """interface for app plugins.""" 2 | import abc 3 | 4 | 5 | class AppPlugin(): 6 | """Interface class for an application.""" 7 | 8 | __metaclass__ = abc.ABCMeta 9 | 10 | @staticmethod 11 | @abc.abstractmethod 12 | def validate_app_config(provider, name, cloud_config, app_config): 13 | """ 14 | Validate and build an internal app config. 15 | 16 | Validate an application config entered by the user and builds a new 17 | processed dictionary of values which will be used by the ``deploy`` 18 | method. Raises a ``ValidationError`` if the application configuration 19 | is invalid. This method must execute quickly and should not contain 20 | long running operations, and is designed to provide quick feedback on 21 | configuration errors to the client. 22 | 23 | @type provider: :class:`CloudBridge.CloudProvider` 24 | @param provider: Cloud provider where the supplied app is to be 25 | created. 26 | 27 | @type name: ``str`` 28 | @param name: Name for this deployment. 29 | 30 | @type cloud_config: ``dict`` 31 | @param cloud_config: A dict containing cloud infrastructure specific 32 | configuration for this app. 33 | 34 | @type app_config: ``dict`` 35 | @param app_config: A dict containing the original, unprocessed version 36 | of the app config. The app config is a merged dict 37 | of database stored settings and user-entered 38 | settings. 39 | 40 | :rtype: ``dict`` 41 | :return: A validated ``dict` containing the app launch configuration. 42 | """ 43 | pass 44 | 45 | @staticmethod 46 | @abc.abstractmethod 47 | def sanitise_app_config(app_config): 48 | """ 49 | Sanitise values in the app_config. 50 | 51 | The returned representation should have all sensitive data such 52 | as passwords and keys removed, so that it can be safely logged. 53 | 54 | @type app_config: ``dict`` 55 | @param app_config: A dict containing the original, unprocessed version 56 | of the app config. The app config is a merged dict 57 | of database stored settings and user-entered 58 | settings. 59 | 60 | :rtype: ``dict`` 61 | :return: A ``dict` containing the launch configuration. 62 | """ 63 | pass 64 | 65 | @abc.abstractmethod 66 | def deploy(self, name, task, app_config, provider_config): 67 | """ 68 | Deploy this app plugin on the supplied provider. 69 | 70 | Perform all the necessary steps to deploy this appliance. This may 71 | involve provisioning cloud resources or configuring existing host(s). 72 | See the definition of each method argument as some have required 73 | structure. 74 | 75 | This operation is designed to be a Celery task, and thus, can contain 76 | long-running operations. 77 | 78 | @type name: ``str`` 79 | @param name: Name of this deployment. 80 | 81 | @type task: :class:`Task` 82 | @param task: A Task object, which can be used to report progress. See 83 | ``tasks.Task`` for the interface details and sample 84 | implementation. 85 | 86 | @type app_config: ``dict`` 87 | @param app_config: A dict containing the appliance configuration. The 88 | app config is a merged dict of database stored 89 | settings and user-entered settings. In addition to 90 | the static configuration of the app, such as 91 | firewall rules or access password, this should 92 | contain a url to a host configuration playbook, if 93 | such configuration step is desired. For example: 94 | ``` 95 | { 96 | "config_cloudman": {}, 97 | "config_appliance": { 98 | "sshUser": "ubuntu", 99 | "runCmd": ["docker run -v /var/run/docker.sock:/var/run/docker.sock afgane/cloudman-boot"], 100 | "runner": "ansible", 101 | "repository": "https://github.com/afgane/Rancher-Ansible", 102 | "inventoryTemplate": "" 103 | }, 104 | "config_cloudlaunch": { 105 | "vmType": "c3.large", 106 | "firewall": [ { 107 | "securityGroup": "cloudlaunch-cm2", 108 | "rules": [ { 109 | "protocol": "tcp", 110 | "from": "22", 111 | "to": "22", 112 | "cidr": "0.0.0.0/0" 113 | } ] } ] } } 114 | ``` 115 | @type provider_config: ``dict`` 116 | @param provider_config: Define the details of the infrastructure 117 | provider where the appliance should be 118 | deployed. It is expected that this dictionary 119 | is composed within a task calling the plugin so 120 | it reflects the supplied info and derived 121 | properties. See ``tasks.py → create_appliance`` 122 | for an example. 123 | The following keys are supported: 124 | * ``cloud_provider``: CloudBridge object of the 125 | cloud provider 126 | * ``cloud_config``: A dict containing cloud 127 | infrastructure specific 128 | configuration for this app 129 | * ``cloud_user_data``: An object returned by 130 | ``validate_app_config()`` 131 | method which contains a 132 | validated and formatted 133 | version of the 134 | ``app_config`` to be 135 | supplied as instance 136 | user data 137 | * ``host_address``: A host IP address or a 138 | hostnames where to deploy 139 | this appliance 140 | * ``ssh_user``: User name with which to access 141 | the host(s) 142 | * ``ssh_public_key``: Public RSA ssh key to be 143 | used when running the app 144 | configuration step. This 145 | should be the actual key. 146 | CloudLaunch will auto-gen 147 | this key for provisioned 148 | instances. For hosted 149 | instances, the user 150 | should retrieve 151 | CloudLaunch's public key 152 | but this value should not 153 | be supplied. 154 | * ``ssh_private_key``: Private portion of an 155 | RSA ssh key. This should 156 | not be supplied by a 157 | user and is intended 158 | only for internal use. 159 | * ``run_cmd``: A list of strings with commands 160 | to run upon system boot. 161 | 162 | :rtype: ``dict`` 163 | :return: Results of the deployment process. 164 | """ 165 | pass 166 | 167 | @abc.abstractmethod 168 | def health_check(self, provider, deployment): 169 | """ 170 | Check the health of this app. 171 | 172 | At a minimum, this will check the status of the VM on which the 173 | deployment is running. Applications can implement more elaborate 174 | health checks. 175 | 176 | @type provider: :class:`CloudBridge.CloudProvider` 177 | @param provider: Cloud provider where the supplied deployment was 178 | created. 179 | 180 | @type deployment: ``dict`` 181 | @param deployment: A dictionary describing an instance of the 182 | app deployment. The dict must have at least 183 | `launch_result` and `launch_status` keys. 184 | 185 | :rtype: ``dict`` 186 | :return: A dictionary with possibly app-specific fields capturing 187 | app health. At a minimum, ``instance_status`` field will be 188 | available. If the deployment instance is not found by the 189 | provider, the default return value is ``deleted`` for the 190 | ``instance_status`` key. 191 | """ 192 | pass 193 | 194 | @abc.abstractmethod 195 | def restart(self, provider, deployment): 196 | """ 197 | Restart the appliance associated with the supplied deployment. 198 | 199 | This can simply restart the virtual machine on which the deployment 200 | is running or issue an app-specific call to perform the restart. 201 | 202 | @type provider: :class:`CloudBridge.CloudProvider` 203 | @param provider: Cloud provider where the supplied deployment was 204 | created. 205 | 206 | @type deployment: ``dict`` 207 | @param deployment: A dictionary describing an instance of the 208 | app deployment to be restarted. The dict must have 209 | at least `launch_result` and `launch_status` keys. 210 | 211 | :rtype: ``bool`` 212 | :return: The result of restart invocation. 213 | """ 214 | pass 215 | 216 | @abc.abstractmethod 217 | def delete(self, provider, deployment): 218 | """ 219 | Delete resource(s) associated with the supplied deployment. 220 | 221 | *Note* that this method will delete resource(s) associated with 222 | the deployment - this is an un-recoverable action. 223 | 224 | @type provider: :class:`CloudBridge.CloudProvider` 225 | @param provider: Cloud provider where the supplied deployment was 226 | created. 227 | 228 | @type deployment: ``dict`` 229 | @param deployment: A dictionary describing an instance of the 230 | app deployment to be deleted. The dict must have at 231 | least `launch_result` and `launch_status` keys. 232 | 233 | :rtype: ``bool`` 234 | :return: The result of delete invocation. 235 | """ 236 | pass 237 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunch/backend_plugins/cl_integration_test_app.py: -------------------------------------------------------------------------------- 1 | from cloudbridge.factory import CloudProviderFactory 2 | from cloudbridge.interfaces import TestMockHelperMixin 3 | from .base_vm_app import BaseVMAppPlugin 4 | 5 | 6 | class CloudLaunchIntegrationTestApp(BaseVMAppPlugin): 7 | """ 8 | This app replaces the provider with a mock version if available, 9 | and is intended to be used exclusively for testing. 10 | """ 11 | 12 | def _get_mock_provider(self, provider): 13 | """ 14 | Returns a mock version of a provider if available. 15 | """ 16 | provider_class = CloudProviderFactory().get_provider_class("mock") 17 | return provider_class(provider.config) 18 | 19 | def deploy(self, name, task, app_config, provider_config): 20 | # Replace provider with mock version if available 21 | provider = self._get_mock_provider(provider_config['cloud_provider']) 22 | if isinstance(provider, TestMockHelperMixin): 23 | provider.setUpMock() 24 | provider_config['cloud_provider'] = provider 25 | return super(CloudLaunchIntegrationTestApp, self).deploy( 26 | name, task, app_config, provider_config) 27 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunch/backend_plugins/cloudman2/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/galaxyproject/cloudlaunch/e96cb45d1c2b19be9b14ac3202df0c708f017425/django-cloudlaunch/cloudlaunch/backend_plugins/cloudman2/__init__.py -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunch/backend_plugins/cloudman2/rancher2_aws_iam_policy.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Effect": "Allow", 6 | "Action": [ 7 | "autoscaling:DescribeAutoScalingGroups", 8 | "autoscaling:DescribeLaunchConfigurations", 9 | "autoscaling:DescribeTags", 10 | "ec2:DescribeInstances", 11 | "ec2:DescribeRegions", 12 | "ec2:DescribeRouteTables", 13 | "ec2:DescribeSecurityGroups", 14 | "ec2:DescribeSubnets", 15 | "ec2:DescribeVolumes", 16 | "ec2:CreateSecurityGroup", 17 | "ec2:CreateTags", 18 | "ec2:CreateVolume", 19 | "ec2:ModifyInstanceAttribute", 20 | "ec2:ModifyVolume", 21 | "ec2:AttachVolume", 22 | "ec2:AuthorizeSecurityGroupIngress", 23 | "ec2:CreateRoute", 24 | "ec2:DeleteRoute", 25 | "ec2:DeleteSecurityGroup", 26 | "ec2:DeleteVolume", 27 | "ec2:DetachVolume", 28 | "ec2:RevokeSecurityGroupIngress", 29 | "ec2:DescribeVpcs", 30 | "ec2:DescribeAvailabilityZones", 31 | "ec2:DescribeInstanceTypes", 32 | "ec2:DescribeInstanceTypeOfferings", 33 | "elasticloadbalancing:AddTags", 34 | "elasticloadbalancing:AttachLoadBalancerToSubnets", 35 | "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer", 36 | "elasticloadbalancing:CreateLoadBalancerPolicy", 37 | "elasticloadbalancing:CreateLoadBalancerListeners", 38 | "elasticloadbalancing:ConfigureHealthCheck", 39 | "elasticloadbalancing:DeleteLoadBalancer", 40 | "elasticloadbalancing:DeleteLoadBalancerListeners", 41 | "elasticloadbalancing:DescribeLoadBalancers", 42 | "elasticloadbalancing:DescribeLoadBalancerAttributes", 43 | "elasticloadbalancing:DetachLoadBalancerFromSubnets", 44 | "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", 45 | "elasticloadbalancing:ModifyLoadBalancerAttributes", 46 | "elasticloadbalancing:RegisterInstancesWithLoadBalancer", 47 | "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer", 48 | "elasticloadbalancing:AddTags", 49 | "elasticloadbalancing:CreateListener", 50 | "elasticloadbalancing:CreateTargetGroup", 51 | "elasticloadbalancing:DeleteListener", 52 | "elasticloadbalancing:DeleteTargetGroup", 53 | "elasticloadbalancing:DescribeListeners", 54 | "elasticloadbalancing:DescribeLoadBalancerPolicies", 55 | "elasticloadbalancing:DescribeTargetGroups", 56 | "elasticloadbalancing:DescribeTargetHealth", 57 | "elasticloadbalancing:ModifyListener", 58 | "elasticloadbalancing:ModifyTargetGroup", 59 | "elasticloadbalancing:RegisterTargets", 60 | "elasticloadbalancing:SetLoadBalancerPoliciesOfListener", 61 | "iam:CreateServiceLinkedRole", 62 | "kms:DescribeKey" 63 | ], 64 | "Resource": [ 65 | "*" 66 | ] 67 | } 68 | ] 69 | } 70 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunch/backend_plugins/cloudman2/rancher2_aws_iam_trust_policy-cn.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Effect": "Allow", 6 | "Principal": { 7 | "Service": "ec2.amazonaws.com.cn" 8 | }, 9 | "Action": "sts:AssumeRole" 10 | } 11 | ] 12 | } 13 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunch/backend_plugins/cloudman2/rancher2_aws_iam_trust_policy.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Effect": "Allow", 6 | "Principal": { 7 | "Service": "ec2.amazonaws.com" 8 | }, 9 | "Action": "sts:AssumeRole" 10 | } 11 | ] 12 | } 13 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunch/backend_plugins/cloudman_app.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | 3 | from celery.utils.log import get_task_logger 4 | from urllib.parse import urlparse 5 | from rest_framework.serializers import ValidationError 6 | 7 | from .simple_web_app import SimpleWebAppPlugin 8 | 9 | log = get_task_logger('cloudlaunch') 10 | 11 | 12 | def get_required_val(data, name, message): 13 | val = data.get(name) 14 | if not val: 15 | raise ValidationError({"error": message}) 16 | return val 17 | 18 | 19 | class CloudManAppPlugin(SimpleWebAppPlugin): 20 | 21 | @staticmethod 22 | def validate_app_config(provider, name, cloud_config, app_config): 23 | cloudman_config = get_required_val( 24 | app_config, "config_cloudman", "CloudMan configuration data must be provided.") 25 | user_data = {} 26 | user_data['bucket_default'] = get_required_val( 27 | cloudman_config, "defaultBucket", "default bucket is required.") 28 | user_data['cm_remote_filename'] = cloudman_config.get('cm_remote_filename', 'cm.tar.gz') 29 | user_data['cluster_name'] = name 30 | if cloudman_config.get('restartCluster') and cloudman_config['restartCluster'].get('cluster_name'): 31 | user_data['cluster_name'] = cloudman_config['restartCluster']['cluster_name'] 32 | user_data['machine_image_id'] = cloudman_config['restartCluster'].get('persistent_data', {}).get('machine_image_id') 33 | user_data['placement'] = cloudman_config['restartCluster']['placement']['placement'] 34 | user_data['password'] = get_required_val( 35 | cloudman_config, "clusterPassword", "cluster password is required.") 36 | user_data['initial_cluster_type'] = get_required_val( 37 | cloudman_config, "clusterType", "cluster type is required.") 38 | user_data['cluster_storage_type'] = get_required_val( 39 | cloudman_config, "storageType", "storage type is required.") 40 | user_data['storage_type'] = user_data['cluster_storage_type'] 41 | user_data['storage_size'] = cloudman_config.get("storageSize") 42 | user_data['post_start_script_url'] = cloudman_config.get( 43 | "masterPostStartScript") 44 | user_data['worker_post_start_script_url'] = cloudman_config.get( 45 | "workerPostStartScript") 46 | if cloudman_config.get("clusterSharedString"): 47 | user_data['share_string'] = cloudman_config.get("clusterSharedString") 48 | user_data['cluster_templates'] = cloudman_config.get( 49 | "cluster_templates", []) 50 | # Adjust filesystem templates according to user selections 51 | for ct in user_data['cluster_templates']: 52 | for ft in ct.get('filesystem_templates', []): 53 | if 'galaxyData' in ft.get('roles', ''): 54 | ft['type'] = user_data['cluster_storage_type'] 55 | # File system template default value for file system size 56 | # overwrites the user-provided storage_size value so remove it 57 | # if both exits 58 | if ft.get('size') and user_data['storage_size']: 59 | del ft['size'] 60 | extra_user_data = cloudman_config.get("extraUserData") 61 | if extra_user_data: 62 | log.debug("Processing CloudMan extra user data: {0}" 63 | .format(extra_user_data)) 64 | for key, value in yaml.load(extra_user_data).items(): 65 | user_data[key] = value 66 | 67 | if provider.PROVIDER_ID == 'aws': 68 | user_data['cloud_type'] = 'ec2' 69 | user_data['region_name'] = provider.region_name 70 | user_data['region_endpoint'] = provider.ec2_cfg.get( 71 | 'endpoint_url') or 'ec2.amazonaws.com' 72 | user_data['ec2_port'] = None 73 | user_data['ec2_conn_path'] = '/' 74 | user_data['is_secure'] = provider.ec2_cfg.get('use_ssl') 75 | user_data['s3_host'] = provider.s3_cfg.get( 76 | 'endpoint_url') or 's3.amazonaws.com' 77 | user_data['s3_port'] = None 78 | user_data['s3_conn_path'] = '/' 79 | user_data['access_key'] = provider.session_cfg.get( 80 | 'aws_access_key_id') 81 | user_data['secret_key'] = provider.session_cfg.get( 82 | 'aws_secret_access_key') 83 | elif provider.PROVIDER_ID == 'openstack': 84 | user_data['cloud_type'] = 'openstack' 85 | ec2_endpoints = provider.security.get_ec2_endpoints() 86 | if not ec2_endpoints.get('ec2_endpoint'): 87 | raise ValidationError( 88 | {"error": "This version of CloudMan supports only " 89 | "EC2-compatible clouds. This OpenStack cloud " 90 | "provider does not appear to have an ec2 " 91 | "endpoint."}) 92 | uri_comp = urlparse(ec2_endpoints.get('ec2_endpoint')) 93 | 94 | user_data['region_name'] = provider.region_name 95 | user_data['region_endpoint'] = uri_comp.hostname 96 | user_data['ec2_port'] = uri_comp.port 97 | user_data['ec2_conn_path'] = uri_comp.path 98 | user_data['is_secure'] = uri_comp.scheme == "https" 99 | 100 | if ec2_endpoints.get('s3_endpoint'): 101 | uri_comp = urlparse(ec2_endpoints.get('s3_endpoint')) 102 | user_data['s3_host'] = uri_comp.hostname 103 | user_data['s3_port'] = uri_comp.port 104 | user_data['s3_conn_path'] = uri_comp.path 105 | else: 106 | user_data['use_object_store'] = False 107 | 108 | ec2_creds = provider.security.get_or_create_ec2_credentials() 109 | user_data['access_key'] = ec2_creds.access 110 | user_data['secret_key'] = ec2_creds.secret 111 | else: 112 | raise ValidationError({ 113 | "error": "This version of CloudMan supports only " 114 | "EC2-compatible clouds."}) 115 | 116 | return user_data 117 | 118 | @staticmethod 119 | def sanitise_app_config(app_config): 120 | app_config = super(CloudManAppPlugin, CloudManAppPlugin).sanitise_app_config(app_config) 121 | app_config['config_cloudman']['clusterPassword'] = '********' 122 | return app_config 123 | 124 | def deploy(self, name, task, app_config, provider_config): 125 | """See the parent class in ``app_plugin.py`` for the docstring.""" 126 | user_data = provider_config.get('cloud_user_data') 127 | ud = yaml.safe_dump(user_data, default_flow_style=False, 128 | allow_unicode=False) 129 | provider_config['cloud_user_data'] = ud 130 | # Make sure the placement and image ID propagate 131 | # (eg from a saved cluster) 132 | if user_data.get('placement'): 133 | app_config.get('config_cloudlaunch')[ 134 | 'placementZone'] = user_data['placement'] 135 | if user_data.get('machine_image_id'): 136 | app_config.get('config_cloudlaunch')[ 137 | 'customImageID'] = user_data['machine_image_id'] 138 | result = super(CloudManAppPlugin, self).deploy( 139 | name, task, app_config, provider_config, check_http=False) 140 | result['cloudLaunch']['applicationURL'] = 'http://{0}/cloud'.format( 141 | result['cloudLaunch']['hostname']) 142 | task.update_state( 143 | state='PROGRESSING', 144 | meta={'action': "Waiting for CloudMan to become ready at %s" 145 | % result['cloudLaunch']['applicationURL']}) 146 | log.info("CloudMan app going to wait for http") 147 | self.wait_for_http(result['cloudLaunch']['applicationURL']) 148 | return result 149 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunch/backend_plugins/docker_app.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | from rest_framework.serializers import ValidationError 3 | from .cloudman_app import CloudManAppPlugin 4 | from .base_vm_app import BaseVMAppPlugin 5 | 6 | 7 | class DockerAppPlugin(BaseVMAppPlugin): 8 | 9 | @staticmethod 10 | def validate_app_config(provider, name, cloud_config, app_config): 11 | docker_config = app_config.get('config_docker') 12 | if not docker_config: 13 | raise ValidationError("Docker configuration data must be provided.") 14 | docker_file_config = {} if not docker_config.get('docker_file') else docker_config['docker_file'] 15 | config_cloudlaunch = app_config.get('config_cloudlaunch') 16 | firewall_config = config_cloudlaunch.get('firewall', []) 17 | if firewall_config: 18 | security_group = firewall_config[0] 19 | security_rules = security_group.get('rules', []) 20 | else: 21 | security_rules = [] 22 | security_group = {'securityGroup': 'cloudlaunch_docker', 23 | 'description': 'Security group for docker containers', 24 | 'rules': security_rules} 25 | firewall_config.append(security_group) 26 | user_data = "#!/bin/bash\ndocker run -d" 27 | for mapping in docker_file_config.get('port_mappings', {}): 28 | host_port = mapping.get('host_port') 29 | if host_port: 30 | user_data += " -p {0}:{1}".format(host_port, mapping.get('container_port')) 31 | security_rules.append( 32 | { 33 | 'protocol': 'tcp', 34 | 'from': host_port, 35 | 'to': host_port, 36 | 'cidr': '0.0.0.0/0' }) 37 | security_group['rules'] = security_rules 38 | config_cloudlaunch['firewall'] = firewall_config 39 | 40 | for envvar in docker_file_config.get('env_vars', {}): 41 | envvar_name = envvar.get('variable') 42 | if envvar_name: 43 | user_data += " -e \"{0}={1}\"".format(envvar_name, envvar.get('value')) 44 | 45 | for vol in docker_file_config.get('volumes', {}): 46 | container_path = vol.get('container_path') 47 | if container_path: 48 | user_data += " -v {0}:{1}:{2}".format(vol.get('host_path'), 49 | container_path, 50 | 'rw' if vol.get('read_write') else 'r') 51 | 52 | user_data += " {0}".format(docker_config.get('repo_name')) 53 | return user_data 54 | 55 | def deploy(self, name, task, app_config, provider_config): 56 | result = super(DockerAppPlugin, self).deploy( 57 | name, task, app_config, provider_config) 58 | result['cloudLaunch']['applicationURL'] = 'http://{0}'.format( 59 | result['cloudLaunch']['hostname']) 60 | return result 61 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunch/backend_plugins/gvl_app.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | from rest_framework.serializers import ValidationError 3 | from .cloudman_app import CloudManAppPlugin 4 | from .simple_web_app import SimpleWebAppPlugin 5 | 6 | 7 | class GVLAppPlugin(SimpleWebAppPlugin): 8 | 9 | @staticmethod 10 | def validate_app_config(provider, name, cloud_config, app_config): 11 | gvl_config = app_config.get("config_gvl") 12 | if not gvl_config: 13 | raise ValidationError("GVL configuration data must be provided.") 14 | user_data = CloudManAppPlugin().validate_app_config( 15 | provider, name, cloud_config, gvl_config) 16 | install_list = [] 17 | install_cmdline = gvl_config.get('gvl_cmdline_utilities', False) 18 | if install_cmdline: 19 | install_list.append('gvl_cmdline_utilities') 20 | install_smrtportal = gvl_config.get('smrt_portal', False) 21 | if install_smrtportal: 22 | install_list.append('smrt_portal') 23 | user_data['gvl_config'] = {'install': install_list} 24 | user_data['gvl_package_registry_url'] = gvl_config.get('gvl_package_registry_url') 25 | return user_data 26 | 27 | @staticmethod 28 | def sanitise_app_config(app_config): 29 | sanitised_config = super(GVLAppPlugin, GVLAppPlugin).sanitise_app_config(app_config) 30 | gvl_config = sanitised_config.get("config_gvl") 31 | sanitised_config['config_gvl'] = CloudManAppPlugin().sanitise_app_config(gvl_config) 32 | return sanitised_config 33 | 34 | def deploy(self, name, task, app_config, provider_config): 35 | user_data = provider_config.get('cloud_user_data') 36 | ud = yaml.safe_dump(user_data, default_flow_style=False, 37 | allow_unicode=False) 38 | provider_config['cloud_user_data'] = ud 39 | result = super(GVLAppPlugin, self).deploy( 40 | name, task, app_config, provider_config) 41 | return result 42 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunch/backend_plugins/pulsar_app.py: -------------------------------------------------------------------------------- 1 | import secrets 2 | 3 | from cloudlaunch.configurers import AnsibleAppConfigurer 4 | 5 | from .base_vm_app import BaseVMAppPlugin 6 | 7 | 8 | class PulsarAppPlugin(BaseVMAppPlugin): 9 | 10 | def deploy(self, name, task, app_config, provider_config): 11 | token = secrets.token_urlsafe() 12 | if not app_config.get('config_pulsar'): 13 | app_config['config_pulsar'] = {} 14 | app_config['config_pulsar']['auth_token'] = token 15 | result = super(PulsarAppPlugin, self).deploy( 16 | name, task, app_config, provider_config) 17 | result['pulsar'] = { 18 | 'api_url': 'http://{0}:8913'.format(result['cloudLaunch']['hostname']), 19 | 'auth_token': token} 20 | return result 21 | 22 | def _get_configurer(self, app_config): 23 | return PulsarAnsibleAppConfigurer() 24 | 25 | 26 | class PulsarAnsibleAppConfigurer(AnsibleAppConfigurer): 27 | """Add Pulsar specific vars to playbook.""" 28 | 29 | def configure(self, app_config, provider_config): 30 | playbook_vars = { 31 | 'docker_container_name': 'Pulsar', 32 | 'docker_boot_image': app_config.get('config_pulsar', {}).get( 33 | 'pulsar_image', 'galaxy/pulsar:cvmfs'), 34 | 'docker_ports': ['8913:8913'], 35 | 'docker_env': { 36 | 'PULSAR_CONFIG_PRIVATE_TOKEN': app_config['config_pulsar']['auth_token'] 37 | } 38 | } 39 | return super().configure(app_config, provider_config, 40 | playbook_vars=playbook_vars) 41 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunch/backend_plugins/simple_web_app.py: -------------------------------------------------------------------------------- 1 | """Plugin implementation for a simple web application.""" 2 | import time 3 | 4 | from celery.utils.log import get_task_logger 5 | import requests 6 | import requests.exceptions 7 | 8 | from .base_vm_app import BaseVMAppPlugin 9 | 10 | log = get_task_logger('cloudlaunch') 11 | 12 | 13 | class SimpleWebAppPlugin(BaseVMAppPlugin): 14 | """ 15 | Implementation for an appliance exposing a web interface. 16 | 17 | The implementation is based on the Base VM app except that it expects 18 | a web frontend. 19 | """ 20 | 21 | def wait_for_http(self, url, ok_status_codes=None, max_retries=200, 22 | poll_interval=5): 23 | """ 24 | Wait till app is responding at http URL. 25 | 26 | :type ok_status_codes: ``list`` of int 27 | :param ok_status_codes: List of HTTP status codes that are considered 28 | OK by the appliance. Code 200 is assumed. 29 | """ 30 | if ok_status_codes is None: 31 | ok_status_codes = [401, 403] 32 | count = 0 33 | while count < max_retries: 34 | time.sleep(poll_interval) 35 | try: 36 | r = requests.head(url, verify=False) 37 | r.raise_for_status() 38 | return 39 | except requests.exceptions.HTTPError as http_exc: 40 | if http_exc.response.status_code in ok_status_codes: 41 | return 42 | except requests.exceptions.ConnectionError: 43 | pass 44 | count += 1 45 | 46 | def deploy(self, name, task, app_config, provider_config, **kwargs): 47 | """ 48 | Handle the app launch process and wait for http. 49 | 50 | Pass boolean ``check_http`` as a ``False`` kwarg if you don't 51 | want this method to perform the app http check and prefer to handle 52 | it in the child class. 53 | """ 54 | result = super(SimpleWebAppPlugin, self).deploy( 55 | name, task, app_config, provider_config) 56 | check_http = kwargs.get('check_http', True) 57 | if check_http and result.get('cloudLaunch', {}).get('hostname') and \ 58 | not result.get('cloudLaunch', {}).get('applicationURL'): 59 | log.info("Simple web app going to wait for http") 60 | result['cloudLaunch']['applicationURL'] = \ 61 | 'http://%s/' % result['cloudLaunch']['hostname'] 62 | task.update_state( 63 | state='PROGRESSING', 64 | meta={"action": "Waiting for application to become ready at %s" 65 | % result['cloudLaunch']['applicationURL']}) 66 | log.info("Waiting on http at %s", 67 | result['cloudLaunch']['applicationURL']) 68 | self.wait_for_http(result['cloudLaunch']['applicationURL'], 69 | ok_status_codes=[], max_retries=200, 70 | poll_interval=5) 71 | elif not result.get('cloudLaunch', {}).get('applicationURL'): 72 | result['cloudLaunch']['applicationURL'] = 'N/A' 73 | return result 74 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunch/configurers.py: -------------------------------------------------------------------------------- 1 | """Application configurers.""" 2 | import abc 3 | import logging 4 | import os 5 | import shutil 6 | import socket 7 | import subprocess 8 | from io import StringIO 9 | from string import Template 10 | import yaml 11 | 12 | from django.conf import settings 13 | 14 | from git import Repo 15 | 16 | import paramiko 17 | from paramiko.ssh_exception import AuthenticationException 18 | from paramiko.ssh_exception import BadHostKeyException 19 | from paramiko.ssh_exception import SSHException 20 | 21 | import tenacity 22 | 23 | import requests 24 | 25 | log = logging.getLogger(__name__) 26 | 27 | DEFAULT_INVENTORY_TEMPLATE = """ 28 | ${host} 29 | 30 | [all:vars] 31 | ansible_ssh_port=22 32 | ansible_user='${user}' 33 | ansible_ssh_private_key_file=pk 34 | ansible_ssh_extra_args='-o StrictHostKeyChecking=no' 35 | """.strip() 36 | 37 | 38 | def create_configurer(app_config): 39 | """Create a configurer based on the 'runner' in app_config.""" 40 | # Default to ansible if no runner 41 | runner = app_config.get('config_appliance', {}).get('runner', 'ansible') 42 | if runner == "ansible": 43 | return AnsibleAppConfigurer() 44 | elif runner == "script": 45 | return ScriptAppConfigurer() 46 | else: 47 | raise ValueError("Unsupported value of 'runner': {}".format(runner)) 48 | 49 | 50 | class AppConfigurer(): 51 | """Interface class for application configurer.""" 52 | 53 | __metaclass__ = abc.ABCMeta 54 | 55 | @abc.abstractmethod 56 | def validate(self, app_config, provider_config): 57 | """Throws exception if provider_config or app_config isn't valid.""" 58 | pass 59 | 60 | @abc.abstractmethod 61 | def configure(self, app_config, provider_config): 62 | """ 63 | Configure application on already provisioned host. 64 | 65 | See AppPlugin.deploy for additional documentation on arguments. 66 | """ 67 | pass 68 | 69 | 70 | class SSHBasedConfigurer(AppConfigurer): 71 | 72 | def validate(self, app_config, provider_config): 73 | # Validate SSH connection info in provider_config 74 | host_config = provider_config.get('host_config', {}) 75 | host = host_config.get('host_address') 76 | user = host_config.get('ssh_user') 77 | ssh_private_key = host_config.get('ssh_private_key') 78 | log.debug("Config ssh key:\n%s", ssh_private_key) 79 | try: 80 | self._check_ssh(host, pk=ssh_private_key, user=user) 81 | except tenacity.RetryError as rte: 82 | raise Exception("Error trying to ssh to host {}: {}".format( 83 | host, rte)) 84 | 85 | def _remove_known_host(self, host): 86 | """ 87 | Remove a host from ~/.ssh/known_hosts. 88 | 89 | :type host: ``str`` 90 | :param host: Hostname or IP address of the host to remove from the 91 | known hosts file. 92 | 93 | :rtype: ``bool`` 94 | :return: True if the host was successfully removed. 95 | """ 96 | cmd = "ssh-keygen -R {0}".format(host) 97 | p = subprocess.Popen( 98 | cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) 99 | (out, err) = p.communicate() 100 | if p.wait() == 0: 101 | return True 102 | return False 103 | 104 | @tenacity.retry(stop=tenacity.stop_after_delay(180), 105 | retry=tenacity.retry_if_result(lambda result: result is False), 106 | wait=tenacity.wait_fixed(5)) 107 | def _check_ssh(self, host, pk=None, user='ubuntu'): 108 | """ 109 | Check for ssh availability on a host. 110 | 111 | :type host: ``str`` 112 | :param host: Hostname or IP address of the host to check. 113 | 114 | :type pk: ``str`` 115 | :param pk: Private portion of an ssh key. 116 | 117 | :type user: ``str`` 118 | :param user: Username to use when trying to login. 119 | 120 | :rtype: ``bool`` 121 | :return: True if ssh connection was successful. 122 | """ 123 | ssh = paramiko.SSHClient() 124 | ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) 125 | pkey = self._get_private_key_from_string(pk) 126 | try: 127 | log.info("Trying to ssh {0}@{1}".format(user, host)) 128 | ssh.connect(host, username=user, pkey=pkey) 129 | self._remove_known_host(host) 130 | return True 131 | except (BadHostKeyException, AuthenticationException, 132 | SSHException, socket.error) as e: 133 | log.warn("ssh connection exception for {0}: {1}".format(host, e)) 134 | self._remove_known_host(host) 135 | return False 136 | 137 | def _get_private_key_from_string(self, private_key): 138 | pkey = None 139 | if private_key: 140 | if 'RSA' not in private_key: 141 | # Paramiko requires key type so add it 142 | log.info("Augmenting private key with RSA type") 143 | private_key = private_key.replace(' PRIVATE', ' RSA PRIVATE') 144 | key_file_object = StringIO(private_key) 145 | pkey = paramiko.RSAKey.from_private_key(key_file_object) 146 | key_file_object.close() 147 | return pkey 148 | 149 | 150 | class ScriptAppConfigurer(SSHBasedConfigurer): 151 | 152 | def validate(self, app_config, provider_config): 153 | super().validate(app_config, provider_config) 154 | config_script = app_config.get('config_appliance', {}).get( 155 | 'config_script') 156 | if not config_script: 157 | raise Exception("config_appliance missing required parameter: " 158 | "config_script") 159 | 160 | def configure(self, app_config, provider_config): 161 | host_config = provider_config.get('host_config', {}) 162 | host = host_config.get('host_address') 163 | user = host_config.get('ssh_user') 164 | ssh_private_key = host_config.get('ssh_private_key') 165 | # TODO: maybe add support for running multiple commands, but how to 166 | # distinguish from run_cmd? 167 | config_script = app_config.get('config_appliance', {}).get( 168 | 'config_script') 169 | 170 | try: 171 | pkey = self._get_private_key_from_string(ssh_private_key) 172 | ssh = paramiko.SSHClient() 173 | ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) 174 | log.info("Trying to ssh {0}@{1}".format(user, host)) 175 | ssh.connect(host, username=user, pkey=pkey) 176 | stdin, stdout, stderr = ssh.exec_command(config_script) 177 | self._remove_known_host(host) 178 | return { 179 | 'stdout': stdout.read(), 180 | 'stderr': stderr.read() 181 | } 182 | except SSHException as sshe: 183 | raise Exception("Failed to execute '{}' on {}".format( 184 | config_script, host)) from sshe 185 | 186 | 187 | class AnsibleAppConfigurer(SSHBasedConfigurer): 188 | 189 | def validate(self, app_config, provider_config): 190 | super().validate(app_config, provider_config) 191 | 192 | # validate required app_config values 193 | playbooks = app_config.get('config_appliance', {}).get('playbooks') 194 | # backward compatibility 195 | repository = app_config.get('config_appliance', {}).get('repository') 196 | if not playbooks and not repository: 197 | raise Exception("config_appliance missing required parameter: " 198 | "playbooks") 199 | 200 | def configure(self, app_config, provider_config, playbook_vars=None): 201 | host_config = provider_config.get('host_config', {}) 202 | host = host_config.get('host_address') 203 | user = host_config.get('ssh_user') 204 | ssh_private_key = host_config.get('ssh_private_key') 205 | if 'RSA' not in ssh_private_key: 206 | ssh_private_key = ssh_private_key.replace( 207 | ' PRIVATE', ' RSA PRIVATE') 208 | log.debug("Augmented ssh key with RSA type: %s" % ssh_private_key) 209 | 210 | playbooks = app_config.get('config_appliance', {}).get('playbooks', []) 211 | # backward compatibility 212 | if app_config.get('config_appliance', {}).get('repository'): 213 | playbook_url = app_config.get('config_appliance', {}).get('repository') 214 | if 'inventoryTemplate' in app_config.get('config_appliance', {}): 215 | inventory = app_config.get( 216 | 'config_appliance', {}).get('inventoryTemplate') 217 | else: 218 | inventory = DEFAULT_INVENTORY_TEMPLATE 219 | playbooks += [{ 220 | 'url': playbook_url, 221 | 'inventory_template': inventory 222 | }] 223 | for playbook in sorted(playbooks, key=lambda p: int(p.get('ordinal', 0))): 224 | playbook_url = playbook.get('url') 225 | inventory = playbook.get('inventory_template') or DEFAULT_INVENTORY_TEMPLATE 226 | self._run_playbook(playbook_url, inventory, host, ssh_private_key, user, 227 | playbook_vars) 228 | return {} 229 | 230 | def _run_playbook(self, playbook, inventory, host, pk, user='ubuntu', 231 | playbook_vars=None): 232 | """ 233 | Run an Ansible playbook to configure a host. 234 | 235 | First clone a playbook from the supplied repo if not already 236 | available, configure the Ansible inventory, and run the playbook. 237 | 238 | The method assumes ``ansible-playbook`` system command is available. 239 | 240 | :type playbook: ``str`` 241 | :param playbook: A URL of a git repository where the playbook resides. 242 | 243 | :type inventory: ``str`` 244 | :param inventory: A string ``Template``-like file 245 | that will be used for running the playbook. The 246 | file should have defined variables for ``host`` and 247 | ``user``. 248 | 249 | :type playbook_vars: ``list`` of tuples 250 | :param playbook_vars: A list of key/value tuples with variables to pass 251 | to the playbook via command line arguments 252 | (i.e., --extra-vars key=value). 253 | 254 | :type host: ``str`` 255 | :param host: Hostname or IP of a machine as the playbook target. 256 | 257 | :type pk: ``str`` 258 | :param pk: Private portion of an ssh key. 259 | 260 | :type user: ``str`` 261 | :param user: Target host system username with which to login. 262 | """ 263 | # Clone the repo in its own dir if multiple tasks run simultaneously 264 | # The path must be to a folder that doesn't already contain a git repo, 265 | # including any parent folders 266 | # TODO: generalize this temporary directory 267 | repo_path = '/tmp/cloudlaunch_plugin_runners/rancher_ansible_%s' % host 268 | try: 269 | log.info("Delete plugin runner folder %s if not empty", repo_path) 270 | shutil.rmtree(repo_path) 271 | except FileNotFoundError: 272 | pass 273 | try: 274 | # Ensure the playbook is available 275 | log.info("Cloning Ansible playbook %s to %s", playbook, repo_path) 276 | Repo.clone_from(playbook, to_path=repo_path) 277 | # Create a private ssh key file 278 | pkf = os.path.join(repo_path, 'pk') 279 | with os.fdopen(os.open(pkf, os.O_WRONLY | os.O_CREAT, 0o600), 280 | 'w') as f: 281 | f.writelines(pk) 282 | # Create an inventory file 283 | inv = Template(inventory) 284 | inventory_path = os.path.join(repo_path, 'inventory.ini') 285 | with open(inventory_path, 'w') as f: 286 | log.info("Creating inventory file %s", inventory_path) 287 | f.writelines(inv.substitute({'host': host, 'user': user})) 288 | # Write the ansible values file 289 | values_file_path = os.path.join(repo_path, 'values.yml') 290 | with open(values_file_path, 'w') as f: 291 | log.info("Creating ansible values file %s", values_file_path) 292 | yaml.dump(playbook_vars or {}, f, default_flow_style=False) 293 | # Run the playbook 294 | cmd = ["ansible-playbook", "-i", "inventory.ini", "playbook.yml"] 295 | if playbook_vars: 296 | cmd += ["-e", "@{}".format(values_file_path)] 297 | # TODO: Sanitize before printing 298 | log.debug("Running Ansible with values:\n%s", 299 | yaml.dump(playbook_vars or {}, default_flow_style=False)) 300 | output_buffer = self._run_ansible_process(cmd, repo_path) 301 | finally: 302 | if not settings.DEBUG: 303 | log.info("Deleting ansible playbook %s", repo_path) 304 | shutil.rmtree(repo_path) 305 | return 0, output_buffer 306 | 307 | @tenacity.retry(stop=tenacity.stop_after_attempt(3), 308 | wait=tenacity.wait_exponential(multiplier=1, min=4, max=256), 309 | reraise=True, 310 | after=lambda *args, **kwargs: log.debug("Error running ansible, rerunning playbook...")) 311 | def _run_ansible_process(self, cmd, repo_path): 312 | log.debug("Running Ansible with command: %s", " ".join(cmd)) 313 | with subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, 314 | universal_newlines=True, cwd=repo_path) as process: 315 | output_buffer = "" 316 | while process.poll() is None: 317 | output = process.stdout.readline() 318 | output_buffer += output 319 | if output: 320 | log.info(output) 321 | # Read any remaining output 322 | output_buffer += process.stdout.readline() 323 | if process.poll() != 0: 324 | raise Exception("An error occurred while running the ansible playbook to" 325 | " configure instance. Check the logs. Last output lines" 326 | " were: {0}".format(output_buffer.split("\n")[-10:])) 327 | log.info("Playbook status: %s", process.poll()) 328 | return output_buffer 329 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunch/forms.py: -------------------------------------------------------------------------------- 1 | from dal import autocomplete 2 | from django.forms import ModelForm 3 | 4 | from . import models 5 | from djcloudbridge import models as cb_models 6 | 7 | 8 | class ApplicationForm(ModelForm): 9 | 10 | def __init__(self, *args, **kwargs): 11 | super(ApplicationForm, self).__init__(*args, **kwargs) 12 | if self.instance: 13 | self.fields['default_version'].queryset = models.ApplicationVersion.objects.filter(application=self.instance) 14 | 15 | class Meta: 16 | model = models.Application 17 | fields = '__all__' 18 | 19 | 20 | class ApplicationVersionForm(ModelForm): 21 | 22 | def __init__(self, *args, **kwargs): 23 | super(ApplicationVersionForm, self).__init__(*args, **kwargs) 24 | if self.instance: 25 | self.fields['default_target'].queryset = models.DeploymentTarget.objects.filter(app_version_config__application_version=self.instance) 26 | 27 | class Meta: 28 | model = models.ApplicationVersion 29 | fields = '__all__' 30 | 31 | 32 | class ApplicationVersionCloudConfigForm(ModelForm): 33 | 34 | # def __init__(self, *args, **kwargs): 35 | # super(ApplicationVersionCloudConfigForm, self).__init__(*args, **kwargs) 36 | # try: 37 | # if self.instance and self.instance.target: 38 | # self.fields['image'].queryset = models.Image.objects.filter( 39 | # region=self.instance.target.target_zone.region) 40 | # except models.ApplicationVersionTargetConfig.target.RelatedObjectDoesNotExist: 41 | # pass 42 | 43 | class Meta: 44 | model = models.ApplicationVersionCloudConfig 45 | fields = '__all__' 46 | widgets = { 47 | 'image': autocomplete.ModelSelect2(url='image-autocomplete', 48 | forward=['target']) 49 | } 50 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunch/management/commands/export_app_data.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | from django.core.management.base import BaseCommand 3 | from cloudlaunch import models as cl_models 4 | 5 | import cloudlaunch.management.commands.serializers as mgmt_serializers 6 | 7 | 8 | class Command(BaseCommand): 9 | help = 'Exports Application Data in yaml format' 10 | 11 | def add_arguments(self, parser): 12 | parser.add_argument('-a', '--applications', nargs='+', type=str, 13 | help='Export only the specified applications') 14 | 15 | def handle(self, *args, **options): 16 | return self.export_app_data(applications=options.get('applications')) 17 | 18 | @staticmethod 19 | def export_app_data(applications=None): 20 | if applications: 21 | queryset = cl_models.Application.objects.filter(pk__in=applications) 22 | else: 23 | queryset = cl_models.Application.objects.all() 24 | 25 | serializer = mgmt_serializers.ApplicationSerializer(queryset, many=True) 26 | data = { 27 | 'apps': serializer.to_representation(serializer.instance) 28 | } 29 | return yaml.safe_dump(data, default_flow_style=False, allow_unicode=True) 30 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunch/management/commands/import_app_data.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import requests 3 | import yaml 4 | 5 | from django.core.management.base import BaseCommand 6 | 7 | import cloudlaunch.management.commands.serializers as mgmt_serializers 8 | 9 | 10 | class Command(BaseCommand): 11 | help = 'Imports Application Data from a given url or file and updates' \ 12 | ' existing models' 13 | 14 | def file_from_url(url): 15 | """ 16 | Reads a file from a url 17 | :return: content of file 18 | """ 19 | try: 20 | r = requests.get(url) 21 | r.raise_for_status() 22 | return r.text 23 | except requests.exceptions.RequestException as e: 24 | raise argparse.ArgumentTypeError(e) 25 | 26 | def add_arguments(self, parser): 27 | group = parser.add_mutually_exclusive_group(required=True) 28 | group.add_argument('-f', '--file', type=argparse.FileType('r')) 29 | group.add_argument('-u', '--url', type=Command.file_from_url) 30 | 31 | def handle(self, *args, **options): 32 | if options['file']: 33 | content = options['file'].read() 34 | else: 35 | content = options['url'] 36 | 37 | registry = yaml.safe_load(content) 38 | return self.import_app_data(registry.get('apps')) 39 | 40 | @staticmethod 41 | def import_app_data(yaml_data): 42 | serializer = mgmt_serializers.ApplicationSerializer( 43 | data=yaml_data, many=True) 44 | if serializer.is_valid(): 45 | serializer.save() 46 | else: 47 | return str(serializer.errors) 48 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunch/management/commands/serializers.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | import yaml 3 | 4 | from rest_framework import serializers 5 | 6 | from cloudlaunch import models as cl_models 7 | 8 | 9 | class StoredYAMLField(serializers.JSONField): 10 | def __init__(self, *args, **kwargs): 11 | super(StoredYAMLField, self).__init__(*args, **kwargs) 12 | 13 | def to_internal_value(self, data): 14 | try: 15 | if data: 16 | return yaml.safe_dump(data, default_flow_style=False, 17 | allow_unicode=True) 18 | else: 19 | return None 20 | except (TypeError, ValueError): 21 | self.fail('invalid') 22 | return data 23 | 24 | def to_representation(self, value): 25 | try: 26 | if value: 27 | return yaml.safe_load(value) 28 | else: 29 | return value 30 | except Exception: 31 | return value 32 | 33 | 34 | class AppVersionSerializer(serializers.ModelSerializer): 35 | default_launch_config = StoredYAMLField(required=False, allow_null=True) 36 | 37 | def get_unique_together_validators(self): 38 | """Overriding method to disable unique together checks""" 39 | return [] 40 | 41 | class Meta: 42 | model = cl_models.ApplicationVersion 43 | fields = ('version', 'frontend_component_path', 'frontend_component_name', 44 | 'backend_component_name', 'default_launch_config') 45 | 46 | 47 | class ApplicationSerializer(serializers.ModelSerializer): 48 | default_launch_config = StoredYAMLField(required=False, allow_null=True, 49 | validators=[]) 50 | default_version = serializers.CharField(source='default_version.version', default=None, 51 | allow_null=True, required=False) 52 | versions = AppVersionSerializer(many=True) 53 | 54 | def create(self, validated_data): 55 | return self.update(None, validated_data) 56 | 57 | def update(self, instance, validated_data): 58 | slug = validated_data.pop('slug') 59 | versions = validated_data.pop('versions') 60 | default_version = validated_data.pop('default_version') 61 | 62 | # create the app 63 | app, _ = cl_models.Application.objects.update_or_create( 64 | slug=slug, defaults=validated_data) 65 | 66 | # create all nested app versions 67 | for version in versions: 68 | ver_id = version.pop('version') 69 | cl_models.ApplicationVersion.objects.update_or_create( 70 | application=app, version=ver_id, defaults=version) 71 | 72 | # Now set the default app version 73 | if default_version and default_version['version']: 74 | app.default_version = cl_models.ApplicationVersion.objects.get( 75 | application=app, version=default_version['version']) 76 | app.save() 77 | 78 | return app 79 | 80 | class Meta: 81 | model = cl_models.Application 82 | fields = ('slug', 'name', 'status', 'summary', 'maintainer', 83 | 'description', 'info_url', 'icon_url', 'display_order', 84 | 'default_version', 'default_launch_config', 'versions') 85 | extra_kwargs = { 86 | 'slug': {'validators': []} 87 | } 88 | 89 | 90 | # Control YAML serialization 91 | 92 | # xref: https://github.com/wimglenn/oyaml/blob/de97b8b2be072a8072e807182ffe3fa11c504fd7/oyaml.py#L10 93 | def map_representer(dumper, data): 94 | return dumper.represent_dict(data.items()) 95 | 96 | 97 | def yaml_literal_representer(dumper, data): 98 | if len(data) > 50: 99 | return dumper.represent_scalar(u'tag:yaml.org,2002:str', data, style='|') 100 | else: 101 | return dumper.represent_scalar(u'tag:yaml.org,2002:str', data) 102 | 103 | 104 | yaml.add_representer(str, yaml_literal_representer) 105 | yaml.add_representer(str, yaml_literal_representer, Dumper=yaml.dumper.SafeDumper) 106 | yaml.add_representer(OrderedDict, map_representer) 107 | yaml.add_representer(OrderedDict, map_representer, Dumper=yaml.dumper.SafeDumper) 108 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunch/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.9 on 2020-01-24 20:33 2 | 3 | from django.conf import settings 4 | from django.db import migrations, models 5 | import django.db.models.deletion 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | initial = True 11 | 12 | dependencies = [ 13 | migrations.swappable_dependency(settings.AUTH_USER_MODEL), 14 | ('contenttypes', '0002_remove_content_type_name'), 15 | ('djcloudbridge', '0001_initial'), 16 | ] 17 | 18 | operations = [ 19 | migrations.CreateModel( 20 | name='AppCategory', 21 | fields=[ 22 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 23 | ('name', models.CharField(blank=True, choices=[('FEATURED', 'Featured'), ('GALAXY', 'Galaxy'), ('SCALABLE', 'Scalable'), ('VM', 'Virtual machine')], max_length=100, null=True, unique=True)), 24 | ], 25 | options={ 26 | 'verbose_name_plural': 'App categories', 27 | }, 28 | ), 29 | migrations.CreateModel( 30 | name='Application', 31 | fields=[ 32 | ('added', models.DateTimeField(auto_now_add=True)), 33 | ('updated', models.DateTimeField(auto_now=True)), 34 | ('name', models.CharField(max_length=60)), 35 | ('slug', models.SlugField(max_length=100, primary_key=True, serialize=False)), 36 | ('status', models.CharField(blank=True, choices=[('DEV', 'Development'), ('CERTIFICATION', 'Certification'), ('LIVE', 'Live')], default='DEV', max_length=50, null=True)), 37 | ('summary', models.CharField(blank=True, max_length=140, null=True)), 38 | ('maintainer', models.CharField(blank=True, max_length=255, null=True)), 39 | ('description', models.TextField(blank=True, max_length=32767, null=True)), 40 | ('info_url', models.URLField(blank=True, max_length=2048, null=True)), 41 | ('icon_url', models.URLField(blank=True, max_length=2048, null=True)), 42 | ('default_launch_config', models.TextField(blank=True, help_text='Application-wide initial configuration data to parameterize the launch with.', max_length=1048576, null=True)), 43 | ('display_order', models.IntegerField(default='10000')), 44 | ('category', models.ManyToManyField(blank=True, to='cloudlaunch.AppCategory')), 45 | ], 46 | options={ 47 | 'abstract': False, 48 | }, 49 | ), 50 | migrations.CreateModel( 51 | name='ApplicationDeployment', 52 | fields=[ 53 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 54 | ('added', models.DateTimeField(auto_now_add=True)), 55 | ('updated', models.DateTimeField(auto_now=True)), 56 | ('name', models.CharField(max_length=60)), 57 | ('archived', models.BooleanField(blank=True, default=False)), 58 | ('application_config', models.TextField(blank=True, help_text='Application configuration data used for this launch.', max_length=16384, null=True)), 59 | ], 60 | options={ 61 | 'abstract': False, 62 | }, 63 | ), 64 | migrations.CreateModel( 65 | name='ApplicationVersion', 66 | fields=[ 67 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 68 | ('version', models.CharField(max_length=30)), 69 | ('frontend_component_path', models.CharField(blank=True, max_length=255, null=True)), 70 | ('frontend_component_name', models.CharField(blank=True, max_length=255, null=True)), 71 | ('backend_component_name', models.CharField(blank=True, max_length=255, null=True)), 72 | ('default_launch_config', models.TextField(blank=True, help_text='Version specific configuration data to parameterize the launch with.', max_length=1048576, null=True)), 73 | ('application', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='versions', to='cloudlaunch.Application')), 74 | ], 75 | ), 76 | migrations.CreateModel( 77 | name='ApplicationVersionTargetConfig', 78 | fields=[ 79 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 80 | ('default_launch_config', models.TextField(blank=True, help_text='Target specific initial configuration data to parameterize the launch with.', max_length=16384, null=True)), 81 | ('application_version', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='app_version_config', to='cloudlaunch.ApplicationVersion')), 82 | ('polymorphic_ctype', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='polymorphic_cloudlaunch.applicationversiontargetconfig_set+', to='contenttypes.ContentType')), 83 | ], 84 | ), 85 | migrations.CreateModel( 86 | name='DeploymentTarget', 87 | fields=[ 88 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 89 | ('polymorphic_ctype', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='polymorphic_cloudlaunch.deploymenttarget_set+', to='contenttypes.ContentType')), 90 | ], 91 | options={ 92 | 'verbose_name': 'Deployment Target', 93 | 'verbose_name_plural': 'Deployment Targets', 94 | }, 95 | ), 96 | migrations.CreateModel( 97 | name='HostDeploymentTarget', 98 | fields=[ 99 | ('deploymenttarget_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='cloudlaunch.DeploymentTarget')), 100 | ], 101 | options={ 102 | 'verbose_name': 'Host', 103 | 'verbose_name_plural': 'Hosts', 104 | }, 105 | bases=('cloudlaunch.deploymenttarget',), 106 | ), 107 | migrations.CreateModel( 108 | name='KubernetesDeploymentTarget', 109 | fields=[ 110 | ('deploymenttarget_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='cloudlaunch.DeploymentTarget')), 111 | ('kube_config', models.CharField(max_length=16384)), 112 | ], 113 | options={ 114 | 'verbose_name': 'Kubernetes Cluster', 115 | 'verbose_name_plural': 'Kubernetes Clusters', 116 | }, 117 | bases=('cloudlaunch.deploymenttarget',), 118 | ), 119 | migrations.CreateModel( 120 | name='Usage', 121 | fields=[ 122 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 123 | ('added', models.DateTimeField(auto_now_add=True)), 124 | ('app_config', models.TextField(blank=True, max_length=16384, null=True)), 125 | ('app_deployment', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='cloudlaunch.ApplicationDeployment')), 126 | ('app_version_target_config', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='cloudlaunch.ApplicationVersionTargetConfig')), 127 | ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), 128 | ], 129 | options={ 130 | 'verbose_name_plural': 'Usage', 131 | 'ordering': ['added'], 132 | }, 133 | ), 134 | migrations.CreateModel( 135 | name='PublicKey', 136 | fields=[ 137 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 138 | ('added', models.DateTimeField(auto_now_add=True)), 139 | ('updated', models.DateTimeField(auto_now=True)), 140 | ('name', models.CharField(max_length=60)), 141 | ('public_key', models.TextField(max_length=16384)), 142 | ('default', models.BooleanField(blank=True, default=False, help_text='If set, use as the default public key')), 143 | ('fingerprint', models.CharField(blank=True, max_length=100, null=True)), 144 | ('user_profile', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='public_key', to='djcloudbridge.UserProfile')), 145 | ], 146 | options={ 147 | 'abstract': False, 148 | }, 149 | ), 150 | migrations.CreateModel( 151 | name='Image', 152 | fields=[ 153 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 154 | ('added', models.DateTimeField(auto_now_add=True)), 155 | ('updated', models.DateTimeField(auto_now=True)), 156 | ('name', models.CharField(max_length=60)), 157 | ('image_id', models.CharField(max_length=50, verbose_name='Image ID')), 158 | ('description', models.CharField(blank=True, max_length=255, null=True)), 159 | ('region', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='djcloudbridge.Region')), 160 | ], 161 | options={ 162 | 'abstract': False, 163 | }, 164 | ), 165 | migrations.AddField( 166 | model_name='applicationversiontargetconfig', 167 | name='target', 168 | field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='app_version_config', to='cloudlaunch.DeploymentTarget'), 169 | ), 170 | migrations.AddField( 171 | model_name='applicationversion', 172 | name='default_target', 173 | field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='cloudlaunch.DeploymentTarget'), 174 | ), 175 | migrations.CreateModel( 176 | name='ApplicationDeploymentTask', 177 | fields=[ 178 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 179 | ('added', models.DateTimeField(auto_now_add=True)), 180 | ('updated', models.DateTimeField(auto_now=True)), 181 | ('celery_id', models.TextField(blank=True, help_text='Celery task id for any background jobs running on this deployment', max_length=64, null=True, unique=True)), 182 | ('action', models.CharField(blank=True, choices=[('LAUNCH', 'Launch'), ('HEALTH_CHECK', 'Health check'), ('RESTART', 'Restart'), ('DELETE', 'Delete')], max_length=255, null=True)), 183 | ('_result', models.TextField(blank=True, db_column='result', help_text='Result of Celery task', max_length=16384, null=True)), 184 | ('_status', models.CharField(blank=True, db_column='status', max_length=64, null=True)), 185 | ('traceback', models.TextField(blank=True, help_text='Celery task traceback, if any', max_length=16384, null=True)), 186 | ('deployment', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='tasks', to='cloudlaunch.ApplicationDeployment')), 187 | ], 188 | ), 189 | migrations.AddField( 190 | model_name='applicationdeployment', 191 | name='application_version', 192 | field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='cloudlaunch.ApplicationVersion'), 193 | ), 194 | migrations.AddField( 195 | model_name='applicationdeployment', 196 | name='credentials', 197 | field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='target_creds', to='djcloudbridge.Credentials'), 198 | ), 199 | migrations.AddField( 200 | model_name='applicationdeployment', 201 | name='deployment_target', 202 | field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='cloudlaunch.DeploymentTarget'), 203 | ), 204 | migrations.AddField( 205 | model_name='applicationdeployment', 206 | name='owner', 207 | field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL), 208 | ), 209 | migrations.AddField( 210 | model_name='application', 211 | name='default_version', 212 | field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='cloudlaunch.ApplicationVersion'), 213 | ), 214 | migrations.CreateModel( 215 | name='AuthToken', 216 | fields=[ 217 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 218 | ('created', models.DateTimeField(auto_now_add=True, verbose_name='Created')), 219 | ('key', models.CharField(db_index=True, max_length=40, unique=True, verbose_name='Key')), 220 | ('name', models.CharField(max_length=64, verbose_name='Name')), 221 | ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='auth_tokens', to=settings.AUTH_USER_MODEL, verbose_name='User')), 222 | ], 223 | options={ 224 | 'unique_together': {('user', 'name')}, 225 | }, 226 | ), 227 | migrations.AlterUniqueTogether( 228 | name='applicationversiontargetconfig', 229 | unique_together={('application_version', 'target')}, 230 | ), 231 | migrations.CreateModel( 232 | name='ApplicationVersionCloudConfig', 233 | fields=[ 234 | ('applicationversiontargetconfig_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='cloudlaunch.ApplicationVersionTargetConfig')), 235 | ('image', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='cloudlaunch.Image')), 236 | ], 237 | options={ 238 | 'abstract': False, 239 | 'base_manager_name': 'objects', 240 | }, 241 | bases=('cloudlaunch.applicationversiontargetconfig',), 242 | ), 243 | migrations.AlterUniqueTogether( 244 | name='applicationversion', 245 | unique_together={('application', 'version')}, 246 | ), 247 | migrations.CreateModel( 248 | name='CloudDeploymentTarget', 249 | fields=[ 250 | ('deploymenttarget_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='cloudlaunch.DeploymentTarget')), 251 | ('target_zone', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='djcloudbridge.Zone')), 252 | ], 253 | options={ 254 | 'verbose_name': 'Cloud', 255 | 'verbose_name_plural': 'Clouds', 256 | 'unique_together': {('deploymenttarget_ptr', 'target_zone')}, 257 | }, 258 | bases=('cloudlaunch.deploymenttarget',), 259 | ), 260 | ] 261 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunch/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/galaxyproject/cloudlaunch/e96cb45d1c2b19be9b14ac3202df0c708f017425/django-cloudlaunch/cloudlaunch/migrations/__init__.py -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunch/signals.py: -------------------------------------------------------------------------------- 1 | """App-wide Django signals.""" 2 | from celery.utils.log import get_task_logger 3 | 4 | from django.db.models.signals import post_save 5 | from django.dispatch import receiver 6 | from django.dispatch import Signal 7 | 8 | from djcloudbridge import models as cb_models 9 | 10 | from . import models 11 | 12 | log = get_task_logger(__name__) 13 | 14 | health_check = Signal() 15 | 16 | 17 | @receiver(health_check) 18 | def delete_old_tasks(sender, deployment, **kwargs): 19 | """ 20 | Delete HEALTH_CHECK task results other than two most recent ones. 21 | 22 | We keep only the two most recent deployment task results while all others 23 | are deleted when this signal is invoked. This includes only the tasks with 24 | ``SUCCESS`` status and correspond to the supplied ``deployment``. 25 | """ 26 | for old_task in models.ApplicationDeploymentTask.objects.filter( 27 | deployment=deployment, 28 | _status="SUCCESS", 29 | action=models.ApplicationDeploymentTask.HEALTH_CHECK).order_by( 30 | '-updated')[2:]: 31 | log.debug('Deleting old health task %s from deployment %s', 32 | old_task.id, deployment.name) 33 | old_task.delete() 34 | 35 | 36 | @receiver(post_save, sender=cb_models.Zone) 37 | def create_cloud_deployment_target(sender, instance, created, **kwargs): 38 | """ 39 | Automatically create a corresponding deployment target for each zone 40 | that's created 41 | """ 42 | if created: 43 | models.CloudDeploymentTarget.objects.create(target_zone=instance) 44 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunch/tasks.py: -------------------------------------------------------------------------------- 1 | """Tasks to be executed asynchronously (via Celery).""" 2 | import copy 3 | import json 4 | import logging 5 | import yaml 6 | 7 | from celery.app import shared_task 8 | from celery.exceptions import SoftTimeLimitExceeded 9 | from celery.result import AsyncResult 10 | from celery.utils.log import get_task_logger 11 | 12 | from djcloudbridge import domain_model 13 | from . import models 14 | from . import signals 15 | from . import util 16 | from . import serializers 17 | 18 | log = get_task_logger('cloudlaunch') 19 | # Limit how much these libraries log 20 | logging.getLogger('boto3').setLevel(logging.WARNING) 21 | logging.getLogger('botocore').setLevel(logging.WARNING) 22 | logging.getLogger('cloudbridge').setLevel(logging.INFO) 23 | 24 | 25 | @shared_task(time_limit=120) 26 | def migrate_launch_task(task_id): 27 | """ 28 | Migrate task result to a persistent model table. 29 | 30 | Task result may contain temporary info that we don't want to keep. This 31 | task is intended to be called some time after the initial task has run to 32 | migrate the info we do want to keep to a model table. 33 | """ 34 | adt = models.ApplicationDeploymentTask.objects.get(celery_id=task_id) 35 | task = AsyncResult(task_id) 36 | task_meta = task.backend.get_task_meta(task.id) 37 | adt.status = task_meta.get('status') 38 | adt.traceback = task_meta.get('traceback') 39 | adt.celery_id = None 40 | sanitized_result = copy.deepcopy(task_meta['result']) 41 | if sanitized_result.get('cloudLaunch', {}).get('keyPair', {}).get( 42 | 'material'): 43 | sanitized_result['cloudLaunch']['keyPair']['material'] = None 44 | adt.result = json.dumps(sanitized_result) 45 | adt.save() 46 | task.forget() 47 | 48 | 49 | @shared_task(time_limit=120) 50 | def update_status_task(task_id): 51 | """ 52 | Update task result to a persistent model table. 53 | 54 | This task is intended to be called as soon as the launch task is over so 55 | we have a fairly up-to-date _status field in the model. 56 | """ 57 | adt = models.ApplicationDeploymentTask.objects.get(celery_id=task_id) 58 | task = AsyncResult(task_id) 59 | task_meta = task.backend.get_task_meta(task.id) 60 | adt.status = task_meta.get('status') 61 | adt.save() 62 | 63 | 64 | @shared_task(expires=120) 65 | def create_appliance(name, cloud_version_config_id, credentials, app_config, 66 | user_data): 67 | """Call the appropriate app plugin and initiate the app launch process.""" 68 | try: 69 | log.debug("Creating appliance %s", name) 70 | cloud_version_conf = models.ApplicationVersionCloudConfig.objects.get( 71 | pk=cloud_version_config_id) 72 | zone = cloud_version_conf.target.target_zone 73 | plugin = util.import_class( 74 | cloud_version_conf.application_version.backend_component_name)() 75 | # FIXME: Should not be instantiating provider here 76 | provider = domain_model.get_cloud_provider(zone, credentials) 77 | # Dump and reload to convert to standard dict 78 | cloud_config = json.loads(json.dumps(serializers.CloudConfigPluginSerializer( 79 | cloud_version_conf).data)) 80 | cloud_config['credentials'] = credentials 81 | # TODO: Add keys (& support) for using existing, user-supplied hosts 82 | provider_config = {'cloud_provider': provider, 83 | 'cloud_config': cloud_config, 84 | 'cloud_user_data': user_data} 85 | # TODO: Sanitize even in debug mode 86 | log.debug("Provider_config: %s", provider_config) 87 | log.info("Creating app %s with the following app config: %s", 88 | name, plugin.sanitise_app_config(app_config)) 89 | deploy_result = plugin.deploy(name, Task(create_appliance), app_config, 90 | provider_config) 91 | # Upgrade task result immediately 92 | update_status_task.apply_async([create_appliance.request.id], 93 | countdown=1) 94 | # Schedule a task to migrate result one hour from now 95 | migrate_launch_task.apply_async([create_appliance.request.id], 96 | countdown=3600) 97 | return deploy_result 98 | except SoftTimeLimitExceeded: 99 | msg = "Create appliance task time limit exceeded; stopping the task." 100 | log.warning(msg) 101 | raise Exception(msg) 102 | except Exception as exc: 103 | msg = "Create appliance task failed: %s" % str(exc) 104 | log.error(msg) 105 | raise Exception(msg) from exc 106 | 107 | 108 | def _get_app_plugin(deployment): 109 | """ 110 | Retrieve appliance plugin for a deployment. 111 | 112 | :rtype: :class:`.AppPlugin` 113 | :return: An instance of the plugin class corresponding to the 114 | deployment app. 115 | """ 116 | target = deployment.deployment_target 117 | target_version_config = models.ApplicationVersionTargetConfig.objects.get( 118 | application_version=deployment.application_version.id, target=target.pk) 119 | return util.import_class( 120 | target_version_config.application_version.backend_component_name)() 121 | 122 | 123 | @shared_task(time_limit=120) 124 | def migrate_task_result(task_id): 125 | """Migrate task results to the database from the broker table.""" 126 | log.debug("Migrating task %s result to the DB" % task_id) 127 | adt = models.ApplicationDeploymentTask.objects.get(celery_id=task_id) 128 | task = AsyncResult(task_id) 129 | task_meta = task.backend.get_task_meta(task.id) 130 | adt.celery_id = None 131 | adt.status = task_meta.get('status') 132 | adt.result = json.dumps(task_meta.get('result')) 133 | adt.traceback = task_meta.get('traceback') 134 | adt.save() 135 | task.forget() 136 | 137 | 138 | def _serialize_deployment(deployment): 139 | """ 140 | Extract appliance info for the supplied deployment and serialize it. 141 | 142 | @type deployment: ``ApplicationDeployment`` 143 | @param deployment: An instance of the app deployment. 144 | 145 | :rtype: ``str`` 146 | :return: Serialized info about the appliance deployment, which corresponds 147 | to the result of the LAUNCH task. 148 | """ 149 | launch_task = deployment.tasks.filter( 150 | action=models.ApplicationDeploymentTask.LAUNCH).first() 151 | result = {'name': deployment.name, 152 | 'app_config': yaml.safe_load(deployment.application_config), 153 | 'launch_status': None, 154 | 'launch_result': {} 155 | } 156 | if launch_task: 157 | result['launch_status'] = launch_task.status 158 | result['launch_result'] = launch_task.result 159 | return result 160 | 161 | 162 | @shared_task(bind=True, time_limit=60, expires=300) 163 | def health_check(self, deployment_id, credentials): 164 | """ 165 | Check the health of the supplied deployment. 166 | 167 | Conceptually, the health check can be as elaborate as the deployed 168 | appliance supports via a custom implementation. At the minimum, and 169 | by default, the health reflects the status of the cloud instance by 170 | querying the cloud provider. 171 | """ 172 | try: 173 | deployment = models.ApplicationDeployment.objects.get(pk=deployment_id) 174 | log.debug("Checking health of deployment %s", deployment.name) 175 | plugin = _get_app_plugin(deployment) 176 | dpl = _serialize_deployment(deployment) 177 | # FIXME: Should not be instantiating provider here 178 | target_zone = deployment.deployment_target.target_zone 179 | provider = domain_model.get_cloud_provider(target_zone, credentials) 180 | result = plugin.health_check(provider, dpl) 181 | except Exception as e: 182 | msg = "Health check failed: %s" % str(e) 183 | log.error(msg) 184 | raise Exception(msg) from e 185 | finally: 186 | # We only keep the two most recent health check task results so delete 187 | # any older ones 188 | signals.health_check.send(sender=None, deployment=deployment) 189 | # Schedule a task to migrate results right after task completion 190 | # Do this as a separate task because until this task completes, we 191 | # cannot obtain final status or traceback. 192 | migrate_task_result.apply_async([self.request.id], countdown=1) 193 | return result 194 | 195 | 196 | @shared_task(bind=True, time_limit=300, expires=120) 197 | def restart_appliance(self, deployment_id, credentials): 198 | """ 199 | Restarts this appliances 200 | """ 201 | try: 202 | deployment = models.ApplicationDeployment.objects.get(pk=deployment_id) 203 | log.debug("Performing restart on deployment %s", deployment.name) 204 | plugin = _get_app_plugin(deployment) 205 | dpl = _serialize_deployment(deployment) 206 | # FIXME: Should not be instantiating provider here 207 | target_zone = deployment.deployment_target.target_zone 208 | provider = domain_model.get_cloud_provider(target_zone, credentials) 209 | result = plugin.restart(provider, dpl) 210 | except Exception as e: 211 | msg = "Restart task failed: %s" % str(e) 212 | log.error(msg) 213 | raise Exception(msg) from e 214 | # Schedule a task to migrate results right after task completion 215 | # Do this as a separate task because until this task completes, we 216 | # cannot obtain final status or traceback. 217 | migrate_task_result.apply_async([self.request.id], countdown=1) 218 | return result 219 | 220 | 221 | @shared_task(bind=True, expires=120) 222 | def delete_appliance(self, deployment_id, credentials): 223 | """ 224 | Deletes this appliances 225 | If successful, will also mark the supplied ``deployment`` as 226 | ``archived`` in the database. 227 | """ 228 | try: 229 | deployment = models.ApplicationDeployment.objects.get(pk=deployment_id) 230 | log.debug("Performing delete on deployment %s", deployment.name) 231 | plugin = _get_app_plugin(deployment) 232 | dpl = _serialize_deployment(deployment) 233 | # FIXME: Should not be instantiating provider here 234 | target_zone = deployment.deployment_target.target_zone 235 | provider = domain_model.get_cloud_provider(target_zone, credentials) 236 | result = plugin.delete(provider, dpl) 237 | if result is True: 238 | deployment.archived = True 239 | deployment.save() 240 | except Exception as e: 241 | msg = "Delete task failed: %s" % str(e) 242 | log.error(msg) 243 | raise Exception(msg) from e 244 | # Schedule a task to migrate results right after task completion 245 | # Do this as a separate task because until this task completes, we 246 | # cannot obtain final status or traceback. 247 | migrate_task_result.apply_async([self.request.id], countdown=1) 248 | return result 249 | 250 | 251 | class Task(object): 252 | """ 253 | An abstraction class for handling task actions. 254 | 255 | Plugins can implement the interface defined here and handle task actions 256 | independent of CloudLaunch and its task broker. 257 | """ 258 | 259 | def __init__(self, broker_task): 260 | self.task = broker_task 261 | 262 | def update_state(self, task_id=None, state=None, meta=None): 263 | """ 264 | Update task state. 265 | 266 | @type task_id: ``str`` 267 | @param task_id: Id of the task to update. Defaults to the id of the 268 | current task. 269 | 270 | @type state: ``str 271 | @param state: New state. 272 | 273 | @type meta: ``dict`` 274 | @param meta: State meta-data. 275 | """ 276 | self.task.update_state(state=state, meta=meta) 277 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunch/templates/admin/import_data.html: -------------------------------------------------------------------------------- 1 | {% extends "admin/base_site.html" %} 2 | 3 | {% block content %} 4 |
5 | {% csrf_token %} 6 |

7 | Specify app registry url to import from: 8 |

9 | 10 | 11 | {% for row in rows %} 12 | 13 | {% endfor %} 14 | 15 | 16 | 17 |
18 | {% endblock %} 19 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunch/templates/rest_framework/api.html: -------------------------------------------------------------------------------- 1 | {% extends "rest_framework/base.html" %} 2 | 3 | {% block branding %} 4 | 5 | CloudLaunch API Browser v0.1 powered by DRF {{ version }} 6 | 7 | {% endblock %} -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunch/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/galaxyproject/cloudlaunch/e96cb45d1c2b19be9b14ac3202df0c708f017425/django-cloudlaunch/cloudlaunch/tests/__init__.py -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunch/tests/test_launch.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import patch 2 | import yaml 3 | 4 | from django.contrib.auth.models import User 5 | from django.urls import reverse 6 | from djcloudbridge import models as cb_models 7 | from rest_framework.test import APILiveServerTestCase 8 | 9 | from cloudlaunch.models import ( 10 | Application, 11 | ApplicationDeployment, 12 | ApplicationVersion, 13 | ApplicationVersionCloudConfig, 14 | ApplicationDeploymentTask, 15 | CloudDeploymentTarget, 16 | Image) 17 | 18 | 19 | class CLLaunchTestBase(APILiveServerTestCase): 20 | 21 | def setUp(self): 22 | self.user = User.objects.create(username='test-user') 23 | self.client.force_authenticate(user=self.user) 24 | 25 | def create_mock_provider(self, name, config): 26 | provider_class = self.get_provider_class("mock") 27 | return provider_class(config) 28 | 29 | patcher2 = patch('cloudbridge.factory.CloudProviderFactory.create_provider', 30 | new=create_mock_provider) 31 | patcher2.start() 32 | self.addCleanup(patcher2.stop) 33 | 34 | patcher3 = patch('cloudlaunch.configurers.SSHBasedConfigurer._check_ssh') 35 | patcher3.start() 36 | self.addCleanup(patcher3.stop) 37 | 38 | patcher4 = patch('cloudlaunch.configurers.AnsibleAppConfigurer.configure') 39 | patcher4.start() 40 | self.addCleanup(patcher4.stop) 41 | 42 | # Patch some background celery tasks to reduce noise in the logs. 43 | # They don't really affect the tests 44 | patcher_update_task = patch('cloudlaunch.tasks.update_status_task') 45 | patcher_update_task.start() 46 | self.addCleanup(patcher_update_task.stop) 47 | patcher_migrate_task = patch('cloudlaunch.tasks.migrate_launch_task') 48 | patcher_migrate_task.start() 49 | self.addCleanup(patcher_migrate_task.stop) 50 | patcher_migrate_result = patch('cloudlaunch.tasks.migrate_task_result') 51 | patcher_migrate_result.start() 52 | self.addCleanup(patcher_migrate_result.stop) 53 | 54 | super().setUp() 55 | 56 | def assertResponse(self, response, status=None, data_contains=None): 57 | if status: 58 | self.assertEqual(response.status_code, status) 59 | if data_contains: 60 | self.assertDictContains(response.data, data_contains) 61 | 62 | def assertDictContains(self, dict1, dict2): 63 | for key in dict2: 64 | self.assertTrue(key in dict1) 65 | if isinstance(dict2[key], dict): 66 | self.assertDictContains(dict1[key], dict2[key]) 67 | else: 68 | self.assertEqual(dict1[key], dict2[key]) 69 | 70 | 71 | class ApplicationLaunchTests(CLLaunchTestBase): 72 | 73 | DEFAULT_LAUNCH_CONFIG = { 74 | 'foo': 1, 75 | 'bar': 2, 76 | } 77 | DEFAULT_APP_CONFIG = { 78 | 'bar': 3, 79 | 'baz': 4, 80 | 'config_cloudlaunch': { 81 | 'instance_user_data': "userdata" 82 | } 83 | } 84 | 85 | def _create_application_version(self): 86 | application = Application.objects.create( 87 | name="Ubuntu", 88 | status=Application.LIVE, 89 | ) 90 | application_version = ApplicationVersion.objects.create( 91 | application=application, 92 | version="16.04", 93 | frontend_component_path="app/marketplace/plugins/plugins.module" 94 | "#PluginsModule", 95 | frontend_component_name="ubuntu-config", 96 | backend_component_name="cloudlaunch.backend_plugins.base_vm_app" 97 | ".BaseVMAppPlugin", 98 | ) 99 | return application_version 100 | 101 | def _create_cloud_region_zone(self): 102 | cloud = cb_models.AWSCloud.objects.create( 103 | name='AWS' 104 | ) 105 | region = cb_models.AWSRegion.objects.create( 106 | cloud=cloud, 107 | name='us-east-1' 108 | ) 109 | zone = cb_models.Zone.objects.create( 110 | region=region, 111 | name='us-east-1a' 112 | ) 113 | return zone 114 | 115 | def _create_credentials(self, target_cloud): 116 | user_profile = cb_models.UserProfile.objects.get(user=self.user) 117 | return cb_models.AWSCredentials.objects.create( 118 | cloud=target_cloud, 119 | aws_access_key='access_key', 120 | aws_secret_key='secret_key', 121 | user_profile=user_profile, 122 | default=True, 123 | ) 124 | 125 | def _create_image(self, target_region): 126 | return Image.objects.create( 127 | image_id='abc123', 128 | region=target_region, 129 | ) 130 | 131 | def _create_app_version_cloud_config(self, 132 | application_version, 133 | target, 134 | image, 135 | launch_config=DEFAULT_LAUNCH_CONFIG): 136 | return ApplicationVersionCloudConfig.objects.create( 137 | application_version=application_version, 138 | target=target, 139 | image=image, 140 | default_launch_config=yaml.safe_dump(launch_config), 141 | ) 142 | 143 | def setUp(self): 144 | super().setUp() 145 | # Create test data 146 | self.application_version = self._create_application_version() 147 | self.target_zone = self._create_cloud_region_zone() 148 | self.target_region = self.target_zone.region 149 | self.target_cloud = self.target_region.cloud 150 | self.ubuntu_image = self._create_image(self.target_region) 151 | self.credentials = self._create_credentials(self.target_cloud) 152 | self.credentials = self._create_credentials(self.target_cloud) 153 | self.deployment_target = CloudDeploymentTarget.objects.get( 154 | target_zone=self.target_zone) 155 | self.app_version_cloud_config = self._create_app_version_cloud_config( 156 | self.application_version, self.deployment_target, self.ubuntu_image) 157 | 158 | def _create_deployment(self): 159 | """Create deployment from 'application' and 'application_version'.""" 160 | return self.client.post(reverse('deployments-list'), { 161 | 'name': 'test-deployment', 162 | 'application': self.application_version.application.slug, 163 | 'application_version': self.application_version.version, 164 | 'deployment_target_id': self.deployment_target.id, 165 | }) 166 | response = self._create_deployment() 167 | 168 | def test_create_deployment(self): 169 | with patch('cloudbridge.providers.aws.services.AWSInstanceService.delete') as mock_del: 170 | response = self._create_deployment() 171 | self.assertResponse(response, status=201, data_contains={ 172 | 'name': 'test-deployment', 173 | 'application_version': self.application_version.id, 174 | 'deployment_target': { 175 | 'id': self.deployment_target.id, 176 | 'target_zone': { 177 | 'zone_id': self.target_zone.name 178 | } 179 | }, 180 | 'application_config': self.DEFAULT_LAUNCH_CONFIG, 181 | 'app_version_details': { 182 | 'version': self.application_version.version, 183 | 'application': { 184 | 'slug': self.application_version.application.slug, 185 | } 186 | }, 187 | }) 188 | # Check that deployment and its LAUNCH task were created 189 | app_deployment = ApplicationDeployment.objects.get() 190 | launch_task = ApplicationDeploymentTask.objects.get( 191 | action=ApplicationDeploymentTask.LAUNCH, 192 | deployment=app_deployment) 193 | self.assertIsNotNone(launch_task) 194 | mock_del.get.assert_not_called() 195 | 196 | def test_launch_error_triggers_cleanup(self): 197 | """ 198 | Checks whether an error during launch triggers a cleanup of the instance. 199 | """ 200 | counter_ref = [0] 201 | 202 | def succeed_on_second_try(count_ref, *args, **kwargs): 203 | count_ref[0] += 1 204 | if count_ref[0] > 0 and count_ref[0] < 3: 205 | raise Exception("Some exception occurred while waiting") 206 | 207 | with patch('cloudbridge.base.resources.BaseInstance.wait_for', 208 | side_effect=lambda *args, **kwargs: succeed_on_second_try( 209 | counter_ref, *args, **kwargs)) as mock_wait: 210 | self._create_deployment() 211 | app_deployment = ApplicationDeployment.objects.get() 212 | launch_task = ApplicationDeploymentTask.objects.get( 213 | action=ApplicationDeploymentTask.LAUNCH, 214 | deployment=app_deployment) 215 | self.assertNotEquals(launch_task.status, "SUCCESS") 216 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunch/tests/test_mgmt_commands.py: -------------------------------------------------------------------------------- 1 | from io import StringIO 2 | import os 3 | from django.conf import settings 4 | from django.core.management import call_command 5 | from django.core.management.base import CommandError 6 | from django.test import TestCase 7 | 8 | from cloudlaunch import models as cl_models 9 | 10 | 11 | TEST_DATA_PATH = os.path.join(os.path.dirname(__file__), 'data') 12 | 13 | 14 | class ImportAppCommandTestCase(TestCase): 15 | 16 | APP_DATA_PATH_NEW = os.path.join( 17 | TEST_DATA_PATH, 'apps_new.yaml') 18 | APP_DATA_PATH_UPDATED = os.path.join( 19 | TEST_DATA_PATH, 'apps_update.yaml') 20 | APP_DATA_PATH_URL = settings.CLOUDLAUNCH_APP_REGISTRY_URL 21 | 22 | def test_import_app_data_no_args(self): 23 | with self.assertRaisesRegex(CommandError, 24 | "-f/--file -u/--url is required"): 25 | call_command('import_app_data') 26 | 27 | def test_import_new_app_data_from_file(self): 28 | call_command('import_app_data', '-f', self.APP_DATA_PATH_NEW) 29 | app_obj = cl_models.Application.objects.get( 30 | slug='biodocklet') 31 | self.assertEquals(app_obj.name, 'BioDocklet') 32 | self.assertIn('bcil/biodocklets:RNAseq_paired', 33 | app_obj.default_version.default_launch_config) 34 | 35 | def test_import_existing_app_data_from_file(self): 36 | call_command('import_app_data', '-f', self.APP_DATA_PATH_NEW) 37 | call_command('import_app_data', '-f', self.APP_DATA_PATH_UPDATED) 38 | app_obj = cl_models.Application.objects.get( 39 | slug='cloudman-20') 40 | self.assertEquals(app_obj.name, 'CloudMan 2.0 Updated') 41 | self.assertEquals(app_obj.summary, 'A different summary') 42 | self.assertIn('some_new_text', 43 | app_obj.default_version.default_launch_config) 44 | 45 | def test_import_new_app_data_from_url(self): 46 | call_command('import_app_data', '--url', self.APP_DATA_PATH_URL) 47 | app_obj = cl_models.Application.objects.get( 48 | slug='pulsar-standalone') 49 | self.assertEquals(app_obj.name, 'Galaxy Cloud Bursting') 50 | self.assertIn('config_cloudlaunch', 51 | app_obj.default_version.default_launch_config) 52 | 53 | def test_export_matches_import(self): 54 | call_command('import_app_data', '-f', self.APP_DATA_PATH_NEW) 55 | out = StringIO() 56 | call_command('export_app_data', stdout=out) 57 | with open(self.APP_DATA_PATH_NEW) as f: 58 | self.assertEquals(f.read(), out.getvalue()) 59 | 60 | def test_export_subset(self): 61 | call_command('import_app_data', '-f', self.APP_DATA_PATH_NEW) 62 | out = StringIO() 63 | call_command('export_app_data', '-a', 'biodocklet', stdout=out) 64 | self.assertNotIn('cloudman-20', out.getvalue()) 65 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunch/urls.py: -------------------------------------------------------------------------------- 1 | """cloudlaunch URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/1.9/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') 9 | Class-based views 10 | 1. Add an import: from other_app.views import Home 11 | 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Add an import: from blog import urls as blog_urls 14 | 2. Import the include() function: from django.urls import re_path, include 15 | 3. Add a URL to urlpatterns: url(r'^blog/', include(blog_urls)) 16 | """ 17 | from django.conf import settings 18 | from django.urls import include 19 | from django.urls import re_path 20 | from rest_framework.schemas import get_schema_view 21 | 22 | from . import views 23 | 24 | from public_appliances import urls as pub_urls 25 | 26 | from djcloudbridge.drf_routers import HybridDefaultRouter, HybridNestedRouter, HybridSimpleRouter 27 | from djcloudbridge.urls import cl_zone_router 28 | 29 | 30 | # from django.contrib import admin 31 | router = HybridDefaultRouter() 32 | router.register(r'infrastructure', views.InfrastructureView, 33 | basename='infrastructure') 34 | router.register(r'applications', views.ApplicationViewSet) 35 | # router.register(r'images', views.ImageViewSet) 36 | router.register(r'deployments', views.DeploymentViewSet, basename='deployments') 37 | 38 | router.register(r'auth', views.AuthView, basename='auth') 39 | router.register(r'auth/tokens', views.AuthTokenViewSet, 40 | basename='auth_token') 41 | 42 | router.register(r'cors_proxy', views.CorsProxyView, basename='corsproxy') 43 | deployments_router = HybridNestedRouter(router, r'deployments', 44 | lookup='deployment') 45 | deployments_router.register(r'tasks', views.DeploymentTaskViewSet, 46 | basename='deployment_task') 47 | 48 | # Extend djcloudbridge endpoints 49 | cl_zone_router.register(r'cloudman', views.CloudManViewSet, basename='cloudman') 50 | 51 | infrastructure_regex_pattern = r'^api/v1/infrastructure/' 52 | auth_regex_pattern = r'^api/v1/auth/' 53 | public_services_regex_pattern = r'^api/v1/public_services/' 54 | 55 | schema_view = get_schema_view(title='CloudLaunch API', url=settings.REST_SCHEMA_BASE_URL, 56 | urlconf='cloudlaunch.urls') 57 | 58 | registration_urls = [ 59 | re_path(r'^$', views.CustomRegisterView.as_view(), name='rest_register'), 60 | re_path(r'', include(('dj_rest_auth.registration.urls', 'rest_auth_reg'), 61 | namespace='rest_auth_reg')) 62 | ] 63 | 64 | urlpatterns = [ 65 | re_path(r'^api/v1/', include(router.urls)), 66 | re_path(r'^api/v1/', include(deployments_router.urls)), 67 | # This generates a duplicate url set with the cloudman url included 68 | # get_urls() must be called or a cached set of urls will be returned. 69 | re_path(infrastructure_regex_pattern, include(cl_zone_router.get_urls())), 70 | re_path(infrastructure_regex_pattern, include('djcloudbridge.urls')), 71 | re_path(auth_regex_pattern, include(('dj_rest_auth.urls', 'rest_auth'), namespace='rest_auth')), 72 | 73 | # Override default register view 74 | re_path(r'%sregistration' % auth_regex_pattern, include((registration_urls, 'rest_auth_reg'), namespace='rest_auth_reg')), 75 | re_path(r'%suser/public-keys/$' % 76 | auth_regex_pattern, views.PublicKeyList.as_view()), 77 | re_path(r'%suser/public-keys/(?P[0-9]+)/$' % 78 | auth_regex_pattern, views.PublicKeyDetail.as_view(), 79 | name='public-key-detail'), 80 | re_path(auth_regex_pattern, include(('rest_framework.urls', 'rest_framework'), 81 | namespace='rest_framework')), 82 | re_path(auth_regex_pattern, include('djcloudbridge.profile.urls')), 83 | # The following is required because rest_auth calls allauth internally and 84 | # reverse urls need to be resolved. 85 | re_path(r'^accounts/', include('allauth.urls')), 86 | # Public services 87 | re_path(public_services_regex_pattern, include('public_appliances.urls')), 88 | re_path(r'^api/v1/schema/$', schema_view), 89 | re_path(r'^image-autocomplete/$', views.ImageAutocomplete.as_view(), 90 | name='image-autocomplete', 91 | ) 92 | ] 93 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunch/util.py: -------------------------------------------------------------------------------- 1 | """A set of utility functions used by the framework.""" 2 | from importlib import import_module 3 | 4 | 5 | def import_class(name): 6 | parts = name.rsplit('.', 1) 7 | cls = getattr(import_module(parts[0]), parts[1]) 8 | return cls 9 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunch/view_helpers.py: -------------------------------------------------------------------------------- 1 | from djcloudbridge import view_helpers as cb_view_helpers 2 | 3 | 4 | def get_cloud_provider(view, cloud_id=None): 5 | """ 6 | Returns a cloud provider for the current user. The relevant 7 | cloud is discovered from the view and the credentials are retrieved 8 | from the request or user profile. Return ``None`` if no credentials were 9 | retrieved. 10 | """ 11 | return cb_view_helpers.get_cloud_provider(view, cloud_id) 12 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunch/views.py: -------------------------------------------------------------------------------- 1 | from django.http import HttpResponse 2 | from django_filters import rest_framework as dj_filters 3 | from dj_rest_auth.registration.views import RegisterView 4 | from rest_framework import authentication 5 | from rest_framework import filters 6 | from rest_framework import generics 7 | from rest_framework import viewsets 8 | from rest_framework.pagination import PageNumberPagination 9 | from rest_framework.permissions import IsAuthenticated 10 | from rest_framework.response import Response 11 | from rest_framework.reverse import reverse 12 | from rest_framework.views import APIView 13 | import requests 14 | 15 | from dal import autocomplete 16 | 17 | from djcloudbridge import drf_helpers 18 | from . import models 19 | from . import serializers 20 | 21 | 22 | class CustomApplicationPagination(PageNumberPagination): 23 | page_size_query_param = 'page_size' 24 | 25 | class ApplicationViewSet(viewsets.ModelViewSet): 26 | """ 27 | API endpoint that allows applications to be viewed or edited. 28 | """ 29 | queryset = models.Application.objects.filter(status=models.Application.LIVE) 30 | serializer_class = serializers.ApplicationSerializer 31 | filter_backends = (filters.OrderingFilter, filters.SearchFilter) 32 | search_fields = ('slug',) 33 | ordering = ('display_order',) 34 | pagination_class = CustomApplicationPagination 35 | 36 | 37 | class InfrastructureView(APIView): 38 | """ 39 | List kinds in infrastructures. 40 | """ 41 | 42 | def get(self, request, format=None): 43 | # We only support cloud infrastructures for the time being 44 | response = {'url': request.build_absolute_uri('clouds')} 45 | return Response(response) 46 | 47 | 48 | class AuthView(APIView): 49 | """ 50 | List authentication endpoints. 51 | """ 52 | 53 | def get(self, request, format=None): 54 | data = { 55 | 'login': request.build_absolute_uri( 56 | reverse('rest_auth:rest_login')), 57 | 'logout': request.build_absolute_uri( 58 | reverse('rest_auth:rest_logout')), 59 | 'user': request.build_absolute_uri( 60 | reverse('rest_auth:rest_user_details')), 61 | 'registration': request.build_absolute_uri( 62 | reverse('rest_auth_reg:rest_register')), 63 | 'tokens': request.build_absolute_uri( 64 | reverse('auth_token-list')), 65 | 'password/reset': request.build_absolute_uri( 66 | reverse('rest_auth:rest_password_reset')), 67 | 'password/reset/confirm': request.build_absolute_uri( 68 | reverse('rest_auth:rest_password_reset_confirm')), 69 | 'password/reset/change': request.build_absolute_uri( 70 | reverse('rest_auth:rest_password_change')), 71 | } 72 | return Response(data) 73 | 74 | 75 | class CorsProxyView(APIView): 76 | """ 77 | API endpoint that allows applications to be viewed or edited. 78 | """ 79 | exclude_from_schema = True 80 | 81 | def get(self, request, format=None): 82 | url = self.request.query_params.get('url') 83 | response = requests.get(url) 84 | return HttpResponse(response.text, status=response.status_code, 85 | content_type=response.headers.get('content-type')) 86 | 87 | 88 | class AuthTokenViewSet(viewsets.ModelViewSet): 89 | """ 90 | Return an auth token for a user that is already logged in. 91 | """ 92 | permission_classes = (IsAuthenticated,) 93 | authentication_classes = (authentication.SessionAuthentication,) 94 | serializer_class = serializers.AuthTokenSerializer 95 | filter_backends = (dj_filters.DjangoFilterBackend,) 96 | filter_fields = ('name',) 97 | 98 | def get_queryset(self): 99 | """ 100 | This view should return a list of all the tokens 101 | for the currently authenticated user. 102 | """ 103 | user = self.request.user 104 | return models.AuthToken.objects.filter(user=user) 105 | 106 | 107 | class CloudManViewSet(drf_helpers.CustomReadOnlySingleViewSet): 108 | """ 109 | List CloudMan related urls. 110 | """ 111 | permission_classes = (IsAuthenticated,) 112 | serializer_class = serializers.CloudManSerializer 113 | 114 | 115 | class DeploymentFilter(dj_filters.FilterSet): 116 | application = dj_filters.CharFilter(field_name="application_version__application__slug") 117 | version = dj_filters.CharFilter(field_name="application_version__version") 118 | status = dj_filters.CharFilter(method='deployment_status_filter') 119 | 120 | def deployment_status_filter(self, queryset, name, value): 121 | return queryset.filter(tasks__action='LAUNCH', tasks___status=value) 122 | 123 | class Meta: 124 | model = models.ApplicationDeployment 125 | fields = ['archived'] 126 | 127 | 128 | class DeploymentViewSet(viewsets.ModelViewSet): 129 | """ 130 | List compute related urls. 131 | """ 132 | permission_classes = (IsAuthenticated,) 133 | serializer_class = serializers.DeploymentSerializer 134 | filter_backends = (filters.OrderingFilter, dj_filters.DjangoFilterBackend) 135 | ordering = ('-added',) 136 | filterset_class = DeploymentFilter 137 | #filter_fields = ('archived','application_version__application__slug', 'application_version__version') 138 | 139 | def get_queryset(self): 140 | """ 141 | This view should return a list of all the deployments 142 | for the currently authenticated user. 143 | """ 144 | user = self.request.user 145 | return models.ApplicationDeployment.objects.filter(owner=user) 146 | 147 | 148 | class DeploymentTaskViewSet(viewsets.ModelViewSet): 149 | """List tasks associated with a deployment.""" 150 | permission_classes = (IsAuthenticated,) 151 | serializer_class = serializers.DeploymentTaskSerializer 152 | filter_backends = (filters.OrderingFilter,) 153 | ordering = ('-updated',) 154 | 155 | def get_queryset(self): 156 | """ 157 | This view should return a list of all the tasks 158 | for the currently associated task. 159 | """ 160 | deployment = self.kwargs.get('deployment_pk') 161 | user = self.request.user 162 | return models.ApplicationDeploymentTask.objects.filter( 163 | deployment=deployment, deployment__owner=user) 164 | 165 | 166 | class PublicKeyList(generics.ListCreateAPIView): 167 | """List public ssh keys associated with the user profile.""" 168 | 169 | permission_classes = (IsAuthenticated,) 170 | serializer_class = serializers.PublicKeySerializer 171 | 172 | def get_queryset(self): 173 | c.incr('user.list.public.keys') 174 | return models.PublicKey.objects.filter( 175 | user_profile__user=self.request.user) 176 | 177 | 178 | class PublicKeyDetail(generics.RetrieveUpdateDestroyAPIView): 179 | """Get a single public ssh keys associated with the user profile.""" 180 | 181 | permission_classes = (IsAuthenticated,) 182 | serializer_class = serializers.PublicKeySerializer 183 | 184 | def get_queryset(self): 185 | return models.PublicKey.objects.filter( 186 | user_profile__user=self.request.user) 187 | 188 | 189 | # Override registration view so that it supports multiple tokens 190 | from django.conf import settings 191 | from allauth.account import app_settings as allauth_settings 192 | from dj_rest_auth.app_settings import TokenSerializer 193 | 194 | class CustomRegisterView(RegisterView): 195 | 196 | def get_default_user_token(self, user): 197 | """ 198 | Returns the default token or None. The default token is 199 | created for the user in 200 | cloudlaunch/authentication.py:default_create_token 201 | """ 202 | return user.auth_tokens.filter(name="default").first() 203 | 204 | def get_response_data(self, user): 205 | if allauth_settings.EMAIL_VERIFICATION == \ 206 | allauth_settings.EmailVerificationMethod.MANDATORY: 207 | return {"detail": _("Verification e-mail sent.")} 208 | 209 | if getattr(settings, 'REST_USE_JWT', False): 210 | data = { 211 | 'user': user, 212 | 'token': self.token 213 | } 214 | return JWTSerializer(data).data 215 | else: 216 | return TokenSerializer(self.get_default_user_token(user)).data 217 | 218 | 219 | class ImageAutocomplete(autocomplete.Select2QuerySetView): 220 | def get_queryset(self): 221 | qs = models.Image.objects.all() 222 | 223 | target_id = self.forwarded.get('target', None) 224 | if target_id: 225 | target = models.CloudDeploymentTarget.objects.get(id=target_id) 226 | return qs.filter(region=target.target_zone.region) 227 | else: 228 | return qs.none() 229 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunchserver/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | # This will make sure the app is always imported when 4 | # Django starts so that shared_task will use this app. 5 | from .celery import app as celery_app # noqa 6 | 7 | __all__ = ['celery_app'] 8 | 9 | default_app_config = 'cloudlaunchserver.apps.CloudLaunchServerConfig' 10 | 11 | # Current version of the library 12 | # Do not edit this number by hand. See Contributing section in the README. 13 | __version__ = '4.0.0' 14 | 15 | 16 | 17 | def get_version(): 18 | """ 19 | Return a string with the current version of the library. 20 | 21 | :rtype: ``string`` 22 | :return: Library version (e.g., "4.0.0"). 23 | """ 24 | return __version__ 25 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunchserver/apps.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import threading 4 | 5 | from celery.worker import WorkController 6 | 7 | from django.apps import AppConfig 8 | from django.dispatch import Signal, receiver 9 | 10 | from .celery import app 11 | 12 | 13 | class CloudLaunchServerConfig(AppConfig): 14 | name = 'cloudlaunchserver' 15 | 16 | def ready(self): 17 | # Only raised during tests when the test server is being shutdown 18 | django_server_shutdown = Signal() 19 | if 'test' in os.environ.get('CELERY_CONFIG_MODULE'): 20 | # Start an in-process threaded celery worker, so that it's not necessary to start 21 | # a separate celery process during testing. 22 | # https://stackoverflow.com/questions/22233680/in-memory-broker-for-celery-unit-tests 23 | # Also refer: https://github.com/celery/celerytest 24 | app.control.purge() 25 | 26 | def mock_import_module(*args, **kwargs): 27 | return None 28 | 29 | # ref: https://medium.com/@erayerdin/how-to-test-celery-in-django-927438757daf 30 | app.loader.import_default_modules = mock_import_module 31 | 32 | worker = WorkController( 33 | app=app, 34 | # not allowed to override TestWorkController.on_consumer_ready 35 | ready_callback=None, 36 | without_heartbeat=True, 37 | without_mingle=True, 38 | without_gossip=True) 39 | 40 | t = threading.Thread(target=worker.start) 41 | t.daemon = True 42 | t.start() 43 | 44 | @receiver(django_server_shutdown) 45 | def on_shutdown(sender, **kwargs): 46 | # Do nothing. Calling stop results in celery hanging waiting for keyboard input 47 | # celery_worker.stop() 48 | pass 49 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunchserver/celery.py: -------------------------------------------------------------------------------- 1 | # File based on: 2 | # http://docs.celeryproject.org/en/latest/django/first-steps-with-django.html 3 | from __future__ import absolute_import 4 | 5 | import logging 6 | import os 7 | 8 | import celery 9 | 10 | from django.conf import settings # noqa 11 | 12 | import sentry_sdk 13 | from sentry_sdk.integrations.celery import CeleryIntegration 14 | 15 | log = logging.getLogger(__name__) 16 | 17 | # set the default Django settings module for the 'celery' program. 18 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'cloudlaunchserver.settings') 19 | 20 | # Set default configuration module name 21 | os.environ.setdefault('CELERY_CONFIG_MODULE', 'cloudlaunchserver.celeryconfig') 22 | 23 | 24 | class Celery(celery.Celery): 25 | 26 | def on_configure(self): 27 | sentry_sdk.init( 28 | dsn=settings.SENTRY_DSN, integrations=[CeleryIntegration()]) 29 | 30 | 31 | app = Celery('proj') 32 | # Changed to use dedicated celery config as detailed in: 33 | # http://docs.celeryproject.org/en/latest/getting-started/first-steps-with-celery.html 34 | # app.config_from_object('django.conf:settings') 35 | app.config_from_envvar('CELERY_CONFIG_MODULE') 36 | app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) 37 | 38 | @app.task(bind=True) 39 | def debug_task(self): 40 | log.debug('Request: {0!r}'.format(self.request)) 41 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunchserver/celeryconfig.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | broker_url = os.environ.get('CELERY_BROKER_URL', 'redis://localhost:6379/0') 4 | result_backend = 'django-db' 5 | beat_scheduler = "django_celery_beat.schedulers:DatabaseScheduler" 6 | result_serializer = 'json' 7 | task_serializer = 'json' 8 | accept_content = ['json'] 9 | #accept_content = ['json', 'yaml'] 10 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunchserver/celeryconfig_test.py: -------------------------------------------------------------------------------- 1 | """ 2 | Celery settings used during cloudlaunch testing 3 | """ 4 | broker_url = 'memory://' 5 | broker_transport_options = {'polling_interval': .01} 6 | broker_backend = 'memory' 7 | result_backend = 'db+sqlite:///results.db' 8 | result_serializer = 'json' 9 | task_serializer = 'json' 10 | accept_content = ['json'] 11 | task_always_eager = True 12 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunchserver/runner/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import click 4 | from django.utils.module_loading import import_string 5 | 6 | import cloudlaunchserver 7 | 8 | 9 | @click.group() 10 | @click.option( 11 | '--config', 12 | default='', 13 | envvar='DJANGO_SETTINGS_MODULE', 14 | help='Path to settings module.', 15 | metavar='PATH') 16 | @click.version_option(version=cloudlaunchserver.__version__) 17 | @click.pass_context 18 | def cli(ctx, config): 19 | """CloudLaunch is a ReSTful, extensible Django app for discovering and launching applications on cloud, container, or local infrastructure. 20 | 21 | Default settings module is `cloudlaunchserver.settings` but 22 | it can be overridden with `DJANGO_CONFIG_MODULE` 23 | or with `--config` parameter. 24 | """ 25 | # Elevate --config option to DJANGO_CONFIG_MODULE env var 26 | if config: 27 | os.environ['DJANGO_SETTINGS_MODULE'] = config 28 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'cloudlaunchserver.settings') 29 | 30 | 31 | list(map(lambda cmd: cli.add_command(import_string(cmd)), ( 32 | 'cloudlaunchserver.runner.commands.help.help', 33 | 'cloudlaunchserver.runner.commands.django.django', 34 | ))) 35 | 36 | 37 | def make_django_command(name, django_command=None, help=None): 38 | "A wrapper to convert a Django subcommand a Click command" 39 | if django_command is None: 40 | django_command = name 41 | 42 | @click.command( 43 | name=name, 44 | help=help, 45 | add_help_option=False, 46 | context_settings=dict( 47 | ignore_unknown_options=True, 48 | )) 49 | @click.argument('management_args', nargs=-1, type=click.UNPROCESSED) 50 | @click.pass_context 51 | def inner(ctx, management_args): 52 | from cloudlaunchserver.runner.commands.django import django 53 | ctx.params['management_args'] = (django_command,) + management_args 54 | ctx.forward(django) 55 | 56 | return inner 57 | 58 | 59 | list(map(cli.add_command, ( 60 | make_django_command('migrate', help=( 61 | 'Run migrations (like `cloudlaunchserver django migrate`).')), 62 | make_django_command('shell', help='Run a Python interactive interpreter.') 63 | ))) 64 | 65 | 66 | def main(): 67 | cli(obj={}, max_content_width=100) 68 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunchserver/runner/__main__.py: -------------------------------------------------------------------------------- 1 | from cloudlaunchserver.runner import main 2 | main() 3 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunchserver/runner/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/galaxyproject/cloudlaunch/e96cb45d1c2b19be9b14ac3202df0c708f017425/django-cloudlaunch/cloudlaunchserver/runner/commands/__init__.py -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunchserver/runner/commands/django.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import click 4 | 5 | 6 | @click.command( 7 | add_help_option=False, context_settings=dict(ignore_unknown_options=True)) 8 | @click.argument('management_args', nargs=-1, type=click.UNPROCESSED) 9 | @click.pass_context 10 | def django(ctx, management_args): 11 | "Execute Django subcommands." 12 | if len(management_args) and management_args[0] == 'test': 13 | if '--settings' not in management_args: 14 | if os.environ['DJANGO_SETTINGS_MODULE'] == 'cloudlaunchserver.settings': 15 | os.environ["DJANGO_SETTINGS_MODULE"] = "cloudlaunchserver.settings_local" 16 | 17 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "cloudlaunchserver.settings") 18 | 19 | from django.core.management import execute_from_command_line 20 | execute_from_command_line(argv=[ctx.command_path] + list(management_args)) 21 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunchserver/runner/commands/help.py: -------------------------------------------------------------------------------- 1 | import click 2 | 3 | 4 | @click.command() 5 | @click.pass_context 6 | def help(ctx): 7 | "Show this message and exit." 8 | click.echo(ctx.parent.get_help()) 9 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunchserver/runner/decorators.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | 4 | def configuration(f): 5 | import click 6 | from functools import update_wrapper 7 | 8 | @click.pass_context 9 | def inner(ctx, *args, **kwargs): 10 | # HACK: We can't call `configure()` from within tests 11 | # since we don't load config files from disk, so we 12 | # need a way to bypass this initialization step 13 | if os.environ.get('_CLOUDLAUNCH_SERVER_SKIP_CONFIGURATION') != '1': 14 | from cloudlaunchserver.runner import configure 15 | configure() 16 | return ctx.invoke(f, *args, **kwargs) 17 | return update_wrapper(inner, f) 18 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunchserver/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for cloudlaunch project. 3 | 4 | Generated by 'django-admin startproject' using Django 1.9. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.9/topics/settings/ 8 | 9 | For the full list of settings and their values, see 10 | https://docs.djangoproject.com/en/1.9/ref/settings/ 11 | """ 12 | import os 13 | 14 | import sentry_sdk 15 | from sentry_sdk.integrations.django import DjangoIntegration 16 | 17 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 18 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 19 | 20 | # Quick-start development settings - unsuitable for production 21 | # See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/ 22 | 23 | # Django site id 24 | SITE_ID = 1 25 | 26 | # SECURITY WARNING: keep the secret key used in production secret! 27 | SECRET_KEY = 'CHANGEthisONinstall' 28 | 29 | # SECURITY WARNING: don't run with debug turned on in production! 30 | DEBUG = os.environ.get('DJANGO_DEBUG', False) 31 | 32 | ALLOWED_HOSTS = ['*'] 33 | 34 | ACCOUNT_AUTHENTICATION_METHOD = "username_email" 35 | ACCOUNT_EMAIL_REQUIRED = False 36 | LOGIN_REDIRECT_URL = "/catalog" 37 | 38 | # Begin: django-cors-headers settings 39 | CORS_ORIGIN_ALLOW_ALL = True 40 | 41 | # Django filters out headers with an underscore by default, so make sure they 42 | # have dashes instead. 43 | from corsheaders.defaults import default_headers 44 | CORS_ALLOW_HEADERS = default_headers + ( 45 | 'cl-credentials-id', 46 | 'cl-os-username', 47 | 'cl-os-password', 48 | 'cl-os-project-id', 49 | 'cl-os-project-name', 50 | 'cl-os-project-domain-id', 51 | 'cl-os-project-domain-name', 52 | 'cl-os-user-domain-id', 53 | 'cl-os-user-domain-name', 54 | 'cl-os-identity-api-version', 55 | 'cl-aws-access-key', 56 | 'cl-aws-secret-key', 57 | 'cl-azure-subscription-id' 58 | 'cl-azure-client-id', 59 | 'cl-azure-secret', 60 | 'cl-azure-tenant', 61 | 'cl-azure-resource-group', 62 | 'cl-azure-storage-account', 63 | 'cl-azure-vm-default-username', 64 | 'cl-gcp-credentials-json', 65 | 'cl-gcp-vm-default-username', 66 | ) 67 | # End: django-cors-headers settings 68 | 69 | # Absolute path to the directory static files should be collected to. 70 | # Don't put anything in this directory yourself; store your static files 71 | # in apps' "static/" subdirectories and in STATICFILES_DIRS. 72 | # Example: "/var/www/example.com/static/" 73 | STATIC_ROOT = os.path.join(BASE_DIR, 'static') 74 | 75 | # Application definition 76 | 77 | INSTALLED_APPS = [ 78 | # Django auto complete light - for autocompleting foreign keys in admin 79 | 'dal', 80 | 'dal_select2', 81 | 'django.contrib.admin', 82 | 'django.contrib.auth', 83 | 'django.contrib.contenttypes', 84 | 'django.contrib.sessions', 85 | 'django.contrib.messages', 86 | 'whitenoise.runserver_nostatic', 87 | 'django.contrib.staticfiles', 88 | 'django.contrib.sites', 89 | 'nested_admin', 90 | 'corsheaders', 91 | 'dj_rest_auth', 92 | 'allauth', 93 | 'allauth.account', 94 | 'dj_rest_auth.registration', 95 | 'allauth.socialaccount', 96 | 'allauth.socialaccount.providers.facebook', 97 | 'allauth.socialaccount.providers.github', 98 | 'allauth.socialaccount.providers.google', 99 | 'allauth.socialaccount.providers.twitter', 100 | 'djcloudbridge', 101 | 'public_appliances', 102 | 'cloudlaunch', 103 | # rest framework must come after cloudlaunch so templates can be overridden 104 | 'rest_framework', 105 | 'django_celery_results', 106 | 'django_celery_beat', 107 | 'django_countries', 108 | 'django_filters', 109 | 'polymorphic', 110 | 'cloudlaunchserver' 111 | ] 112 | 113 | MIDDLEWARE = [ 114 | 'django.middleware.security.SecurityMiddleware', 115 | 'whitenoise.middleware.WhiteNoiseMiddleware', 116 | 'django.contrib.sessions.middleware.SessionMiddleware', 117 | 'corsheaders.middleware.CorsMiddleware', 118 | 'django.middleware.common.CommonMiddleware', 119 | 'django.middleware.csrf.CsrfViewMiddleware', 120 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 121 | 'django.contrib.messages.middleware.MessageMiddleware', 122 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 123 | ] 124 | 125 | ROOT_URLCONF = 'cloudlaunchserver.urls' 126 | 127 | TEMPLATES = [ 128 | { 129 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 130 | 'DIRS': [], 131 | 'APP_DIRS': True, 132 | 'OPTIONS': { 133 | 'context_processors': [ 134 | 'django.template.context_processors.debug', 135 | 'django.template.context_processors.request', 136 | 'django.contrib.auth.context_processors.auth', 137 | 'django.contrib.messages.context_processors.messages', 138 | ], 139 | }, 140 | }, 141 | ] 142 | 143 | WSGI_APPLICATION = 'cloudlaunchserver.wsgi.application' 144 | 145 | 146 | # Database 147 | # https://docs.djangoproject.com/en/1.9/ref/settings/#databases 148 | 149 | DATABASES = { 150 | 'default': { 151 | 'ENGINE': 'django.db.backends.' + os.environ.get('CLOUDLAUNCH_DB_ENGINE', 'sqlite3'), 152 | 'NAME': os.environ.get('CLOUDLAUNCH_DB_NAME', os.path.join(BASE_DIR, 'db.sqlite3')), 153 | # The following settings are not used with sqlite3: 154 | 'USER': os.environ.get('CLOUDLAUNCH_DB_USER'), 155 | 'HOST': os.environ.get('CLOUDLAUNCH_DB_HOST'), # Empty for localhost through domain sockets or '127.0.0.1' for localhost through TCP. 156 | 'PORT': os.environ.get('CLOUDLAUNCH_DB_PORT'), # Set to empty string for default. 157 | 'PASSWORD': os.environ.get('CLOUDLAUNCH_DB_PASSWORD'), 158 | } 159 | } 160 | 161 | 162 | SITE_ID = 1 163 | 164 | # Password validation 165 | # https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators 166 | 167 | AUTH_PASSWORD_VALIDATORS = [ 168 | { 169 | 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', 170 | }, 171 | { 172 | 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', 173 | }, 174 | { 175 | 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', 176 | }, 177 | { 178 | 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', 179 | }, 180 | ] 181 | 182 | AUTHENTICATION_BACKENDS = [ 183 | # Needed to login by username in Django admin, regardless of `allauth` 184 | 'django.contrib.auth.backends.ModelBackend', 185 | # `allauth` specific authentication methods, such as login by e-mail 186 | 'allauth.account.auth_backends.AuthenticationBackend', 187 | ] 188 | 189 | # Internationalization 190 | # https://docs.djangoproject.com/en/1.9/topics/i18n/ 191 | 192 | LANGUAGE_CODE = 'en-us' 193 | 194 | TIME_ZONE = 'US/Eastern' 195 | 196 | USE_I18N = False 197 | 198 | USE_L10N = True 199 | 200 | USE_TZ = True 201 | 202 | 203 | CLOUDLAUNCH_PATH_PREFIX = os.environ.get('CLOUDLAUNCH_PATH_PREFIX', '') 204 | FORCE_SCRIPT_NAME = CLOUDLAUNCH_PATH_PREFIX 205 | 206 | # Static files (CSS, JavaScript, Images) 207 | # https://docs.djangoproject.com/en/1.9/howto/static-files/ 208 | STATIC_URL = CLOUDLAUNCH_PATH_PREFIX + '/cloudlaunch/static/' 209 | 210 | 211 | # Installed apps settings 212 | 213 | REST_FRAMEWORK = { 214 | 'PAGE_SIZE': 50, 215 | 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.PageNumberPagination', 216 | 'DEFAULT_AUTHENTICATION_CLASSES': ( 217 | # 'rest_framework.authentication.BasicAuthentication', 218 | 'rest_framework.authentication.SessionAuthentication', 219 | 'cloudlaunch.authentication.TokenAuthentication' 220 | ), 221 | 'DEFAULT_SCHEMA_CLASS': 'rest_framework.schemas.AutoSchema' 222 | } 223 | 224 | REST_AUTH_SERIALIZERS = { 225 | 'USER_DETAILS_SERIALIZER': 'djcloudbridge.serializers.UserSerializer' 226 | } 227 | REST_AUTH_TOKEN_MODEL = 'cloudlaunch.models.AuthToken' 228 | REST_AUTH_TOKEN_CREATOR = 'cloudlaunch.authentication.default_create_token' 229 | 230 | REST_SESSION_LOGIN = True 231 | 232 | REST_SCHEMA_BASE_URL = CLOUDLAUNCH_PATH_PREFIX + '/cloudlaunch/' 233 | 234 | SENTRY_DSN = os.environ.get('SENTRY_DSN', '') 235 | sentry_sdk.init( 236 | # dsn="https://@sentry.io/", 237 | dsn=SENTRY_DSN, 238 | integrations=[DjangoIntegration()] 239 | ) 240 | 241 | LOGGING = { 242 | 'version': 1, 243 | 'disable_existing_loggers': False, 244 | 'formatters': { 245 | 'verbose': { 246 | 'format': '%(asctime)s %(levelname)s %(pathname)s:%(lineno)d - %(message)s' 247 | }, 248 | }, 249 | 'filters': { 250 | 'require_debug_true': { 251 | '()': 'django.utils.log.RequireDebugTrue', 252 | }, 253 | }, 254 | 'handlers': { 255 | 'console': { 256 | 'filters': ['require_debug_true'], 257 | 'class': 'logging.StreamHandler', 258 | 'formatter': 'verbose', 259 | 'level': 'DEBUG', 260 | }, 261 | 'file-cloudlaunch': { 262 | 'class': 'logging.FileHandler', 263 | 'formatter': 'verbose', 264 | 'level': 'INFO', 265 | 'filename': 'cloudlaunch.log', 266 | }, 267 | 'file-django': { 268 | 'class': 'logging.FileHandler', 269 | 'formatter': 'verbose', 270 | 'level': 'WARNING', 271 | 'filename': 'cloudlaunch-django.log', 272 | } 273 | }, 274 | 'loggers': { 275 | 'django': { 276 | 'handlers': ['console', 'file-django'], 277 | 'level': os.getenv('DJANGO_LOG_LEVEL', 'DEBUG'), 278 | }, 279 | 'django.db.backends': { 280 | 'handlers': ['file-django'], 281 | 'level': 'INFO', 282 | }, 283 | 'django.template': { 284 | 'handlers': ['console', 'file-django'], 285 | 'level': 'INFO', 286 | 'propagate': True, 287 | }, 288 | 'django.server': { 289 | 'handlers': ['console', 'file-django'], 290 | 'level': 'ERROR', 291 | 'propagate': True, 292 | }, 293 | 'django.utils.autoreload': { 294 | 'level': 'INFO' 295 | }, 296 | 'cloudlaunch': { 297 | 'handlers': ['console', 'file-cloudlaunch'], 298 | 'level': 'DEBUG', 299 | 'propagate': False 300 | } 301 | } 302 | } 303 | 304 | # CloudLaunch specific settings 305 | CLOUDLAUNCH_APP_REGISTRY_URL = 'https://raw.githubusercontent.com/galaxyproject/' \ 306 | 'cloudlaunch-registry/master/app-registry.yaml' 307 | 308 | 309 | STATICFILES_STORAGE = 'whitenoise.storage.CompressedStaticFilesStorage' 310 | 311 | # Allow settings to be overridden in a cloudlaunch/settings_local.py 312 | try: 313 | from cloudlaunchserver.settings_local import * # noqa 314 | except ImportError: 315 | pass 316 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunchserver/settings_local.py.sample: -------------------------------------------------------------------------------- 1 | import os 2 | BASE_DIR = os.path.realpath(os.path.dirname(__file__)) 3 | 4 | DEBUG = True 5 | # 6 | # Edit the following Django settings 7 | # 8 | 9 | # Set the desired database, sqlite for local/dev installations works good; 10 | # Postgres is better and should definitely be enabled for production installs. 11 | DATABASES = { 12 | 'default': { 13 | 'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2' or 'sqlite3'. 14 | 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), # Or path to database file if using sqlite3. 15 | 'USER': '', # Not used with sqlite3. 16 | 'PASSWORD': '', # Not used with sqlite3. 17 | 'HOST': '', # Set to empty string for localhost. Not used with sqlite3. 18 | 'PORT': '', # Set to empty string for default. Not used with sqlite3. 19 | } 20 | } 21 | 22 | # Read more about the encryption keys at django-fernet-fields.readthedocs.org 23 | #FERNET_KEYS = [ 24 | # 'A key for encrypting sensitive data - change this and keep it safe!', 25 | #] 26 | 27 | # Set this absolute path then run: python biocloudcentral/manage.py collectstatic 28 | STATIC_ROOT = '/srv/cloudlaunch/media' 29 | 30 | # 31 | # App-specific settings (i.e., not Django-related) 32 | # 33 | 34 | # Page title and brand text 35 | BRAND = "Cloud Launch" 36 | # Supply a string that will be prominently displayed on the launch page. 37 | # You can use HTML tags as part of the string. 38 | NOTICE = None 39 | 40 | # Whether to add an email field to the form and make it required or optional. 41 | ASK_FOR_EMAIL = False 42 | REQUIRE_EMAIL = False 43 | 44 | 45 | # 46 | # You probably do not want to edit these settings 47 | # 48 | SESSION_ENGINE = "django.contrib.sessions.backends.db" 49 | REDIRECT_BASE = None 50 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunchserver/settings_prod.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings used during cloudlaunch production 3 | """ 4 | from cloudlaunchserver.settings import * # noqa 5 | 6 | DATABASES = { 7 | 'default': { 8 | 'ENGINE': 'django.db.backends.' + os.environ.get('CLOUDLAUNCH_DB_ENGINE', 'sqlite3'), 9 | 'NAME': os.environ.get('CLOUDLAUNCH_DB_NAME', os.path.join(BASE_DIR, 'db.sqlite3')), 10 | 'USER': os.environ.get('CLOUDLAUNCH_DB_USER'), 11 | 'HOST': os.environ.get('CLOUDLAUNCH_DB_HOST'), 12 | 'PORT': os.environ.get('CLOUDLAUNCH_DB_PORT'), 13 | 'PASSWORD': os.environ.get('CLOUDLAUNCH_DB_PASSWORD'), 14 | } 15 | } 16 | 17 | SECRET_KEY = os.environ.get('CLOUDLAUNCH_SECRET_KEY') or SECRET_KEY 18 | 19 | # Read fernet keys from env or default to existing settings keys 20 | ENV_FERNET_KEYS = os.environ.get('CLOUDLAUNCH_FERNET_KEYS') 21 | if ENV_FERNET_KEYS: 22 | ENV_FERNET_KEYS = ENV_FERNET_KEYS.split(",") 23 | try: 24 | # Use FERNET_KEYS defined in cloudlaunchserver.settings if defined 25 | FERNET_KEYS = ENV_FERNET_KEYS or FERNET_KEYS 26 | except NameError: 27 | FERNET_KEYS = ENV_FERNET_KEYS 28 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunchserver/settings_test.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings used during cloudlaunch testing 3 | """ 4 | import signal 5 | import sys 6 | 7 | 8 | # The integration test script sends a SIGINT to terminate the django server 9 | # after the tests are complete. Handle the SIGINT here and terminate 10 | # gracefully, or coverage will terminate abruptly without writing the 11 | # .coverage file 12 | def test_signal_handler(*args, **kwargs): 13 | sys.exit(0) 14 | signal.signal(signal.SIGINT, test_signal_handler) 15 | 16 | from cloudlaunchserver.settings import * # noqa 17 | 18 | # Turn on Django debugging 19 | DEBUG = True 20 | 21 | DATABASES = { 22 | 'default': { 23 | 'ENGINE': 'django.db.backends.sqlite3', 24 | 'NAME': '/tmp/cloudlaunch_testdb.sqlite3', 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunchserver/urls.py: -------------------------------------------------------------------------------- 1 | """cloudlaunchserver URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/1.9/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') 9 | Class-based views 10 | 1. Add an import: from other_app.views import Home 11 | 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Add an import: from blog import urls as blog_urls 14 | 2. Import the include() function: from django.urls import re_path, include 15 | 3. Add a URL to urlpatterns: url(r'^blog/', include(blog_urls)) 16 | """ 17 | from django.conf import settings # noqa 18 | from django.urls import include 19 | from django.urls import re_path 20 | from django.contrib import admin 21 | 22 | 23 | urlpatterns = [ 24 | re_path(r'^cloudlaunch/admin/', admin.site.urls), 25 | re_path(r'^cloudlaunch/nested_admin/', include('nested_admin.urls')), 26 | re_path(r'^cloudlaunch/', include('cloudlaunch.urls')) 27 | ] 28 | -------------------------------------------------------------------------------- /django-cloudlaunch/cloudlaunchserver/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for cloudlaunch project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.9/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "cloudlaunchserver.settings") 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /django-cloudlaunch/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | import signal 5 | 6 | 7 | if __name__ == "__main__": 8 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "cloudlaunchserver.settings") 9 | 10 | from django.core.management import execute_from_command_line 11 | 12 | execute_from_command_line(sys.argv) 13 | -------------------------------------------------------------------------------- /django-cloudlaunch/public_appliances/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/galaxyproject/cloudlaunch/e96cb45d1c2b19be9b14ac3202df0c708f017425/django-cloudlaunch/public_appliances/__init__.py -------------------------------------------------------------------------------- /django-cloudlaunch/public_appliances/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | from . import models 4 | 5 | ### Public Services ### 6 | class SponsorsAdmin(admin.ModelAdmin): 7 | models = models.Sponsor 8 | 9 | 10 | class LocationAdmin(admin.ModelAdmin): 11 | models = models.Location 12 | 13 | 14 | class PublicServicesAdmin(admin.ModelAdmin): 15 | prepopulated_fields = {"slug": ("name",)} 16 | models = models.PublicService 17 | 18 | 19 | ### Public Services Admin Registration ### 20 | admin.site.register(models.PublicService, PublicServicesAdmin) 21 | admin.site.register(models.Sponsor, SponsorsAdmin) 22 | admin.site.register(models.Location, LocationAdmin) 23 | 24 | -------------------------------------------------------------------------------- /django-cloudlaunch/public_appliances/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class PublicAppliancesConfig(AppConfig): 5 | name = 'public_appliances' 6 | -------------------------------------------------------------------------------- /django-cloudlaunch/public_appliances/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.9 on 2020-01-24 20:33 2 | 3 | from django.db import migrations, models 4 | import django.db.models.deletion 5 | import django_countries.fields 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | initial = True 11 | 12 | dependencies = [ 13 | ] 14 | 15 | operations = [ 16 | migrations.CreateModel( 17 | name='Location', 18 | fields=[ 19 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 20 | ('latitude', models.FloatField(blank=True, null=True)), 21 | ('longitude', models.FloatField(blank=True, null=True)), 22 | ('city', models.TextField(blank=True, null=True)), 23 | ('country', django_countries.fields.CountryField(blank='(select country)', max_length=2)), 24 | ], 25 | ), 26 | migrations.CreateModel( 27 | name='Sponsor', 28 | fields=[ 29 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 30 | ('name', models.TextField()), 31 | ('url', models.URLField(null=True)), 32 | ], 33 | ), 34 | migrations.CreateModel( 35 | name='Tag', 36 | fields=[ 37 | ('name', models.TextField(primary_key=True, serialize=False)), 38 | ], 39 | ), 40 | migrations.CreateModel( 41 | name='PublicService', 42 | fields=[ 43 | ('added', models.DateTimeField(auto_now_add=True)), 44 | ('updated', models.DateTimeField(auto_now=True)), 45 | ('name', models.CharField(max_length=60)), 46 | ('slug', models.SlugField(max_length=100, primary_key=True, serialize=False)), 47 | ('links', models.URLField()), 48 | ('purpose', models.TextField(blank=True, null=True)), 49 | ('comments', models.TextField(blank=True, null=True)), 50 | ('email_user_support', models.EmailField(blank=True, max_length=254, null=True)), 51 | ('quotas', models.TextField(blank=True, null=True)), 52 | ('featured', models.BooleanField(default=False)), 53 | ('logo', models.URLField(blank=True, null=True)), 54 | ('location', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='public_appliances.Location')), 55 | ('sponsors', models.ManyToManyField(blank=True, to='public_appliances.Sponsor')), 56 | ('tags', models.ManyToManyField(blank=True, to='public_appliances.Tag')), 57 | ], 58 | options={ 59 | 'abstract': False, 60 | }, 61 | ), 62 | ] 63 | -------------------------------------------------------------------------------- /django-cloudlaunch/public_appliances/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/galaxyproject/cloudlaunch/e96cb45d1c2b19be9b14ac3202df0c708f017425/django-cloudlaunch/public_appliances/migrations/__init__.py -------------------------------------------------------------------------------- /django-cloudlaunch/public_appliances/models.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | from django.db import models 4 | 5 | from django_countries.fields import CountryField 6 | 7 | from djcloudbridge import models as cb_models 8 | 9 | from urllib.parse import urlparse 10 | from django.core.exceptions import ObjectDoesNotExist 11 | 12 | 13 | ### PublicServer Models ### 14 | class Tag(models.Model): 15 | """ 16 | Tag referencing a keyword for search features 17 | """ 18 | name = models.TextField(primary_key=True) 19 | 20 | 21 | class Sponsor(models.Model): 22 | """ 23 | A Sponsor is defined by his name and his link url. 24 | Directly inspired by https://wiki.galaxyproject.org/PublicGalaxyServers Sponsor(s) part 25 | """ 26 | name = models.TextField() 27 | url = models.URLField(null=True) 28 | 29 | def __str__(self): 30 | return "{0}".format(self.name) 31 | 32 | 33 | class Location(models.Model): 34 | """ 35 | A location containing the latitude and longitude (fetched from the ip) and 36 | a django_country https://github.com/SmileyChris/django-countries 37 | """ 38 | latitude = models.FloatField(blank=True, null=True) 39 | longitude = models.FloatField(blank=True, null=True) 40 | 41 | city = models.TextField(blank=True, null=True) 42 | 43 | country = CountryField(blank='(select country)') 44 | 45 | def __str__(self): 46 | return "Country: {0}, Latitude: {1}, Longitude: {2}".format(self.country, 47 | self.latitude, 48 | self.longitude, 49 | self.city) 50 | 51 | 52 | class PublicService(cb_models.DateNameAwareModel): 53 | """ 54 | Public Service class to display the public services available, 55 | for example, on https://wiki.galaxyproject.org/PublicGalaxyServers 56 | The fields have been inspired by this public galaxy page 57 | """ 58 | slug = models.SlugField(max_length=100, primary_key=True) 59 | links = models.URLField() 60 | location = models.ForeignKey(Location, on_delete=models.CASCADE, blank=True, null=True) 61 | purpose = models.TextField(blank=True, null=True) 62 | comments = models.TextField(blank=True, null=True) 63 | email_user_support = models.EmailField(blank=True, null=True) 64 | quotas = models.TextField(blank=True, null=True) 65 | sponsors = models.ManyToManyField(Sponsor, blank=True) 66 | # Featured links means a more important link to show "first" 67 | featured = models.BooleanField(default=False) 68 | # The referenced application, if existing 69 | # application = models.ForeignKey(Application, on_delete=models.CASCADE, blank=True, null=True) 70 | # The url link to the logo of the Service 71 | logo = models.URLField(blank=True, null=True) 72 | tags = models.ManyToManyField(Tag, blank=True) 73 | 74 | def __str__(self): 75 | return "{0}".format(self.name) 76 | 77 | def save(self, *args, **kwargs): 78 | if not self.slug: 79 | # Newly created object, so set slug 80 | self.slug = slugify(self.name) 81 | 82 | # Construct the API to find geolocation from ip 83 | api_hostname = 'http://ip-api.com' 84 | return_format = 'json' 85 | parsed_url = urlparse(self.links) 86 | netloc = parsed_url.netloc 87 | geolocation_api = '{0}/{1}/{2}'.format(api_hostname, return_format, netloc) 88 | 89 | response = requests.get(geolocation_api) 90 | if response.status_code != 200: 91 | raise Exception("Couldn't find the geolocation from ip {0}: {1}".format(geolocation_api, response.status_code)) 92 | # Construct or get the Location 93 | json_geoloc = response.json() 94 | self.location = Location.objects.get_or_create(longitude=json_geoloc["lon"], 95 | latitude=json_geoloc["lat"], 96 | defaults={ 97 | 'country': json_geoloc["countryCode"], 98 | 'city': json_geoloc["city"], 99 | },)[0] 100 | 101 | super(PublicService, self).save(*args, **kwargs) 102 | -------------------------------------------------------------------------------- /django-cloudlaunch/public_appliances/serializers.py: -------------------------------------------------------------------------------- 1 | from rest_framework import serializers 2 | 3 | from django_countries.serializer_fields import CountryField 4 | 5 | from . import models 6 | 7 | 8 | 9 | ### Public Services Serializers ### 10 | class LocationSerializer(serializers.HyperlinkedModelSerializer): 11 | url = serializers.HyperlinkedIdentityField( 12 | view_name="pubapp:location-detail", 13 | ) 14 | country = CountryField(country_dict=True) 15 | 16 | class Meta: 17 | model = models.Location 18 | fields = '__all__' 19 | 20 | 21 | class SponsorSerializer(serializers.HyperlinkedModelSerializer): 22 | class Meta: 23 | model = models.Sponsor 24 | fields = '__all__' 25 | 26 | 27 | class PublicServiceSerializer(serializers.HyperlinkedModelSerializer): 28 | url = serializers.HyperlinkedIdentityField( 29 | view_name="pubapp:publicservice-detail", 30 | ) 31 | 32 | location = LocationSerializer(read_only=True) 33 | 34 | class Meta: 35 | model = models.PublicService 36 | fields = '__all__' 37 | -------------------------------------------------------------------------------- /django-cloudlaunch/public_appliances/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /django-cloudlaunch/public_appliances/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import include 2 | from django.urls import re_path 3 | 4 | from . import views 5 | 6 | from djcloudbridge.drf_routers import HybridDefaultRouter, HybridNestedRouter, HybridSimpleRouter 7 | 8 | 9 | router = HybridSimpleRouter() 10 | 11 | ### Public services ### 12 | router.register(r'services', views.PublicServiceViewSet) 13 | router.register(r'sponsors', views.SponsorViewSet) 14 | router.register(r'locations', views.LocationViewSet) 15 | 16 | public_services_regex_pattern = r'' 17 | 18 | app_name = 'pubapp' 19 | 20 | urlpatterns = [ 21 | re_path(public_services_regex_pattern, include(router.urls)), 22 | ] 23 | -------------------------------------------------------------------------------- /django-cloudlaunch/public_appliances/views.py: -------------------------------------------------------------------------------- 1 | from rest_framework import viewsets 2 | 3 | from . import models 4 | from . import serializers 5 | 6 | 7 | ### Public Services ### 8 | class LocationViewSet(viewsets.ModelViewSet): 9 | """ 10 | List of all locations 11 | """ 12 | queryset = models.Location.objects.all() 13 | serializer_class = serializers.LocationSerializer 14 | 15 | 16 | class SponsorViewSet(viewsets.ModelViewSet): 17 | """ 18 | List sponsors 19 | """ 20 | queryset = models.Sponsor.objects.all() 21 | serializer_class = serializers.SponsorSerializer 22 | 23 | 24 | class PublicServiceViewSet(viewsets.ModelViewSet): 25 | """ 26 | List public services 27 | """ 28 | queryset = models.PublicService.objects.all() 29 | serializer_class = serializers.PublicServiceSerializer 30 | 31 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = cloudlaunch 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # cloudlaunch documentation build configuration file, created by 5 | # sphinx-quickstart on Mon Dec 18 16:10:21 2017. 6 | # 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | # 20 | import os 21 | import sys 22 | # sys.path.insert(0, os.path.abspath('.')) 23 | 24 | import sphinx_rtd_theme 25 | 26 | # If extensions (or modules to document with autodoc) are in another directory, 27 | # add these directories to sys.path here. If the directory is relative to the 28 | # documentation root, use os.path.abspath to make it absolute, like shown here. 29 | sys.path.insert(0, os.path.abspath('../django-cloudlaunch/cloudlaunch')) 30 | 31 | # -- General configuration ------------------------------------------------ 32 | 33 | # If your documentation needs a minimal Sphinx version, state it here. 34 | # 35 | # needs_sphinx = '1.0' 36 | 37 | # Add any Sphinx extension module names here, as strings. They can be 38 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 39 | # ones. 40 | extensions = ['sphinx.ext.autodoc', 41 | 'sphinx.ext.coverage'] 42 | 43 | # Add any paths that contain templates here, relative to this directory. 44 | templates_path = ['_templates'] 45 | 46 | # The suffix(es) of source filenames. 47 | # You can specify multiple suffix as a list of string: 48 | # 49 | # source_suffix = ['.rst', '.md'] 50 | source_suffix = '.rst' 51 | 52 | # The master toctree document. 53 | master_doc = 'index' 54 | 55 | # General information about the project. 56 | project = 'cloudlaunch' 57 | copyright = '2020, CloudVE' 58 | author = 'Galaxy and GVL projects' 59 | 60 | # The version info for the project you're documenting, acts as replacement for 61 | # |version| and |release|, also used in various other places throughout the 62 | # built documents. 63 | # 64 | # Do not edit this number by hand. See Contributing section in the README. 65 | version = '4.0.0' 66 | # The full version, including alpha/beta/rc tags. 67 | release = '4.0.0' 68 | 69 | # The language for content autogenerated by Sphinx. Refer to documentation 70 | # for a list of supported languages. 71 | # 72 | # This is also used if you do content translation via gettext catalogs. 73 | # Usually you set "language" from the command line for these cases. 74 | language = None 75 | 76 | # List of patterns, relative to source directory, that match files and 77 | # directories to ignore when looking for source files. 78 | # This patterns also effect to html_static_path and html_extra_path 79 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 80 | 81 | # The name of the Pygments (syntax highlighting) style to use. 82 | pygments_style = 'sphinx' 83 | 84 | # If true, `todo` and `todoList` produce output, else they produce nothing. 85 | todo_include_todos = False 86 | 87 | 88 | # -- Options for HTML output ---------------------------------------------- 89 | 90 | # The theme to use for HTML and HTML Help pages. See the documentation for 91 | # a list of builtin themes. 92 | # 93 | html_theme = 'sphinx_rtd_theme' 94 | 95 | # Theme options are theme-specific and customize the look and feel of a theme 96 | # further. For a list of options available for each theme, see the 97 | # documentation. 98 | # 99 | # html_theme_options = {} 100 | 101 | # Add any paths that contain custom static files (such as style sheets) here, 102 | # relative to this directory. They are copied after the builtin static files, 103 | # so a file named "default.css" will overwrite the builtin "default.css". 104 | html_static_path = ['_static'] 105 | 106 | # Custom sidebar templates, must be a dictionary that maps document names 107 | # to template names. 108 | # 109 | # This is required for the alabaster theme 110 | # refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars 111 | html_sidebars = { 112 | '**': [ 113 | 'relations.html', # needs 'show_related': True theme option to display 114 | 'searchbox.html', 115 | ] 116 | } 117 | 118 | 119 | # -- Options for HTMLHelp output ------------------------------------------ 120 | 121 | # Output file base name for HTML help builder. 122 | htmlhelp_basename = 'cloudlaunchdoc' 123 | 124 | 125 | # -- Options for LaTeX output --------------------------------------------- 126 | 127 | latex_elements = { 128 | # The paper size ('letterpaper' or 'a4paper'). 129 | # 130 | # 'papersize': 'letterpaper', 131 | 132 | # The font size ('10pt', '11pt' or '12pt'). 133 | # 134 | # 'pointsize': '10pt', 135 | 136 | # Additional stuff for the LaTeX preamble. 137 | # 138 | # 'preamble': '', 139 | 140 | # Latex figure (float) alignment 141 | # 142 | # 'figure_align': 'htbp', 143 | } 144 | 145 | # Grouping the document tree into LaTeX files. List of tuples 146 | # (source start file, target name, title, 147 | # author, documentclass [howto, manual, or own class]). 148 | latex_documents = [ 149 | (master_doc, 'cloudlaunch.tex', 'cloudlaunch Documentation', 150 | 'CloudVE', 'manual'), 151 | ] 152 | 153 | 154 | # -- Options for manual page output --------------------------------------- 155 | 156 | # One entry per manual page. List of tuples 157 | # (source start file, name, description, authors, manual section). 158 | man_pages = [ 159 | (master_doc, 'cloudlaunch', 'cloudlaunch Documentation', 160 | [author], 1) 161 | ] 162 | 163 | 164 | # -- Options for Texinfo output ------------------------------------------- 165 | 166 | # Grouping the document tree into Texinfo files. List of tuples 167 | # (source start file, target name, title, author, 168 | # dir menu entry, description, category) 169 | texinfo_documents = [ 170 | (master_doc, 'cloudlaunch', 'cloudlaunch Documentation', 171 | author, 'cloudlaunch', 'One line description of project.', 172 | 'Miscellaneous'), 173 | ] 174 | 175 | 176 | 177 | -------------------------------------------------------------------------------- /docs/images/add-social-app-sm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/galaxyproject/cloudlaunch/e96cb45d1c2b19be9b14ac3202df0c708f017425/docs/images/add-social-app-sm.png -------------------------------------------------------------------------------- /docs/images/add-social-app.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/galaxyproject/cloudlaunch/e96cb45d1c2b19be9b14ac3202df0c708f017425/docs/images/add-social-app.png -------------------------------------------------------------------------------- /docs/images/github-oauth-app.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/galaxyproject/cloudlaunch/e96cb45d1c2b19be9b14ac3202df0c708f017425/docs/images/github-oauth-app.png -------------------------------------------------------------------------------- /docs/images/github-ouath-app-sm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/galaxyproject/cloudlaunch/e96cb45d1c2b19be9b14ac3202df0c708f017425/docs/images/github-ouath-app-sm.png -------------------------------------------------------------------------------- /docs/images/twitter-oauth-app-sm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/galaxyproject/cloudlaunch/e96cb45d1c2b19be9b14ac3202df0c708f017425/docs/images/twitter-oauth-app-sm.png -------------------------------------------------------------------------------- /docs/images/twitter-oauth-app.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/galaxyproject/cloudlaunch/e96cb45d1c2b19be9b14ac3202df0c708f017425/docs/images/twitter-oauth-app.png -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. cloudlaunch documentation master file, created by 2 | sphinx-quickstart on Mon Dec 18 16:10:21 2017. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to the CloudLaunch developer documentation 7 | ================================================== 8 | 9 | CloudLaunch is a ReSTful, extensible Django app for discovering and launching 10 | applications on cloud, container, or local infrastructure. A live version is 11 | available at https://launch.usegalaxy.org/. 12 | 13 | CloudLaunch can be extended with your own plug-ins which can provide custom 14 | launch logic for arbitrary custom applications. Visit the live site to see 15 | currently available applications in the Catalog. CloudLaunch is also tightly 16 | integrated with `CloudBridge `_, which makes 17 | CloudLaunch natively multi-cloud. 18 | 19 | CloudLaunch has a web and commandline front-end. The Web UI is maintained in the 20 | `CloudLaunch-UI `_ repository. 21 | The commandline client is maintained in the 22 | `cloudlaunch-cli `_ repository. 23 | 24 | Installation 25 | ------------ 26 | 27 | The recommended method for installing CloudLaunch is via the 28 | `CloudLaunch Helm chart `_. 29 | 30 | To install a development version, take a look at 31 | `development installation page `_. 32 | 33 | 34 | Application Configuration 35 | ------------------------- 36 | 37 | Once the application components are installed and running (regardless of the 38 | method utilized), it is necessary to load appliance and cloud provider 39 | connection properties. See `this page `_ for how to 40 | do this. 41 | 42 | Authentication Configuration 43 | ---------------------------- 44 | 45 | User authentication to CloudLaunch should be managed via social auth. For 46 | development purposes, it is possible to use Django authentication in which 47 | case simply creating a superuser is sufficient. If you intend on having users 48 | of your CloudLaunch installation, you will want to configure 49 | `social auth `_. 50 | 51 | 52 | Table of contents 53 | ----------------- 54 | .. toctree:: 55 | :maxdepth: 1 56 | 57 | topics/overview.rst 58 | topics/production_server_mgmt.rst 59 | topics/development_server_installation.rst 60 | topics/configuration.rst 61 | topics/social_auth.rst 62 | -------------------------------------------------------------------------------- /docs/topics/configuration.rst: -------------------------------------------------------------------------------- 1 | Configure CloudLaunch with data 2 | =============================== 3 | 4 | Once running, it is necessary to load the CloudLaunch database with information 5 | about the appliances available for launching as well as cloud providers 6 | where those appliances can be launched. 7 | 8 | The following commands show how to load the information that is available on 9 | the hosted CloudLaunch server available at https://launch.usegalaxy.org/. It 10 | is recommended to load those values and then edit them to fit your needs. 11 | 12 | Loading clouds 13 | -------------- 14 | 15 | Appliances define properties required to properly launch an application on a 16 | cloud provider. Run the following commands from the CloudLaunch server 17 | repository with the suitable Conda environment activated. 18 | 19 | .. code-block:: bash 20 | 21 | cd django-cloudlaunch 22 | curl https://raw.githubusercontent.com/CloudVE/cloudlaunch-helm/master/cloudlaunchserver/data/1_clouds.json --output clouds.json 23 | python manage.py loaddata clouds.json 24 | 25 | If we start the CloudLaunch server now and navigate to the admin console, 26 | ``DJCLOUDBRIDGE -> Clouds``, we can see a list of cloud providers that have 27 | been loaded and CloudLaunch can target. 28 | 29 | If you would like to add a new cloud provider to be included in either the 30 | hosted service or for distribution, please issue a pull request with the 31 | necessary connection properties to 32 | https://github.com/CloudVE/cloudlaunch-helm/blob/master/cloudlaunchserver/data/1_clouds.json 33 | 34 | 35 | Loading appliances 36 | ------------------ 37 | 38 | Rather than loading application-specific information by hand, we can load apps 39 | from an application registry in bulk. At the moment, this action needs to be 40 | performed from the CloudLaunch admin console. 41 | 42 | On the CloudLaunch admin console, head to the ``CloudLaunch -> Applications`` page 43 | and click ``Add application`` button in the top right corner. Provide an 44 | arbitrary name, say `placeholder`, for the application name and click save. Any 45 | information provided for this application will get overwritten with the 46 | information from the application registry. Back on the page listing 47 | applications, select the checkbox next to the newly created application and 48 | then from the ``Action`` menu, select ``Import app data from url``. Click 49 | ``Update`` on the next page to load the default set of applications and your 50 | installation of CloudLaunch will have loaded all currently available apps. 51 | -------------------------------------------------------------------------------- /docs/topics/development_server_installation.rst: -------------------------------------------------------------------------------- 1 | Installation for development 2 | ============================ 3 | 4 | CloudLaunch is made up of three services: the server, the user interface (UI), 5 | and a message queue. All three processes need to run for the application to 6 | function properly. See instructions below on how to install and start each 7 | of the processes. 8 | 9 | Install the server 10 | ------------------ 11 | 12 | CloudLaunch is based on Python 3.6 and although it may work on older Python 13 | versions, 3.6 is the only supported version. Use of Conda or virtualenv is also 14 | highly advised. 15 | 16 | 1. Checkout CloudLaunch and create an isolated environment 17 | 18 | .. code-block:: bash 19 | 20 | $ conda create --name cl --yes python=3.6 21 | $ conda activate cl 22 | $ git clone https://github.com/galaxyproject/cloudlaunch.git 23 | $ cd cloudlaunch 24 | $ pip install -r requirements_dev.txt 25 | $ cd django-cloudlaunch 26 | 27 | 2. Create a local copy of the settings file and make any desired configuration 28 | changes. No changes are required for CloudLaunch to run but it is advisable 29 | to at least update the value of the fernet key. 30 | 31 | .. code-block:: bash 32 | $ cp cloudlaunchserver/settings_local.py.sample cloudlaunchserver/settings_local.py 33 | 34 | 3. Run the migrations and create a superuser 35 | 36 | .. code-block:: bash 37 | 38 | $ python manage.py migrate 39 | $ python manage.py createsuperuser 40 | 41 | 4. Start the web server and Celery in separate tabs. If you do not have Redis 42 | installed, you can install it via Conda: ``conda install -c anaconda redis`` 43 | 44 | .. code-block:: bash 45 | 46 | $ python manage.py runserver 47 | $ redis-server & celery -A cloudlaunchserver worker -l info --beat 48 | 49 | 5. Visit http://127.0.0.1:8000/cloudlaunch/admin/ to define appliances and 50 | add cloud providers. 51 | 52 | 6. Visit http://127.0.0.1:8000/cloudlaunch/api/v1/ to explore the API. 53 | 54 | 55 | Install the UI 56 | -------------- 57 | 58 | 1. Clone the source code repository 59 | 60 | .. code-block:: bash 61 | 62 | $ git clone https://github.com/galaxyproject/cloudlaunch-ui.git 63 | $ cd cloudlaunch-ui 64 | 65 | 2. Install required libraries 66 | 67 | Make sure you have ``node`` (version 6.*) installed (eg, via Conda, 68 | ``conda install -c conda-forge nodejs``). Then install the dependencies. 69 | 70 | .. code-block:: bash 71 | 72 | # Install typescript development support 73 | npm install -g tsd 74 | # Install angular-cli 75 | npm install -g @angular/cli 76 | # Install dependencies 77 | npm install 78 | 79 | 3. Run the development server 80 | 81 | Start the development server with 82 | 83 | .. code-block:: bash 84 | 85 | npm start 86 | 87 | Or if you use yarn as your preferred package manager, ``yarn start``. 88 | 89 | Access the server at ``http://localhost:4200/``. The app will 90 | automatically reload if you change any of the source files. 91 | 92 | If you are installing this on a VM instead your local machine and need to 93 | access the UI over the network, instead of using ``npm start``, use 94 | ``ng serve --host 0.0.0.0 --disable-host-check --proxy-config proxy.conf.json`` 95 | The UI should be availale on the host IP address, port 4200. 96 | -------------------------------------------------------------------------------- /docs/topics/overview.rst: -------------------------------------------------------------------------------- 1 | Overview 2 | =============== 3 | This section provides a quick overview of the various CloudLaunch pages. 4 | 5 | Catalog 6 | ------- 7 | Use the catalog to search through the available appliances. A virtual appliance 8 | is a virtual machine that packages a ready-to-run application(s), eliminating 9 | the need to install and configure complex stacks of software. (e.g. Galaxy, 10 | Genomics Virtual Lab, SLURM). 11 | 12 | Public Appliances 13 | ----------------- 14 | Public Appliances are appliances which have been made publicly available by 15 | an organization or individual. Although these applications are publicly 16 | available, they may require registration or impose usage quotas. You can 17 | contact us if you would like to list an applainces as public. 18 | 19 | My Appliances 20 | ------------- 21 | My Appliances lists all currently actie appliances. You can use this page to 22 | monitor the state of an appliances or to delete an appliance. You can also 23 | archive an appliance, in which case it will be moved to the Launch History 24 | page. 25 | 26 | User Profile 27 | ------------ 28 | Use this section to manage your user profile. You can use this page to save 29 | or edit credentials for various clouds. 30 | -------------------------------------------------------------------------------- /docs/topics/production_server_mgmt.rst: -------------------------------------------------------------------------------- 1 | Installing a production server 2 | ============================== 3 | 4 | Upgrading running chart 5 | ----------------------- 6 | 7 | 1. Fetch latest chart version through 8 | :code:`helm repo update` 9 | 10 | 11 | 2. Docker pull the latest image 12 | 13 | .. code-block:: bash 14 | 15 | sudo docker pull cloudve/cloudlaunch-server:latest 16 | sudo docker pull cloudve/cloudlaunch-ui:latest 17 | 18 | 19 | 3. Upgrade then helm chart 20 | 21 | .. code-block:: bash 22 | 23 | helm upgrade --reuse-values galaxyproject/cloudlaunch 24 | 25 | 26 | Reinstalling chart from scratch 27 | ------------------------------- 28 | 29 | 0. Note down the existing secrets for fernet keys, secret keys, db password etc. through kubernetes in the cloudlaunch namespace. 30 | Dashboard access link: https://149.165.157.211:4430/k8s/clusters/c-nmrvs/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy/#!/login 31 | 32 | To obtain login token: https://gist.github.com/superseb/3a9c0d2e4a60afa3689badb1297e2a44 33 | 34 | .. code-block:: bash 35 | 36 | kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep admin-user | awk '{print $1}') 37 | 38 | 1. :code:`helm delete ` 39 | 40 | 2. :code:`kubectl delete namespace cloudlaunch` 41 | 42 | 3. Optionally, delete all cached docker images using 43 | 44 | .. code-block:: bash 45 | 46 | docker images 47 | docker rmi 48 | 49 | 4. Delete existing persistent volume in rancher. This does not delete the local folder, so the database will survive. Recreate with following settings: 50 | 51 | .. code-block:: bash 52 | 53 | Name: cloudlaunch-database 54 | Capacity: 30 55 | Volume Plugin: Local Node Path 56 | Path on the node: /opt/cloudlaunch/database 57 | Path on node: A directory, or create 58 | Customize -> Many nodes read write 59 | 60 | 61 | 5. :code:`helm install galaxyproject/cloudlaunch --set cloudlaunch-server.postgresql.postgresqlPassword= --namespace cloudlaunch --set cloudlaunch-server.fernet_keys[0]='' --set cloudlaunch-server.fernet_keys[1]='' --set cloudlaunch-server.secret_key=` 62 | 63 | 64 | -------------------------------------------------------------------------------- /docs/topics/social_auth.rst: -------------------------------------------------------------------------------- 1 | Social Auth Setup 2 | ----------------- 3 | 4 | After you have setup the server, you will probably want to setup social 5 | auth to be able to log in using an external service. This setup is required 6 | for end-users so they can self register. If you are setting this up on 7 | localhost, use GitHub or Twitter. 8 | 9 | Integration with GitHub 10 | ~~~~~~~~~~~~~~~~~~~~~~~ 11 | 12 | 1. Register your server with GitHub: Visit your Github account Settings → 13 | `Developer settings `_ and add a new 14 | OAuth application. Settings should look as in the following screenshot. Note 15 | port 4200 on the *Authorization callback URL*; this needs to match the port on 16 | which the CloudLaunch UI is served (4200 is the default). Also take note of the 17 | *Client ID* and *Client Secret* at the top of that page as we'll need that back 18 | in CloudLaunch. 19 | 20 | .. image:: ../images/github-ouath-app-sm.png 21 | :target: ../images/github-oauth-app.png 22 | 23 | 2. Back on the local server, login to Django admin and change the domain of 24 | example.com in Sites to ``http://127.0.0.1:8080``. To login to Admin, you'll 25 | need the superuser account info that was created when setting up the server. 26 | 27 | 3. Still in Django Admin, now navigate to *Social Accounts → Social 28 | applications* and add a new application. Select GitHub as the provider, supply a 29 | desired application name, and enter the *Client ID* and *Client Secret* we got 30 | from GitHub. Also choose the site we updated in Step 2. 31 | 32 | .. image:: ../images/add-social-app-sm.png 33 | :target: ../images/add-social-app.png 34 | 35 | Save the model and integration with GitHub is complete! You can now log in to 36 | the CloudLaunch UI using Github. 37 | 38 | 39 | Integration with Twitter 40 | ~~~~~~~~~~~~~~~~~~~~~~~~ 41 | 42 | 1. Register your dev server under your Twitter account. Visit 43 | https://apps.twitter.com/, click *Create New App*, and fill out the form as in 44 | the following screenthot. Once the app has been added, click on the *Keys and 45 | Access Tokens* tab and take a note of *Consumer Key (API Key)* and *Consumer 46 | Secret (API Secret)*. 47 | 48 | .. image:: ../images/twitter-oauth-app-sm.png 49 | :target: ../images/twitter-oauth-app.png 50 | 51 | 2. Proceed with the same steps as in the docs about about GitHub integration, 52 | supplying the *Consumer Key (API Key)* and *Consumer Secret (API Secret)* as the 53 | values of *Client ID* and *Client Secret* for the new defintion of the Social 54 | application. 55 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # install edge till this is released: https://github.com/encode/django-rest-framework/pull/7571 2 | git+https://github.com/encode/django-rest-framework 3 | # install edge till this is released: https://github.com/celery/django-celery-results/issues/157 4 | git+https://github.com/celery/django-celery-results 5 | git+https://github.com/CloudVE/cloudbridge#egg=cloudbridge[full] 6 | git+https://github.com/CloudVE/djcloudbridge#egg=djcloudbridge 7 | -e ".[prod]" 8 | -------------------------------------------------------------------------------- /requirements_dev.txt: -------------------------------------------------------------------------------- 1 | # install edge till this is released: https://github.com/encode/django-rest-framework/pull/7571 2 | git+https://github.com/encode/django-rest-framework 3 | # install edge till this is released: https://github.com/celery/django-celery-results/issues/157 4 | git+https://github.com/celery/django-celery-results 5 | git+git://github.com/CloudVE/cloudbridge#egg=cloudbridge[dev] 6 | git+git://github.com/CloudVE/djcloudbridge#egg=djcloudbridge[dev] 7 | -e ".[dev]" 8 | -------------------------------------------------------------------------------- /requirements_test.txt: -------------------------------------------------------------------------------- 1 | # install edge till this is released: https://github.com/encode/django-rest-framework/pull/7571 2 | git+https://github.com/encode/django-rest-framework 3 | # install edge till this is released: https://github.com/celery/django-celery-results/issues/157 4 | git+https://github.com/celery/django-celery-results 5 | # needed by moto 6 | sshpubkeys 7 | git+https://github.com/CloudVE/moto@fix_unknown_instance_type 8 | git+git://github.com/CloudVE/cloudlaunch-cli#egg=cloudlaunch-cli 9 | git+git://github.com/CloudVE/cloudbridge#egg=cloudbridge[dev] 10 | git+git://github.com/CloudVE/djcloudbridge#egg=djcloudbridge[dev] 11 | -e ".[test]" 12 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 4.0.0+dev0 3 | tag = False 4 | commit = True 5 | parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+)(?P\d+))? 6 | serialize = 7 | {major}.{minor}.{patch}+{release}{build} 8 | {major}.{minor}.{patch} 9 | 10 | [bumpversion:part:release] 11 | optional_value = prod 12 | first_value = dev 13 | values = 14 | dev 15 | prod 16 | 17 | [bumpversion:file:django-cloudlaunch/cloudlaunchserver/__init__.py] 18 | 19 | [bumpversion:file:docs/conf.py] 20 | search = {current_version} 21 | replace = {new_version} 22 | 23 | [metadata] 24 | description-file = README.rst 25 | 26 | [flake8] 27 | exclude = 28 | build, docs, dist, 29 | */migrations, 30 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | import os 4 | import re 5 | import sys 6 | 7 | from setuptools import setup 8 | from setuptools import find_packages 9 | 10 | 11 | def get_version(*file_paths): 12 | """Retrieves the version from django-cloudlaunch/__init__.py""" 13 | filename = os.path.join(os.path.dirname(__file__), *file_paths) 14 | version_file = open(filename).read() 15 | version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", 16 | version_file, re.M) 17 | if version_match: 18 | return version_match.group(1) 19 | raise RuntimeError('Unable to find version string.') 20 | 21 | 22 | version = get_version("django-cloudlaunch", "cloudlaunchserver", "__init__.py") 23 | 24 | 25 | if sys.argv[-1] == 'publish': 26 | try: 27 | import wheel 28 | print("Wheel version: ", wheel.__version__) 29 | except ImportError: 30 | print('Wheel library missing. Please run "pip install wheel"') 31 | sys.exit() 32 | os.system('python setup.py sdist upload') 33 | os.system('python setup.py bdist_wheel upload') 34 | sys.exit() 35 | 36 | if sys.argv[-1] == 'tag': 37 | print("Tagging the version on git:") 38 | os.system("git tag -a %s -m 'version %s'" % (version, version)) 39 | os.system("git push --tags") 40 | sys.exit() 41 | 42 | readme = open('README.rst').read() 43 | history = open('HISTORY.rst').read().replace('.. :changelog:', '') 44 | 45 | REQS_BASE = [ 46 | 'Django>=3.0', 47 | # ======== Celery ========= 48 | 'celery>=5.0', 49 | # celery results backend which uses the django DB 50 | 'django-celery-results>=1.0.1', 51 | # celery background task monitor which uses the django DB 52 | 'django-celery-beat>=1.3.0', 53 | # ======== DRF ========= 54 | 'djangorestframework>=3.7.3', 55 | # login support for DRF through restful endpoints 56 | 'dj-rest-auth', 57 | # pluggable social auth for django login 58 | 'django-allauth>=0.34.0', 59 | # Provides nested routing for DRF 60 | 'drf-nested-routers>=0.90.0', 61 | # For DRF filtering by querystring 62 | 'django-filter>=1.1.0', 63 | # Provides REST API schema 64 | 'coreapi>=2.2.3', 65 | # ======== CloudBridge ========= 66 | 'cloudbridge[full]', 67 | 'djcloudbridge', 68 | # ======== Django ========= 69 | # Provides better inheritance support for django models 70 | 'django-model-utils', 71 | # for encryption of user credentials 72 | 'djfernet', 73 | # Middleware for automatically adding CORS headers to responses 74 | 'django-cors-headers>=2.1.0', 75 | # for nested object editing in django admin 76 | 'django-nested-admin>=3.0.21', 77 | # For dependencies between key fields in django admin 78 | 'django-autocomplete-light>=3.3.2', 79 | # ======== Public Appliances ========= 80 | # Used by public_appliances for retrieving country data 81 | 'django-countries>=5.0', 82 | # ======== Misc ========= 83 | # For the CloudMan launcher 84 | 'bioblend', 85 | # For merging userdata/config dictionaries 86 | 'jsonmerge>=1.4.0', 87 | # For commandline option handling 88 | 'click', 89 | # Integration with Sentry 90 | 'sentry-sdk==0.6.9', 91 | # For CloudMan2 plugin 92 | 'gitpython', 93 | 'ansible<2.10', # pin ansible due to: https://github.com/ansible/ansible/issues/68399 94 | 'netaddr', 95 | # Utility package for retrying operations 96 | 'tenacity', 97 | # For serving static files in production mode 98 | 'whitenoise[brotli]', 99 | 'paramiko' 100 | ] 101 | 102 | REQS_PROD = ([ 103 | # postgres database driver 104 | 'psycopg2-binary', 105 | 'gunicorn[gevent]'] + REQS_BASE 106 | ) 107 | 108 | REQS_TEST = ([ 109 | 'pydevd', 110 | 'sqlalchemy', # for celery results backend 111 | 'tox>=2.9.1', 112 | 'coverage>=4.4.1', 113 | 'flake8>=3.4.1', 114 | 'flake8-import-order>=0.13'] + REQS_BASE 115 | ) 116 | 117 | REQS_DEV = ([ 118 | # As celery message broker during development 119 | 'redis', 120 | 'sphinx>=1.3.1', 121 | 'sphinx_rtd_theme', 122 | 'bump2version', 123 | 'pylint-django'] + REQS_TEST 124 | ) 125 | 126 | setup( 127 | name='cloudlaunch-server', 128 | version=version, 129 | description=("CloudLaunch is a ReSTful, extensible Django app for" 130 | " discovering and launching applications on cloud, container," 131 | " or local infrastructure"), 132 | long_description=readme + '\n\n' + history, 133 | author='Galaxy Project', 134 | author_email='help@cloudve.org', 135 | url='https://github.com/galaxyproject/cloudlaunch', 136 | package_dir={'': 'django-cloudlaunch'}, 137 | packages=find_packages('django-cloudlaunch'), 138 | package_data={ 139 | 'cloudlaunch': [ 140 | 'backend_plugins/cloudman2/rancher2_aws_iam_policy.json', 141 | 'backend_plugins/cloudman2/rancher2_aws_iam_trust_policy.json'], 142 | }, 143 | include_package_data=True, 144 | install_requires=REQS_BASE, 145 | extras_require={ 146 | 'dev': REQS_DEV, 147 | 'test': REQS_TEST, 148 | 'prod': REQS_PROD 149 | }, 150 | entry_points={ 151 | 'console_scripts': [ 152 | 'cloudlaunch-server = cloudlaunchserver.runner:main'] 153 | }, 154 | license="MIT", 155 | keywords='cloudlaunch', 156 | classifiers=[ 157 | 'Development Status :: 5 - Production/Stable', 158 | 'Framework :: Django', 159 | 'Framework :: Django :: 2.0', 160 | 'Intended Audience :: Developers', 161 | 'License :: OSI Approved :: BSD License', 162 | 'Natural Language :: English', 163 | 'Programming Language :: Python :: 3.6', 164 | 'Topic :: Internet :: WWW/HTTP', 165 | 'Topic :: Internet :: WWW/HTTP :: WSGI :: Application' 166 | ], 167 | ) 168 | -------------------------------------------------------------------------------- /tests/run_cli_integration_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export DJANGO_SETTINGS_MODULE="${DJANGO_SETTINGS_MODULE:-cloudlaunchserver.settings_test}" 4 | export CELERY_CONFIG_MODULE="${CELERY_CONFIG_MODULE:-cloudlaunchserver.celeryconfig_test}" 5 | export CLOUDLAUNCH_SERVER_URL=http://localhost:8000/cloudlaunch/api/v1 6 | export CLOUDLAUNCH_AUTH_TOKEN=272f075f152e59fd5ea55ca2d21728d2bfe37077 7 | 8 | # Change working directory so everything is resolved relative to cloudlaunch root folder 9 | SCRIPT_DIR=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P ) 10 | cd $SCRIPT_DIR/.. 11 | 12 | # Delete the existing database 13 | rm -f /tmp/cloudlaunch_testdb.sqlite3 14 | 15 | # Initialize database 16 | python django-cloudlaunch/manage.py migrate 17 | 18 | # Load initial test data 19 | python django-cloudlaunch/manage.py loaddata tests/fixtures/initial_test_data.json 20 | 21 | # Run cloudlaunch in background. Use noreload so that it runs in the same process as coverage 22 | coverage run --source django-cloudlaunch --branch django-cloudlaunch/manage.py runserver --noreload & 23 | 24 | # Wait for cloudlaunch to start 25 | TIMEOUT=100 26 | echo "Waiting for cloudlaunch to start..." 27 | while ! nc -z localhost 8000; do 28 | if [[ $TIMEOUT -lt 0 ]]; then 29 | echo "Timeout waiting for cloudlaunch to start" 30 | exit 124 31 | fi 32 | sleep 0.1 33 | TIMEOUT=$((TIMEOUT-1)) 34 | done 35 | 36 | # Clone temp cloudlaunch-cli repo 37 | git clone https://github.com/CloudVE/cloudlaunch-cli /tmp/cloudlaunch-cli 38 | 39 | # Run cloudlaunch-cli test suite against cloudlaunch 40 | cd /tmp/cloudlaunch-cli/ && python setup.py test 41 | # Cache return value of tests 42 | ret_value=$? 43 | 44 | # Kill the django process afterwards ($! is the last background process). 45 | # There's a special SIGINT handler in manage.py that will terminate cloudlaunch 46 | # gracefully, so coverage has a chance to write out its report 47 | kill -SIGINT $! 48 | 49 | exit $ret_value -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | # Tox (http://tox.testrun.org/) is a tool for running tests 2 | # in multiple virtualenvs. This configuration file will run the 3 | # test suite on all supported python versions. To use it, "pip install tox" 4 | # and then run "tox" from this directory. 5 | 6 | [tox] 7 | envlist = py38,cli_integration 8 | skipsdist = True 9 | usedevelop = True 10 | 11 | [testenv:cli_integration] 12 | commands = bash tests/run_cli_integration_tests.sh 13 | allowlist_externals = bash 14 | passenv = 15 | SENTRY_DSN 16 | deps = 17 | -rrequirements_test.txt 18 | coverage 19 | 20 | [testenv] 21 | commands = {envpython} -m coverage run --source django-cloudlaunch --branch django-cloudlaunch/manage.py test django-cloudlaunch 22 | setenv = 23 | CELERY_CONFIG_MODULE=cloudlaunchserver.celeryconfig_test 24 | DJANGO_SETTINGS_MODULE=cloudlaunchserver.settings_test 25 | # Fix for import issue: https://github.com/travis-ci/travis-ci/issues/7940 26 | BOTO_CONFIG=/dev/null 27 | passenv = 28 | SENTRY_DSN 29 | deps = 30 | -rrequirements_test.txt 31 | coverage 32 | --------------------------------------------------------------------------------