├── .gitignore ├── .hgignore ├── .hgtags ├── AUTHORS.txt ├── INSTALL.txt ├── LICENSE.txt ├── MANIFEST.in ├── README.txt ├── bin └── woven-admin.py ├── docs ├── changelog.rst ├── commands.rst ├── conf.py ├── conventions.rst ├── index.rst ├── main.rst ├── settings.rst ├── troubleshooting.rst ├── tutorial.rst └── woven │ ├── deployment.rst │ ├── linux.rst │ ├── main.rst │ ├── project.rst │ ├── virtualenv.rst │ ├── webservers.rst │ └── woven.rst ├── setup.cfg ├── setup.py ├── tests ├── __init__.py ├── dec.py ├── dep.py ├── env.py ├── fabfile.py ├── lin.py ├── vir.py └── web.py └── woven ├── __init__.py ├── api.py ├── decorators.py ├── deploy.py ├── deployment.py ├── environment.py ├── linux.py ├── management ├── __init__.py ├── base.py └── commands │ ├── __init__.py │ ├── activate.py │ ├── bundle.py │ ├── deploy.py │ ├── node.py │ ├── patch.py │ ├── setupnode.py │ └── startsites.py ├── project.py ├── templates ├── distribution_template │ ├── project_name │ │ ├── __init__.py │ │ ├── manage.py │ │ ├── settings.py │ │ └── urls.py │ └── setup.py ├── setup.py └── woven │ ├── django-apache-template.txt │ ├── django-wsgi-template.txt │ ├── etc │ ├── apache2 │ │ └── ports.conf │ ├── init.d │ │ └── nginx │ ├── nginx │ │ ├── nginx.conf │ │ ├── proxy.conf │ │ └── sites-available │ │ │ └── default │ └── postgresql │ │ └── 8.4 │ │ └── main │ │ ├── pg_hba.conf │ │ └── postgresql.conf │ ├── gunicorn.conf │ ├── maintenance.html │ ├── nginx-gunicorn-template.txt │ ├── nginx-template.txt │ ├── requirements.txt │ ├── sitesettings.txt │ ├── ssh │ └── sshd_config │ └── ufw.txt ├── virtualenv.py └── webservers.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.DS_Store 3 | *.komodoproject 4 | *.Python 5 | *.db -------------------------------------------------------------------------------- /.hgignore: -------------------------------------------------------------------------------- 1 | syntax: glob 2 | MANIFEST 3 | *.pyc 4 | *.kpf 5 | .DS_Store 6 | *.gz 7 | *.egg 8 | build/* 9 | dist/* 10 | *.pybundle 11 | *.db 12 | *.tmproj 13 | html/* 14 | tests/example.com/* 15 | *.egg-info/* 16 | docs/Makefile 17 | *.zip 18 | _build/* 19 | -------------------------------------------------------------------------------- /.hgtags: -------------------------------------------------------------------------------- 1 | a1d6ba9c4305e3fb1ef1929d4a1af74ecaff1667 v0.1 2 | 8c8afdcbf9bdaddecc4097e3b9fe9158e446ac69 v0.2 3 | d4c7c666441790f8267fc5b0babe89276549607b v0.2.1 4 | b4e007ac36ad9500f722c2598ce5d3c9dd0f4d44 0.3 5 | 396ca385ef8c2bbbb12a6997d19238ff9649967d 0.3.1 6 | 17a2d0571ff792de86fa692d7dbca16ae1e0673c 0.4 7 | deb8958c422bef8ddad127c7bc432de9066be7e8 0.5 8 | 5ee93f47204e82ee3531e29b3c45bc92a119c09d 0.5.1 9 | 9e187c93756e50dc93254f0855f90ca02e14cc53 0.5.2 10 | 9e187c93756e50dc93254f0855f90ca02e14cc53 0.5.2 11 | 2a228f364616850d7010eda06d1c795602e56719 0.5.2 12 | 7e2abdf871a130672b74f960b74e42e6c18dddf4 0.5.3 13 | -------------------------------------------------------------------------------- /AUTHORS.txt: -------------------------------------------------------------------------------- 1 | The primary author of woven is Brett Haydon 2 | Other contributors to woven include: 3 | Wil Tan 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /INSTALL.txt: -------------------------------------------------------------------------------- 1 | You are using pip right? 2 | 3 | ``pip install woven`` 4 | 5 | You may also want to install the following recommended packages: 6 | 7 | ``pip install south`` 8 | 9 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2010, Brett Haydon 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are 6 | met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | * Redistributions in binary form must reproduce the above 11 | copyright notice, this list of conditions and the following 12 | disclaimer in the documentation and/or other materials provided 13 | with the distribution. 14 | * Neither the name of the author nor the names of other 15 | contributors may be used to endorse or promote products derived 16 | from this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include INSTALL.txt 2 | include LICENSE.txt 3 | include MANIFEST.in 4 | include README.txt 5 | include AUTHORS.txt 6 | recursive-include woven/templates * 7 | recursive-include docs *.py *.rst Makefile 8 | global-exclude *.pyc .DS_Store 9 | prune tests 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /README.txt: -------------------------------------------------------------------------------- 1 | 2 | Woven deploys versioned Django projects onto Linux servers behind Nginx using Apache modwsgi by default. Woven is is built on `Fabric `_. 3 | 4 | :doc:`RTFM.` 5 | 6 | .. note:: 7 | 8 | Woven is still alpha software, and the api and conventions may change between versions. It is not recommended for production deployment at this stage. It has been tested (in the most loose sense of the term) using Ubuntu Server 10.04 hosts and greater. It may work on other debian based distributions (reports welcome). It currently *won't* work at all with Windows, due to use of rsync, and needs some work to be compatible with redhat based distributions. 9 | 10 | -------------------------------------------------------------------------------- /bin/woven-admin.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys, os 3 | from distutils.core import run_setup 4 | from random import choice 5 | import re 6 | import optparse 7 | 8 | from django.core.management import execute_from_command_line, call_command 9 | from django.core.management.base import CommandError, _make_writeable 10 | from django.utils.importlib import import_module 11 | 12 | from fabric.contrib.console import confirm, prompt 13 | from fabric.api import settings 14 | from fabric.state import env 15 | from fabric.main import find_fabfile 16 | 17 | import woven 18 | 19 | TEMPLATE_DIR = os.path.join(os.path.dirname(os.path.realpath(woven.__file__)) 20 | ,'templates','distribution_template') 21 | 22 | def copy_helper(app_or_project, name, directory, dist, template_dir, noadmin): 23 | """ 24 | 25 | Replacement for django copy_helper 26 | Copies a Django project layout template into the specified distribution directory 27 | 28 | """ 29 | 30 | import shutil 31 | if not re.search(r'^[_a-zA-Z]\w*$', name): # If it's not a valid directory name. 32 | # Provide a smart error message, depending on the error. 33 | if not re.search(r'^[_a-zA-Z]', name): 34 | message = 'make sure the name begins with a letter or underscore' 35 | else: 36 | message = 'use only numbers, letters and underscores' 37 | raise CommandError("%r is not a valid project name. Please %s." % (name, message)) 38 | top_dir = os.path.join(directory, dist) 39 | try: 40 | os.mkdir(top_dir) 41 | except OSError, e: 42 | raise CommandError(e) 43 | 44 | for d, subdirs, files in os.walk(template_dir): 45 | relative_dir = d[len(template_dir)+1:].replace('project_name', name) 46 | if relative_dir: 47 | os.mkdir(os.path.join(top_dir, relative_dir)) 48 | for subdir in subdirs[:]: 49 | if subdir.startswith('.'): 50 | subdirs.remove(subdir) 51 | for f in files: 52 | if not f.endswith('.py'): 53 | # Ignore .pyc, .pyo, .py.class etc, as they cause various 54 | # breakages. 55 | continue 56 | path_old = os.path.join(d, f) 57 | path_new = os.path.join(top_dir, relative_dir, f.replace('project_name', name)) 58 | fp_old = open(path_old, 'r') 59 | fp_new = open(path_new, 'w') 60 | if noadmin: 61 | fp_new.write(fp_old.read().replace('{{ project_name }}', name)) 62 | else: 63 | fp_new.write(fp_old.read().replace('{{ project_name }}', name).replace('## ','')) 64 | fp_old.close() 65 | fp_new.close() 66 | try: 67 | shutil.copymode(path_old, path_new) 68 | _make_writeable(path_new) 69 | except OSError: 70 | sys.stderr.write(style.NOTICE("Notice: Couldn't set permission bits on %s. You're probably using an uncommon filesystem setup. No problem.\n" % path_new)) 71 | 72 | 73 | def start_distribution(project_name, template_dir, dist, noadmin): 74 | """ 75 | Custom startproject command to override django default 76 | """ 77 | 78 | directory = os.getcwd() 79 | 80 | # Check that the project_name cannot be imported. 81 | try: 82 | import_module(project_name) 83 | except ImportError: 84 | pass 85 | else: 86 | raise CommandError("%r conflicts with the name of an existing Python module and cannot be used as a project name. Please try another name." % project_name) 87 | #woven override 88 | copy_helper('project', project_name, directory, dist, template_dir, noadmin) 89 | 90 | #Create a random SECRET_KEY hash, and put it in the main settings. 91 | main_settings_file = os.path.join(directory, dist, project_name, 'settings.py') 92 | settings_contents = open(main_settings_file, 'r').read() 93 | fp = open(main_settings_file, 'w') 94 | secret_key = ''.join([choice('abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)') for i in range(50)]) 95 | settings_contents = re.sub(r"(?<=SECRET_KEY = ')'", secret_key + "'", settings_contents) 96 | fp.write(settings_contents) 97 | fp.close() 98 | 99 | #import settings and create start directories 100 | sys.path.append(os.path.join(directory, dist)) 101 | 102 | s = import_module('.'.join([project_name,'settings'])) 103 | sys.path.pop() 104 | if s.DATABASES['default']['ENGINE']=='django.db.backends.sqlite3': 105 | if s.DATABASES['default']['NAME'] and not os.path.exists(s.DATABASES['default']['NAME']): 106 | os.mkdir(os.path.dirname(s.DATABASES['default']['NAME'])) 107 | if s.STATIC_ROOT and os.path.isabs(s.STATIC_ROOT) and not os.path.exists(s.STATIC_ROOT): 108 | os.mkdir(s.STATIC_ROOT) 109 | if s.MEDIA_ROOT and os.path.isabs(s.MEDIA_ROOT) and not os.path.exists(s.MEDIA_ROOT): 110 | os.mkdir(s.MEDIA_ROOT) 111 | if s.TEMPLATE_DIRS: 112 | for t in s.TEMPLATE_DIRS: 113 | if not os.path.exists(t) and os.path.sep in t: 114 | os.mkdir(t) 115 | 116 | 117 | if __name__ == "__main__": 118 | #Inject woven into the settings only if it is a woven command 119 | settings_mod = None 120 | inject = False 121 | startproject = False 122 | orig_cwd = os.getcwd() 123 | 124 | for arg in sys.argv: 125 | if '--settings' in arg: 126 | settings_mod = arg.split('=')[1].strip() 127 | elif arg in ['activate','deploy','startsites','setupnode','node','bundle','patch', 'validate']: 128 | inject = True 129 | elif arg == 'startproject': 130 | #call woven startproject in place of django startproject 131 | startproject = True 132 | inject = True 133 | parser = optparse.OptionParser(usage="usage: %prog startproject [project_name] [username@domain] [options]\n\n" 134 | "project_name is the name of your django project\n" 135 | "username@domain is an optional email address to setup a superuser") 136 | parser.add_option('-t', '--template-dir', dest='src_dir', 137 | help='project template directory to use', 138 | default=TEMPLATE_DIR) 139 | parser.add_option('-d','--dist', dest='dist_name', 140 | help="alternative distribution name", 141 | default='') 142 | parser.add_option('--noadmin', 143 | action='store_true', 144 | default=False, 145 | help="admin disabled", 146 | ), 147 | parser.add_option('--nosyncdb', 148 | action='store_true', 149 | default=False, 150 | help="Does not syncdb", 151 | ) 152 | options, args = parser.parse_args() 153 | if len(args) not in (2, 3, 4): 154 | parser.print_help() 155 | sys.exit(1) 156 | if not options.dist_name: 157 | dist = args[1] 158 | else: 159 | dist = options.dist_name 160 | project_name = args[1] 161 | try: 162 | email = args[2] 163 | except IndexError: 164 | email = '' 165 | 166 | start_distribution(project_name,options.src_dir, dist, noadmin = options.noadmin) 167 | 168 | #get the name of the settings from setup.py if DJANGO_SETTINGS_MODULE is not set 169 | if not os.environ.get('DJANGO_SETTINGS_MODULE') and not settings_mod: 170 | if startproject: 171 | os.chdir(os.path.join(orig_cwd,dist)) 172 | elif not 'setup.py' in os.listdir(os.getcwd()): 173 | #switch the working directory to the distribution root where setup.py is 174 | with settings(fabfile='setup.py'): 175 | env.setup_path = find_fabfile() 176 | if not env.setup_path: 177 | print 'Error: You must have a setup.py file in the current or a parent folder' 178 | sys.exit(1) 179 | local_working_dir = os.path.split(env.setup_path)[0] 180 | os.chdir(local_working_dir) 181 | 182 | woven_admin = sys.argv[0] 183 | setup = run_setup('setup.py',stop_after="init") 184 | settings_mod = '.'.join([setup.packages[0],'settings']) 185 | os.environ['DJANGO_SETTINGS_MODULE'] = settings_mod 186 | sys.argv.remove('setup.py') 187 | sys.argv.insert(0, woven_admin) 188 | 189 | 190 | if inject: 191 | if settings_mod: 192 | os.environ['DJANGO_SETTINGS_MODULE'] = settings_mod 193 | 194 | from django.conf import settings 195 | settings.INSTALLED_APPS += ('woven',) 196 | 197 | #switch to the settings module directory 198 | proj = settings_mod.split('.')[0] 199 | proj_mod = import_module(proj) 200 | if not proj_mod: 201 | sys.exit(0) 202 | moddir = os.path.dirname(proj_mod.__file__) 203 | os.chdir(moddir) 204 | 205 | 206 | if startproject: 207 | if not options.nosyncdb: 208 | call_command('syncdb',interactive=False) 209 | if 'django.contrib.auth' in settings.INSTALLED_APPS: 210 | if '@' in email: 211 | u = email.split('@')[0] 212 | else: 213 | u = project_name 214 | email = '%s@example.com'% project_name 215 | print "\nA superuser will be created with '%s' as username and password"% u 216 | print "Alternatively you can run the standard createsuperuser command separately" 217 | csuper = confirm('Would you like to create a superuser now?',default=True) 218 | if csuper: 219 | from django.contrib.auth.models import User 220 | 221 | User.objects.create_superuser(username=u, email=email, password=u) 222 | print "\nA default superuser was created:" 223 | print "Username:", u 224 | print "Password:", u 225 | print "Email:", email 226 | print "Change your password with 'woven-admin.py changepassword %s'"% u 227 | else: 228 | print "\nNo superuser created. " 229 | print "Run 'woven-admin.py createsuperuser' to create one" 230 | 231 | #run command as per django-admin.py 232 | else: 233 | #switch back to the original directory just in case some command needs it 234 | os.chdir(orig_cwd) 235 | execute_from_command_line() 236 | 237 | -------------------------------------------------------------------------------- /docs/changelog.rst: -------------------------------------------------------------------------------- 1 | 2 | Changelog 3 | ========== 4 | 5 | Release 0.8.0 (19-Dec-2010) 6 | --------------------------- 7 | 8 | ** Changes from 0.7** 9 | 10 | * A woven-admin.py script is now the preferred way to run django commands. Woven no longer needs to be in the INSTALLED_APPS setting, and woven no longer installs fabric or python-paramiko on the node. 11 | 12 | * Removed the DJANGO_REQUIREMENT setting. Django now gets added to the requirement file instead. 13 | 14 | **New Features** 15 | 16 | * The woven-admin.py script replaces the need to use the django-admin.py or project manage.py script. It can automatically pick up the settings from the setup.py file or be overriden with the usual --settings option. It injects itself into the INSTALLED_APPS for the execution of woven django commands so that woven will 'appear' to be installed. 17 | 18 | * A custom startproject command creates a much more complete project layout with distribution parent folder, setup.py and some sensible default settings to get started quickly. It also has the option of using your own template directory to source a starting layout. 19 | 20 | Release 0.7.0 (8-Dec-2010) 21 | -------------------------- 22 | 23 | **Changes from 0.6** 24 | 25 | * Multi-site functionality has changed dramatically. Instead of using one wsgi file and settings file per domain, woven now uses dynamic SITE_IDs in a single settings file based on the OS user, and process groups in modwsgi. Subdomains with the same SITE_ID can also be catered for by prefixing the settings filename. For example, if the SITE_ID=1 matches example.com, a settings file for the subdomain admin.example.com on the same SITE_ID would be admin_settings.py 26 | 27 | * The ubuntu module has become linux. Woven may be compatible with other debian based distributions, and it would be nice to add support for redhat if someone wants to. I don't imagine there is too much difference beyond apt/yum (and ufw) between lsb distros for the purposes of using woven. 28 | 29 | * run_once_per_host_version has changed name to run_once_per_version and another decorator has been added run_once_per_node. Both have moved to new decorators module. 30 | 31 | **New Features** 32 | 33 | * Implemented hooks. Can define a deploy.py with post_setupnode, post_install_package, or post_deploy functions. The deploy.py can be at project level or app level. 34 | * Added a DISABLE_APACHE_MODULES setting to disable a number of unneeded modules on setup. 35 | * Can add Personal Package Archives (PPA's) to a LINUX_PACKAGE_REPOSITORIES setting 36 | * Basic support and template for gunicorn added, and auto adds a PPA for gunicorn 37 | * Backups of configuration files don't pollute the package directories anymore. 38 | * Auto-uninstalls packages that are no longer listed in settings 39 | * Auto-uninstalls or doesn't install Apache modwsgi if you use gunicorn. 40 | * Only uploads templates if they have changed or if any packages are installed 41 | * only installs postgres/mysql db adaptors if your project uses them. 42 | 43 | **Bug fixes** 44 | 45 | * fix per role/host firewall rules and package installation 46 | * lots of other small improvements.. 47 | 48 | 49 | 50 | Release 0.6.1 (26-Nov-2010) 51 | --------------------------- 52 | 53 | * Fix UFW rules 54 | * Don't try and restart webservers if they don't exist. 55 | * Fixed problem with upload_etc logic 56 | * added very basic templates for postgresql 57 | * don't upload local_settings.py 58 | 59 | Release 0.6 (22-Nov-2010) 60 | ------------------------- 61 | 62 | **Changes from 0.5** 63 | 64 | * Bundles are now just .zip files instead of .pybundles. Just rename any existing .pybundles or re-run bundle 65 | * ROOT_DISABLED setting has been named DISABLE_ROOT instead. 66 | * changed the name of the function deploy_public to deploy_media 67 | 68 | **New Features** 69 | 70 | * added new setting PROJECT_APPS_PATH for local apps, so that you can put an apps folder under your project folder. 71 | * can now name your distribution differently from your project name 72 | * --overwrite option in deploy to completely remove an existing virtualenv 73 | * handles simple multi database, multi site migrations 74 | * added new setting SSH_KEY_FILENAME which maps to fabric's KEY_FILENAME 75 | * default webconf templates now listen on any ip (not fixed to one) 76 | * can set ufw port rules in a template `woven_project` 77 | * added ENABLE_UFW setting - defaults to true 78 | 79 | **Bug fixes** 80 | 81 | * fix an issue where you had to name your database after your project 82 | * fix an issue when there is no files in a directory to deploy 83 | * corrected the sitesettings template MEDIA_ROOT and STATIC_ROOT paths 84 | 85 | Release 0.5.3 (9-Nov-2010) 86 | --------------------------- 87 | 88 | * fix missing dist directory 89 | * fix trailing / in manifest 90 | 91 | Release 0.5.2 (7-Nov-2010) 92 | ---------------------------- 93 | 94 | * fix missing import 95 | 96 | Release 0.5.1 (25-Aug-2010) 97 | -------------------------------- 98 | 99 | * fix to setupnode sshd_config rollback 100 | * fix issue where setupnode fails if TEMPLATE_DIRS has not been set. 101 | 102 | 103 | Release 0.5.0 (16-Aug-2010) 104 | --------------------------------- 105 | 106 | Note: To upgrade from 0.4 to 0.5 you should move your first [your_project].sitesettings.domain.py to [your_project].sitesettings.settings.py and create a new [your_project].sitesettings.domain.py that just imports [your_project].sitesettings.settings 107 | 108 | * changed the [project_name].sitesettings.settings file to be the primary settings file for all non-dev sites 109 | * enable hosts to be setup with different packages through the ROLEDEFS, ROLE_PACKAGES, and ROLE_UFW_RULES settings. 110 | * added a maintenance.html template for nginx 111 | * added a default deny all nginx conf to setupnode 112 | * domains are now determined by dumping sites from the 1st database in the host list being executed against. If the project hasn't deployed we determine from the hostname or ask for it. 113 | * simplified deployment_root and added back in a django setting to override it. 114 | * added --noprofile to env.shell to make peace with virtualenvwrapper 115 | * removed --overwrite option from setupnode 116 | * fixed an issue with syncdb & migrate using the wrong settings file 117 | * changed the name of function deploy_webservers to deploy_webconf 118 | * setupnode now starts/restarts apache2 & nginx at the end of setup 119 | 120 | 121 | Release 0.4.0 (10-Aug-2010) 122 | --------------------------------- 123 | 124 | Note: This release is backwards incompatable with earlier releases. You will need to re-run setupnode and re-deploy your project. 125 | 126 | * can now use ROLEDEFS to define roles to group node functionality and use them in commands. ie ./manage.py deploy staging 127 | * moved logs back to /var/log/apache2 nginx etc and link into them instead 128 | * moved almost all woven /etc templates files into a new woven/etc template directory. 129 | * user can create their own woven/etc templates to upload any arbitrary /etc/ files into their corresponding directories on the host 130 | * changed deployment_root to the users home directory to allow integration with virtualenvwrapper 131 | * integrate with virtualenvwrapper. Can now run workon [projectname] to drop into the current version on the node 132 | * added a convenience settings.py, manage.py to sitesettings. The settings.py just imports the first sites settings 133 | * integrate with south for migrations, and added syncdb to activation 134 | * added manage.py patch subcommand where subcommand is an individual part of the deploy process. 135 | * removed unattended upgrades - due to unreliability 136 | * added an modified nginx init.d conf - the default init.d doesn't work under some boot timing circumstances 137 | * use nginx reload command instead of start stop 138 | * symlink the project directory to site-packages 139 | 140 | Release 0.3.1 (1-Aug-2010) 141 | -------------------------- 142 | 143 | * fixed a failure where trying to disable apparmor 144 | * shifted from apache2ctl to init.d for starting and stopping apache2 145 | * fixed an issue with requirements files 146 | * uses the first domain SITE_ID = 1 sitesettings for project settings 147 | 148 | Release 0.3 (22-Jul-2010) 149 | ------------------------- 150 | 151 | * Major api refactor. Moved away from classes to function with decorator pattern. Codebase should be much clearer now. 152 | * abstracted out a generic ``deploy_files`` function into deployment module that uses rsync but is more useful than fabric rsync_project where the remote_dir is not the same as the local parent dir. Stages files for network efficiency, and can deploy specific patterns of files in a directory and render templates if needed. 153 | * new decorator ``run_once_per_host_version`` and state functions simplify where a function may be called multiple times but needs only finish once per host and project version. 154 | * The public api can be imported ``from woven.api import *`` 155 | * Allow any host strings to be used instead of just ip addresses. 156 | * Resolves the host string where an ip is needed for apache/nginx 157 | * implements an activate command to activate to a specific project version (env + webserver conf etc) 158 | * ``bundle`` command bundles the requirements files for efficient deployment 159 | * added a template pip requirements file 160 | * added a ``node`` command to run arbitrary django management commands on hosts 161 | 162 | Release 0.2.1 (4-Jul-2010) 163 | --------------------------- 164 | 165 | * Fixed issue with installation fabric dependency 166 | 167 | Release 0.2 (3-Jul-2010) 168 | --------------------------- 169 | 170 | * Added deploy and patch management commands 171 | 172 | Release 0.1.1 (22-Jun-2010) 173 | --------------------------- 174 | 175 | * Changed serverserver to setupnode 176 | 177 | 178 | Release 0.1 (21-Jun-2010) 179 | ----------------------------- 180 | 181 | * Initial Release 182 | 183 | 184 | 185 | 186 | -------------------------------------------------------------------------------- /docs/commands.rst: -------------------------------------------------------------------------------- 1 | Management Commands 2 | =================== 3 | 4 | Commands that run on a host require a hoststring, role or have HOSTS or ROLEDEFS defined in your settings. 5 | 6 | As per fabric a hoststring can be username@hostname, or use just the hostname or ip address. If no username is defined then woven will use the current user or settings.HOST_USER. You never need to set the port. It will always use port 10022 though another port can be defined using the HOST_SSH_PORT setting. 7 | 8 | A common and recommended deployment pattern is to separate out staging servers from production servers. To do this in woven you would define your hoststrings in the ROLEDEFS settings (like fabfile roledefs). For example to have one staging server and two production servers you could define:: 9 | 10 | ROLEDEFS = {'staging':['user@192.168.188.10'], 'production':['user@host1.example.com', user@host2.example.com]} 11 | 12 | You would then use the role in place of the hoststring e.g. ``woven-admin.py deploy staging`` 13 | 14 | startproject 15 | ------------ 16 | 17 | Alternative to Django startproject. Creates a distribution folder as well as the project folder. You can also use your own alternative project layout template. 18 | 19 | Usage: 20 | 21 | ``woven-admin.py startproject [project_name] [options]`` 22 | 23 | Options: 24 | 25 | ``-t --template`` path to an alternative template directory. 26 | 27 | ``-d --dist`` an alternate distribution name for the project. Defaults to the project name. 28 | 29 | ``--noadmin`` don't uncomment admin. 30 | 31 | setupnode 32 | --------- 33 | 34 | Setup Ubuntu host[s]. By default this will just setup a baseline host for django deployment but it can be used to setup other types of host such as postgresql nodes and varnish. 35 | 36 | By defining ROLEDEFS in your settings you define packages for those hosts in the ROLE_PACKAGES settings. For example to setup a postgresql server you might define a role ROLEDEFS = {'database':['woven@db1.example.com']}, then set ROLE_PACKAGES = {'database':['postgresql']}, and finally set the firewall to ROLE_UFW_RULES = {'database':['allow tcp/5432']}. Finally postgresql configuration can be set in the project TEMPLATES_DIR woven/etc subdirectory. 37 | 38 | Basic Usage:: 39 | 40 | ``woven-admin.py setupnode [hoststring] [options]`` 41 | 42 | Lets go through what this actually does: 43 | 44 | 1. Creates the new `user` and disables the `root` user 45 | 2. Changes the default ssh port to 10022 46 | 3. Uploads your public ssh-key 47 | 4. Restricts ssh login to the ssh-key and adds a few other restrictions 48 | 5. Adds additional sources `universe` to sources.list 49 | 6. Updates and upgrades your packages 50 | 7. Installs UFW firewall 51 | 8. Installs a baseline of Ubuntu packages including Apache, Nginx, and mod-wsgi 52 | 9. Install any etc templates/files sourced from the woven or project woven/etc template directories 53 | 10. Sets the timezone according to your settings file 54 | 55 | 56 | bundle 57 | ------ 58 | 59 | Pip bundle your requirements into pip zip bundles for efficient deployment 60 | 61 | ``woven-admin.py bundle`` 62 | 63 | 64 | deploy 65 | ------ 66 | 67 | Deploy your project to host[s] run syncdb and activate 68 | 69 | Basic Usage: 70 | 71 | ``woven-admin.py deploy [hoststring] [options]`` 72 | 73 | *options* 74 | 75 | The ``--overwrite`` option will remove and re-deploy the entire project for that version. By default deploy will never overwrite an existing version of your project. 76 | 77 | *South migration options* 78 | 79 | deploy integrates with south if it is installed. By default *all* migrations are run. 80 | 81 | ``-m --migration`` Specify a specific migration to run (see South documentation) 82 | 83 | ``--fake`` Fake a South migration (see South documentation) 84 | 85 | ``--nomigration`` Do not migrate 86 | 87 | ``--manualmigration`` Manage the database migration manually. With this option you can drop out of the current deployment to migrate the database manually, or pause the deployment while migrating in a separate shell. To migrate the database you could login to your host and then run ``workon [yourproject-version]`` to drop into the new versions environment and migrate your database using south, then logout and re-run deploy or continue the existing deploy. 88 | 89 | The deploy command does the following: 90 | 91 | 1. For your first deployment it will deploy your development sqlite database (if it exists) 92 | 2. Create a virtualenv for the distribution version 93 | 3. Install django. By default it will install the local version. You can set a pip requirements string DJANGO_REQUIREMENT in your settings.py if you want svn trunk or some other specific version. 94 | 4. Install dependencies from one or more requirement req* files. eg. req, requirements.txt etc. If one doesn't exist then it will create one locally and add woven in it by default. 95 | 5. Creates a local sitesettings folder and a settings file for your server settings.py if it doesn't already exist. You can see how woven lays out your project on the server in the sitesettings\settings.py file. 96 | 6. Deploys your project to the virtualenv on the server 97 | 7. Deploys your root (shortest path) TEMPLATE_DIR into a templates directory on the server. 98 | 8. Deploys admin media or STATIC_ROOT setting (if you use django-staticfiles) into a virtualenv static directory. 99 | 9. Deploys anything at MEDIA_ROOT into a non-virtualenv public directory. 100 | 10. Deploys your wsgi file into a virtualenv wsgi directory as settings.wsgi 101 | 11. Renders your apache and nginx templates and deploys them into the sites-available with the version in the name. 102 | 12. Stops the webservices 103 | 13. Syncs the database 104 | 14. Runs South migrate if you have South installed 105 | 15. Symlinks the webserver conf versions into sites-enabled 106 | 16. Symlinks the project virtualenv version to the active virtualenv. 107 | 17. Starts the webservices 108 | 109 | 110 | patch 111 | ----- 112 | 113 | Patch the current version of your project on host[s] and restart\reload webservices 114 | Includes project, web configuration, media, and wsgi but does not pip install 115 | 116 | Basic Usage: 117 | 118 | ``woven-admin.py patch [subcommand] [hoststring] [options]`` 119 | 120 | You can just patch a part of the deployment with a subcommand. 121 | 122 | The possible subcommands are:: 123 | 124 | project, templates, static, media, wsgi, webconf 125 | 126 | Example: 127 | 128 | ``woven-admin.py patch media woven@host.example.com`` 129 | 130 | 131 | activate 132 | -------- 133 | 134 | Activate a project version 135 | 136 | Usage: 137 | 138 | ``woven-admin.py activate version [options]`` 139 | 140 | Example: 141 | 142 | ``woven-admin.py activate 0.1 woven@host.example.com`` 143 | 144 | node 145 | ---- 146 | 147 | Run a no arguments management command on host[s]. You can supply command options through the 148 | --options option --options="[option ...]" 149 | 150 | Basic Usage: 151 | 152 | ``woven-admin.py node command [hoststring] [options]`` 153 | 154 | Example: 155 | 156 | ``woven-admin.py node flush woven@host.example.com --options="--noinput"`` 157 | 158 | startsites 159 | ---------- 160 | 161 | Deploy webconf for the new sites and create a new user ``site_n`` where n is the SITE_ID of the new site(s). 162 | 163 | Within Django sites are created on the database but use the SITE_ID in the settings file to designate which site is loaded. This command does not create the sites in the database but merely creates and deploys the configuration files needed to serve them. 164 | 165 | Basic Usage: 166 | 167 | ``woven-admin.py startsites [hoststring] [options]`` 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Woven documentation build configuration file, created by 4 | # sphinx-quickstart on Sat Jun 19 13:59:33 2010. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | import sys, os 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | sys.path.append(os.path.abspath('../')) 20 | 21 | # -- General configuration ----------------------------------------------------- 22 | 23 | # Add any Sphinx extension module names here, as strings. They can be extensions 24 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 25 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo'] 26 | 27 | # Add any paths that contain templates here, relative to this directory. 28 | templates_path = ['_templates'] 29 | 30 | # The suffix of source filenames. 31 | source_suffix = '.rst' 32 | 33 | # The encoding of source files. 34 | #source_encoding = 'utf-8' 35 | 36 | # The master toctree document. 37 | master_doc = 'index' 38 | 39 | # General information about the project. 40 | project = u'Woven' 41 | copyright = u'2010, Brett Haydon' 42 | 43 | # The version info for the project you're documenting, acts as replacement for 44 | # |version| and |release|, also used in various other places throughout the 45 | # built documents. 46 | # 47 | # The short X.Y version. 48 | from woven import get_version 49 | version = get_version() 50 | # The full version, including alpha/beta/rc tags. 51 | release = get_version() 52 | 53 | # The language for content autogenerated by Sphinx. Refer to documentation 54 | # for a list of supported languages. 55 | #language = None 56 | 57 | # There are two options for replacing |today|: either, you set today to some 58 | # non-false value, then it is used: 59 | #today = '' 60 | # Else, today_fmt is used as the format for a strftime call. 61 | #today_fmt = '%B %d, %Y' 62 | 63 | # List of documents that shouldn't be included in the build. 64 | #unused_docs = [] 65 | 66 | # List of directories, relative to source directory, that shouldn't be searched 67 | # for source files. 68 | exclude_trees = ['_build'] 69 | 70 | # The reST default role (used for this markup: `text`) to use for all documents. 71 | #default_role = None 72 | 73 | # If true, '()' will be appended to :func: etc. cross-reference text. 74 | #add_function_parentheses = True 75 | 76 | # If true, the current module name will be prepended to all description 77 | # unit titles (such as .. function::). 78 | #add_module_names = True 79 | 80 | # If true, sectionauthor and moduleauthor directives will be shown in the 81 | # output. They are ignored by default. 82 | #show_authors = False 83 | 84 | # The name of the Pygments (syntax highlighting) style to use. 85 | pygments_style = 'sphinx' 86 | 87 | # A list of ignored prefixes for module index sorting. 88 | #modindex_common_prefix = [] 89 | 90 | 91 | # -- Options for HTML output --------------------------------------------------- 92 | 93 | # The theme to use for HTML and HTML Help pages. Major themes that come with 94 | # Sphinx are currently 'default' and 'sphinxdoc'. 95 | html_theme = 'sphinxdoc' 96 | 97 | # Theme options are theme-specific and customize the look and feel of a theme 98 | # further. For a list of options available for each theme, see the 99 | # documentation. 100 | #html_theme_options = {} 101 | 102 | # Add any paths that contain custom themes here, relative to this directory. 103 | #html_theme_path = [] 104 | 105 | # The name for this set of Sphinx documents. If None, it defaults to 106 | # " v documentation". 107 | #html_title = None 108 | 109 | # A shorter title for the navigation bar. Default is the same as html_title. 110 | #html_short_title = None 111 | 112 | # The name of an image file (relative to this directory) to place at the top 113 | # of the sidebar. 114 | #html_logo = None 115 | 116 | # The name of an image file (within the static path) to use as favicon of the 117 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 118 | # pixels large. 119 | #html_favicon = None 120 | 121 | # Add any paths that contain custom static files (such as style sheets) here, 122 | # relative to this directory. They are copied after the builtin static files, 123 | # so a file named "default.css" will overwrite the builtin "default.css". 124 | html_static_path = ['_static'] 125 | 126 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 127 | # using the given strftime format. 128 | #html_last_updated_fmt = '%b %d, %Y' 129 | 130 | # If true, SmartyPants will be used to convert quotes and dashes to 131 | # typographically correct entities. 132 | #html_use_smartypants = True 133 | 134 | # Custom sidebar templates, maps document names to template names. 135 | #html_sidebars = {} 136 | 137 | # Additional templates that should be rendered to pages, maps page names to 138 | # template names. 139 | #html_additional_pages = {} 140 | 141 | # If false, no module index is generated. 142 | #html_use_modindex = True 143 | 144 | # If false, no index is generated. 145 | #html_use_index = True 146 | 147 | # If true, the index is split into individual pages for each letter. 148 | #html_split_index = False 149 | 150 | # If true, links to the reST sources are added to the pages. 151 | #html_show_sourcelink = True 152 | 153 | # If true, an OpenSearch description file will be output, and all pages will 154 | # contain a tag referring to it. The value of this option must be the 155 | # base URL from which the finished HTML is served. 156 | #html_use_opensearch = '' 157 | 158 | # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). 159 | #html_file_suffix = '' 160 | 161 | # Output file base name for HTML help builder. 162 | htmlhelp_basename = 'Wovendoc' 163 | 164 | 165 | # -- Options for LaTeX output -------------------------------------------------- 166 | 167 | # The paper size ('letter' or 'a4'). 168 | #latex_paper_size = 'letter' 169 | 170 | # The font size ('10pt', '11pt' or '12pt'). 171 | #latex_font_size = '10pt' 172 | 173 | # Grouping the document tree into LaTeX files. List of tuples 174 | # (source start file, target name, title, author, documentclass [howto/manual]). 175 | latex_documents = [ 176 | ('index', 'Woven.tex', u'Woven Documentation', 177 | u'Brett Haydon', 'manual'), 178 | ] 179 | 180 | # The name of an image file (relative to this directory) to place at the top of 181 | # the title page. 182 | #latex_logo = None 183 | 184 | # For "manual" documents, if this is true, then toplevel headings are parts, 185 | # not chapters. 186 | #latex_use_parts = False 187 | 188 | # Additional stuff for the LaTeX preamble. 189 | #latex_preamble = '' 190 | 191 | # Documents to append as an appendix to all manuals. 192 | #latex_appendices = [] 193 | 194 | # If false, no module index is generated. 195 | #latex_use_modindex = True 196 | -------------------------------------------------------------------------------- /docs/conventions.rst: -------------------------------------------------------------------------------- 1 | Conventions 2 | =========== 3 | 4 | Woven will use the following conventions to layout your project on the target host, and uses some basic coventions on the local development machine. 5 | 6 | In the following examples we are deploying a distribution ``example_distribution`` with the django project ``example_project`` to a site ``example.com`` 7 | 8 | .. _setup.py: 9 | 10 | Setup.py 11 | -------- 12 | 13 | A setup file must be defined in your distribution. 14 | 15 | **setup.py**:: 16 | 17 | from distutils.core import setup 18 | 19 | setup(name='example_distribution', #This is what your virtualenvs will be called 20 | version='0.1', 21 | packages=['example_project'], 22 | ) 23 | 24 | Project Development Layout 25 | -------------------------- 26 | 27 | While woven tries to be agnostic about your project layout there are some conventions. 28 | 29 | :: 30 | 31 | distribution folder 32 | |--dist (a dist folder will be created if you *bundle* requirements) 33 | | |--requirements.zip (bundle will zip up python packages here with matching req*.txt file name) 34 | |--setup.py (a minimal setup.py is required with name, version, and packages defined) 35 | |--requirements.txt (a requirements.txt will be created if one doesn't exist) 36 | |--req2 (extra requirements can be defined prefixed with 'req') 37 | |--example_project (the standard django startproject) 38 | |--__init__.py 39 | |--deploy.py (optional hooks for custom setupnode or deploy functions) 40 | |--manage.py 41 | |--settings.py (global settings & local development settings) 42 | |--urls.py 43 | |--sitesettings (will be created if it doesn't exist) 44 | |--__init__.py 45 | |--settings.py (the sites specific setting overrides) 46 | |--subdomain_settings.py (a site subdomain with the same SITE_ID) 47 | |--manage.py (a convenience for running on node against site settings.py) 48 | |--local_apps (you can define an optional PROJECT_APPS_PATH in settings that will hold apps and be in site-packages path on deployment) 49 | |--app1 50 | |--deploy.py (hooks can be at app level as well) 51 | |--... 52 | |--media (actual location as defined in settings) 53 | | |--(any user content here) 54 | |--static (actual location as defined in settings) 55 | | |--(any application media here) 56 | |--templates (actual location as defined in settings) 57 | | #woven comes with a number of default templates in the distribution 58 | | #you can copy them and override them in your template directory. 59 | |--woven 60 | |--etc (optional - copy and override from woven package) 61 | |--apache2 62 | |--ports.conf 63 | |--init.d 64 | |--nginx 65 | |--nginx 66 | |--nginx.conf 67 | |--proxy.conf 68 | |--ufw 69 | |--applications.d 70 | |--woven_project (optional - see notes*) 71 | |--ssh 72 | |--sshd_config 73 | |--django-apache-template.txt (sites conf) 74 | |--django-wsgi-template.txt 75 | |--maintenance.html (for when your site is being deployed) 76 | |--nginx-template.txt (sites conf) 77 | |--requirements.txt 78 | |--sitesettings.txt (default sitesetting) 79 | |--ufw.txt (ssh firewall rules) 80 | |--ufw-woven_project.txt (project firewall rules) 81 | |--[template.html] (global projecttemplates here) 82 | 83 | 84 | **Notes:** 85 | 86 | *etc templates* 87 | 88 | Templates in the etc folder are uploaded using the following rules: 89 | 90 | - templates are uploaded from the project directory first, and if they don't exist the woven package installed templates as per the standard django template loader 91 | 92 | - If an etc package subdirectory exists on the node (eg apache2), all templates in the folder are uploaded. 93 | 94 | - If an etc subdirectory is not package related (eg init.d) the template will only be uploaded to overwrite an existing template. 95 | 96 | etc templates are uploaded if the template has changed or if the packages on the node change. 97 | 98 | *UFW firewall rules* 99 | 100 | UFW can use app templates defined in the `/etc/ufw/applications.d` directory. Woven uploads `ufw.txt` as `woven` in this directory to set the SSH firewall rules. 101 | 102 | A firewall rule for all nodes can be defined at UFW_RULES or exclusively for a role at ROLE_UFW_RULES. A rule is something like `allow 5432/tcp`. See UFW documentation for rule syntax. Removing a rule will delete it from the firewall next time setupnode is run. 103 | 104 | Project Deployment Layout 105 | ------------------------- 106 | 107 | Within the root folder on the node are the following:: 108 | 109 | ~/.staging (all rsynced files are staged here before copying to final destination for network efficiency) 110 | ~/.pip (pip installation logs) 111 | | |--cache (Pip will cache packages here) 112 | | |--src (pip will store any editable source repositories here) 113 | ~/--database (for sqlite if it is used) 114 | | |--example_project.db (will always be deployed as the [project-name].db) 115 | |--env (The root directory for all virtual environments) 116 | |--example_distribution (symlink to the current virtualenv version) 117 | |--example_distribution-0.1 (The virtualenv root for this version) 118 | |--bin 119 | |--dist 120 | |--requirements.pybundle 121 | |--include 122 | |--lib 123 | |--project 124 | |--example_project (package directory - symlinked to site-packages) 125 | |--manage.py (your development manage.py) 126 | |--settings.py (global & dev specific settings) 127 | |--sitesettings (site local setting files) 128 | |--__init__.py 129 | |--manage.py (you run this on the node) 130 | |--settings.py (primary settings file for nodes) 131 | |--templates (your project templates go here) 132 | |--static (admin and other app media) 133 | |--wsgi (web server scripts go here including modwsgi python file) 134 | |--settings.wsgi (for modwsgi) 135 | |--example_distribution-0.2 (next release version - as above) 136 | ... 137 | |--log (symlinks to /var/log) 138 | | Another media directory for files that in the user domain (MEDIA_URL) rather than required for the application 139 | |--public (for single domain deployments any project media goes here if you are hosting media locally) 140 | 141 | 142 | Apache & Nginx Configuration files 143 | ---------------------------------- 144 | 145 | /etc/apache2/sites-available/ 146 | By convention the configuration file will be saved using the domain name as follows. 147 | 148 | /etc/apache2/sites-available/example_com-0.1.conf 149 | 150 | Nginx for media is done the same way 151 | 152 | Server-side State 153 | --------------------- 154 | 155 | Woven keeps track of server state and other housekeeping functions using the 156 | 157 | `/var/local/woven/` directory 158 | 159 | Currently state is stored as a filename with or without content. 160 | 161 | Backups of configuration files are stored at 162 | 163 | `/var/local/woven-backup` 164 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. Woven documentation master file, created by 2 | sphinx-quickstart on Sat Jun 19 13:59:33 2010. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | ===== 7 | Woven 8 | ===== 9 | 10 | .. include:: ../README.txt 11 | 12 | Contents: 13 | ========= 14 | 15 | .. toctree:: 16 | :maxdepth: 2 17 | 18 | main 19 | tutorial 20 | commands 21 | settings 22 | conventions 23 | fabric 24 | troubleshooting 25 | changelog 26 | 27 | 28 | 29 | Indices and tables 30 | ================== 31 | 32 | * :ref:`genindex` 33 | * :ref:`modindex` 34 | * :ref:`search` 35 | 36 | -------------------------------------------------------------------------------- /docs/main.rst: -------------------------------------------------------------------------------- 1 | Woven 2 | ======== 3 | 4 | * utilizes Pip, Virtualenv, Fabric and Django templates to simplify deployment making *release early and often* a reality. 5 | 6 | * provides six Django commands/functions; ``setupnode``, ``deploy``, ``patch``, ``activate``, ``bundle``, ``startsites`` 7 | 8 | * provides a standard Apache & Nginx webserver configuration that you can change as required, and includes experimental support for Gunicorn. 9 | 10 | * versions your project with virtualenv 11 | 12 | * enables you to define custom linux etc configuration files in your project 13 | 14 | * enable you to define roles for your servers with combinations of linux packages and firewall rules 15 | 16 | * provides an api for fabfile.py scripts. 17 | 18 | * hooks for custom functionality 19 | 20 | * integrates with South for migrations 21 | 22 | * basic multi-site multi-db capabilities 23 | 24 | 25 | **Woven currently doesn't:** 26 | 27 | * Create and launch your node from any cloud providers - it just does the setup of a baseline installation with setupnode 28 | 29 | * bring you beer 30 | 31 | Installation 32 | ============ 33 | 34 | .. include:: ../INSTALL.txt 35 | 36 | Getting Started 37 | =============== 38 | 39 | To use Woven you must have root access to a linux host or vm. Woven has currently been tested on Ubuntu >= 10.04. 40 | 41 | Woven uses a custom ``woven-admin.py`` script that serves to replace ``django-admin.py`` and ``manage.py`` in your development environment, and allows you to use woven commands without adding woven to your installed apps. 42 | 43 | Run ``woven-admin.py startproject`` which will create a basic django project distribution layout. 44 | 45 | Woven provides six management commands for your Django project: 46 | 47 | ``setupnode``, ``deploy``, ``patch``, ``activate``, ``bundle``, ``startsites`` 48 | 49 | You can walk through some of the commands in a simple example django project :doc:`tutorial` or read the :doc:`commands` docs. You'll also want to have a look at the :doc:`settings` and the project :doc:`conventions`. 50 | 51 | Integrating with Fabric 52 | ======================= 53 | 54 | Woven is just fabric. To integrate with your own fabfiles you can do: 55 | 56 | :: 57 | 58 | import os 59 | 60 | #import any other woven functions you want to use or all of them 61 | from woven.api import * 62 | 63 | #Set the environ for Django if you need to 64 | #(assumes you have your project on the pythonpath) 65 | os.environ['DJANGO_SETTINGS_MODULE'] = 'example_project.settings' 66 | 67 | #set_env will initialise your env for woven functions and use your settings 68 | set_env() 69 | 70 | #define your own fabric functions that use woven 71 | def your_function(): 72 | ... 73 | 74 | Custom Hooks 75 | ============ 76 | 77 | Woven provides hooks into the setupnode, deploy, and post package installation commands. 78 | 79 | To add custom functionality to a woven deployment create a ``deploy.py`` file in your project or django app, and define any of the following. 80 | 81 | 82 | Post install package 83 | --------------------- 84 | 85 | Define a ``def post_install_[package_name]()`` function to run code if the Ubuntu package is installed by woven, and after any /etc configuration is uploaded. For example you might define ``def post_install_postgresql()`` to setup a postgresql database. Replace any dashes or fullstops with underscores to make it a valid python function. 86 | 87 | A sample hook is defined in Woven for installing postgresql 88 | 89 | Post setupnode 90 | -------------- 91 | 92 | ``def post_setupnode()`` executes at the end of the setupnode process. 93 | 94 | Post deploy 95 | ----------- 96 | 97 | ``def post_deploy()`` executes at the end of deployment of your project but prior to activation. 98 | 99 | Hooks execute in a project, app, woven order of precedence. Only one hook per function will execute. A project scope hook for instance will override the same function at app or provided by woven itself. 100 | 101 | Multiple Sites 102 | ============== 103 | 104 | Woven creates a user on the node for each SITE_ID. The users will be called ``site_1``, ``site_2`` ... The Apache site templates use process groups to launch modwsgi daemons running as the site user. The settings file gets the current user and dynamically sets the SITE_ID in the settings. In this fashion a single settings file can be used for multiple sites. 105 | 106 | Since Django sites uses a SITE_ID rather than a domain to represent the site, it doesn't really know about subdomains, but you might want might make a default admin view of your SITE_ID = 1 at admin.example.com. To accommodate this you can make a settings file called admin_settings.py in the sitesettings folder. You would add a prefix for any other alternative view of the same site using [sub_domain]_settings.py 107 | 108 | Development 109 | =========== 110 | 111 | The core highlevel functions setupnode, deploy, patch, bundle, and activate will not change, but some other parts of the api may still change between versions. I am aiming to release a beta version (something less than 1.0) sometime after the release of Fabric 1.0, since Fabric 1.0 will currently break Woven due to backwards incompatabilities. After that time Woven will depend on Fabric >= 1.0. 112 | 113 | Future Goals 114 | ------------ 115 | 116 | * I would like to see other ways of serving django behind nginx implemented. Ubuntu 11.04 will ship with a nginx that doesn't need to be compiled to use uwsgi, so that will be a good time to add support for it. 117 | * Although it's currently tested on Ubuntu, I'm happy to accept patches or feedback to make it work on debian based distro's or other linux distributions. 118 | * The future is distutils2. I actually think the 1.0 version of woven should make full use of the new packaging system and metadata. When packaging no longer sucks, I can simplify woven to leverage it, and it has implications for django project conventions. I'll be looking for distutils2 to move to beta before I begin to develop for it. 119 | 120 | Testing 121 | -------- 122 | Woven stores tests in a fabfile.py in the tests directory. The tests are pretty incomplete at this time. 123 | 124 | Individual tests are split across multiple modules corresponding to the module they are testing. 125 | 126 | Tests require a vm to be setup with ``root`` login and password ``root`` on 192.168.188.10 127 | 128 | Tests can be run by ``fab test`` to run all tests or ``fab test_[modulename]`` to run all tests in the given test module (eg ``fab test_env``), or ``fab [full_test_name]`` to run an individual test from any test module. 129 | 130 | Contact 131 | ======= 132 | 133 | The woven project is hosted on github at http://github.com/bretth/woven. Feature requests and bug reports are welcome. 134 | 135 | You can contact me at brett at haydon dot id dot au 136 | 137 | -------------------------------------------------------------------------------- /docs/settings.rst: -------------------------------------------------------------------------------- 1 | 2 | Settings 3 | ======== 4 | 5 | Woven has a number of configuration options that can be set in your project's 6 | Django settings.py. They are all optional. 7 | 8 | :: 9 | 10 | #List of hoststrings to setup on as per Fabric. 11 | HOSTS = [] #eg ['woven@example.com','example.com','10.0.0.1'] 12 | #You can group collections of servers instead of HOSTS as per fabric 13 | ROLEDEFS = {} #This would be used instead of HOSTS eg {'staging':['woven@example.com']} 14 | #The ssh port to be setup. We change the port for security 15 | HOST_SSH_PORT = 10022 #default 16 | #User can be defined here instead of in the hosts/roledefs 17 | HOST_USER = '' 18 | #Since password login will normally be disabled you can define it here 19 | #for just ssh key security per host 20 | HOST_PASSWORD = '' 21 | 22 | #As per fabric KEY_FILENAME option to specify a path to an ssh key to use 23 | SSH_KEY_FILENAME = '' 24 | 25 | #The first setup task is usually disabling the default root account and changing the ssh port. 26 | ROOT_USER = 'root', #optional - mostly the default administrative account is root 27 | DISABLE_ROOT = False, #optional - disable the default administrative account 28 | ROOT_PASSWORD = '', #optional - blank by default 29 | #The default ssh port, prior to woven changing it. Defaults to 22 30 | DEFAULT_SSH_PORT = 22 #default 31 | DISABLE_SSH_PASSWORD = #optional - setting this to true will disable password login and use ssh keys only. 32 | #Firewall rules (note HOST_SSH_PORT/tcp is always allowed) 33 | ENABLE_UFW = True #default - setting this to false will disable UFW 34 | #Any rule(s) defined will overwrite the default 35 | UFW_RULES = ['allow 80,443/tcp'] #default - see Ubuntu UFW for more details about rules 36 | ROLE_UFW_RULES = {} # eg {'postgresql':['allow 5432/tcp']} 37 | 38 | #The default ubuntu packages that are setup. It is NOT recommended you overwrite these 39 | #use ROLE_PACKAGES if you want to define all your packages 40 | HOST_BASE_PACKAGES = [ 41 | 'subversion','git-core','mercurial','bzr', #version control 42 | 'gcc','build-essential', 'python-dev', 'python-setuptools', #build 43 | 'apache2','libapache2-mod-wsgi', #wsgi server 44 | 'nginx', #webserver 45 | 'python-imaging', #pil 46 | 'python-psycopg2','python-mysqldb','python-pysqlite2'] #default database drivers 47 | 48 | #Package notes: 49 | #If you define gunicorn in your extra packages then apache and mod-wsgi will not be 50 | #installed, or will be removed. psycopg2 or mysqldb will only be installed 51 | #if they are required by your DATABASES engine settings. 52 | 53 | #Put any additional packages here to save overwriting the base_packages 54 | HOST_EXTRA_PACKAGES = [] 55 | 56 | #define a list of repositories/sources to search for packages 57 | #Current just handles Personal Package Archives (PPAs) 58 | LINUX_PACKAGE_REPOSITORIES = [] # eg ['ppa:bchesneau/gunicorn'] 59 | 60 | #Role packages give you complete flexibility in defining packages with ROLEDEFS. 61 | #By default any role that does not have role packages defined installs the HOST_BASE_PACKAGES + EXTRA_PACKAGES instead 62 | ROLE_PACKAGES = {} #eg ROLE_PACKAGES = {'postgresql':['postgresql']} 63 | 64 | #Apache list of modules to disable for performance and memory efficiency 65 | #defaults to the following: 66 | APACHE_DISABLE_MODULES=['alias','auth_basic','authn_file','authz_default','authz_groupfile', 67 | 'authz_user','autoindex','cgid','dir', 68 | 'setenvif','status'], 69 | #Virtualenv/Pip 70 | DEPLOYMENT_ROOT = ''# defaults to /home/$USER. 71 | 72 | PIP_REQUIREMENTS = [] # list of text pip requirements files (not pybundles). Defaults to any file in the setup.py directory with `req` prefix 73 | # Note: Woven will also look for zip files matching the requirements in the dist directory. 74 | #you can use the bundle management command to create these. 75 | 76 | PROJECT_APPS_PATH = '' #a relative path from the project package directory for any local apps. See also the wsgi template. 77 | 78 | #Application media - as per build_static app 79 | STATIC_URL = '' #by default this is set to the ADMIN_MEDIA_PREFIX 80 | STATIC_ROOT = '' #by default this gets set to the admin media directory if admin is used 81 | 82 | #Database migrations 83 | MANUAL_MIGRATION = False #Manage database migrations manually 84 | 85 | 86 | -------------------------------------------------------------------------------- /docs/troubleshooting.rst: -------------------------------------------------------------------------------- 1 | Troubleshooting 2 | =============== 3 | 4 | Stuff happens. 5 | 6 | Adding --verbosity=2 to setupnode or deploy will show the debug view of fabric. This is more than you need but will help in diagnosing opaque error messages. 7 | 8 | **setupnode hangs on upgrade** 9 | 10 | Most likely some package in the apt-get upgrade process is incorrectly asking for user input. 11 | 12 | *do not kill the process at the client end* 13 | 14 | This will most likely leave your Ubuntu package database in a broken state. 15 | 16 | The best solution is to log directly onto the host without killing the hung setupnode and then run sudo apt-get upgrade directly on the host. It will need you to dpkg configure -a and possibly remove the /var/lib/dpkg/updates if you are still having issues. 17 | 18 | **ERROR: There was an error running django-admin.py on the node** 19 | 20 | By deploying early and often it will be easier to diagnose misconfiguration issues. 21 | Usually this error will mean that django can't initialize your project due to import or other related issues, due to a particular app or code within your project. 22 | In the event it is a particular version of one of your requirements you can overwrite an existing installation by running ``woven-admin.py deploy --overwrite`` which will wipe your existing failed deployment. 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /docs/tutorial.rst: -------------------------------------------------------------------------------- 1 | Tutorial 2 | ======== 3 | 4 | A simple django project is the best way to illustrate how to get started with woven (and django) 5 | 6 | Starting the project 7 | -------------------- 8 | 9 | .. Note:: 10 | 11 | This first bit doesn't have much to do with woven, and is more about personal preference in setting up your development environment but lets walk through *one* way you can get setup. For this you probably want pip, virtualenv and virtualenvwrapper installed and working before you can begin. 12 | 13 | We're going to create a virtual python environment ``first-django``. You don't need to do this but virtualenv makes it easy to experiment without polluting your system installed packages. 14 | 15 | ``mkproject first-django`` 16 | 17 | Activate the env if it isn't already.. 18 | 19 | ``workon first-django`` 20 | 21 | Finally create a first-django distribution directory if mkproject hasn't already created one. 22 | 23 | Okay lets get into woven. 24 | 25 | Installing the packages 26 | ----------------------- 27 | 28 | Install woven - making sure it correctly installs in the virtualenv. 29 | 30 | ``pip install django`` 31 | ``pip install woven`` 32 | 33 | It should also install fabric, and paramiko if they aren't already installed. 34 | 35 | Creating the distribution and project 36 | -------------------------------------- 37 | 38 | Create a project distribution and files using woven's ``woven-admin.py`` script. A distribution is all the files including media and other data relating to the project whereas the project is just the Django project. We're going to call the distribution something different from the actual project. 39 | 40 | ``woven-admin.py startproject helloadmin woven@example.com --dist=first-django`` 41 | 42 | ``woven-admin.py`` extends ``django-admin.py`` allowing us to use woven commands without adding woven itself to the ``INSTALLED_APPS`` setting. 43 | 44 | You'll notice that it's a little different from ``django-admin.py startproject`` in that it creates a ``setup.py`` and a few other folders, and merges in the syncdb and creates a default superuser based on the user part of the optional email address (in this case 'woven') or the project name if you want one, though you can skip this behaviour with ``--nosyncdb``. As you'll discover, woven's ethos combines convenience with flexibility. Sometimes you need to get somewhere in a hurry, or just want a project you can throw away after using. 45 | 46 | The setup.py is where woven gets your distribution name, project name and project version metadata which is used in deployments, it's just not used to package your project... yet. That will come when it morphs into setup.cfg and uses distutils2. 47 | 48 | Woven's alternative to django's ``startproject`` also creates some sensible folders for media, static (app) content, templates, and database, and uses an alternative settings file from the django default. Nothing stopping you from changing it later if you want or you can also use ``startproject -t`` option to specify alternative starting templates for your project. 49 | 50 | In the urls.py uncomment the admin lines then at the end of your urls.py we'll add a simple index page.:: 51 | 52 | urlpatterns += patterns('django.views.generic.simple', 53 | (r'^$', 'direct_to_template', {'template': 'index.html'}), 54 | ) 55 | 56 | Finally in your templates folder create an index.html template file:: 57 | 58 | 59 | 60 | 61 | 62 | Hello admin 63 | 64 | 65 | 66 | Hello admin 67 | 68 | 69 | 70 | To add a package to the PYTHONPATH you would normally install it using setup.py, but since we're developing, link the project folder ``helloadmin`` into your site packages. I recommend these pylink/pyunlink shell scripts: https://gist.github.com/21649. I don't recommend using add2virtualenv since it puts the setup.py in the path. 71 | 72 | Change directory into the distribution folder (the one with setup.py in it) and run ``woven-admin.py runserver``, opening http://127.0.0.1:8000/ in your browser. 73 | 74 | ``woven-admin.py`` doesn't require you to set a DJANGO_SETTINGS_MODULE in the environment (though you can). Instead it infers your settings from the ``setup.py`` project name if you're somewhere under or in the setup.py folder, but you can also use ``--settings`` option as per normal. 75 | 76 | If you have done everything right you should now see ``hello admin`` and be able to login to the django admin with the superuser. You're ready to deploy! 77 | 78 | Setting up your server 79 | ---------------------- 80 | 81 | Although woven does allow you to scale your deployment to multiple nodes, it doesn't support creating the initial image, so for now you'll need to purchase and startup an Ubuntu virtual machine separately. 82 | 83 | Obtain an Ubuntu 10.04 or greater VM on the host of your choice with root and ssh access. For this tutorial I'm going to use a micro instance on Amazon EC2. See http://docs.amazonwebservices.com/AWSEC2/latest/GettingStartedGuide/ for how to get started on EC2. 84 | 85 | 1. Create a micro Ubuntu 10.04 x32instance ami-95c694d0 ebs on us-west (or similar) and a woven keypair. 86 | 2. Save the woven.pem key into your distribution folder 87 | 3. ``chmod 400 woven.pem`` 88 | 4. Create a security group with ports 22,80,10022 tcp open 89 | 5. Create and associate an elastic IP to the instance 90 | 91 | Because django uses ``example.com`` as it's first site, we'll stick with that for this tutorial deployment. In your local ``/etc/hosts`` file add an entry for example.com pointing to the ip address of the ubuntu host (and on osx, run ``dscacheutil -flushcache`` just to be sure). 92 | 93 | Just to be sure - check that you can login to your ec2 instance:: 94 | 95 | ssh -i woven.pem ubuntu@example.com 96 | 97 | Setupnode 98 | --------- 99 | 100 | Now run setupnode to setup a baseline node for your intended user - in this case woven. The first thing woven will do is setup an additional user and if 'root' is the default user it will be disabled. It will also change the ssh port which is how woven determines that it has been pre-installed. 101 | 102 | .. code-block:: bash 103 | 104 | woven-admin.py setupnode -i woven.pem woven@example.com 105 | 106 | or for non ec2, just woven-admin.py setupnode woven@example.com 107 | Answer according to your instance. Your root user may vary according to the provider. In our case with ec2 it is `ubuntu`. 108 | 109 | .. Note:: 110 | 111 | You might have noticed that setupnode uploads some files to the ubuntu ``etc`` directories. *Your node (host) configuration is stored in your project*. Woven allows you to define your own etc configuration files for ubuntu packages as standard templates in your project. If you want to modify the woven default templates you can copy them from the installed woven package into a woven folder in your projects templates folder like any other django app templates. 112 | 113 | You can re-run setupnode at any time to alter your node configuration and update, upgrade and install new debian packages. 114 | 115 | Now that your server is setup it's time to deploy our helloadmin project. 116 | 117 | Deploy 118 | ---------------- 119 | 120 | *Deploy early. Deploy often.* 121 | 122 | Lets collect the static media assets and deploy. 123 | 124 | .. code-block:: bash 125 | 126 | woven-admin.py collectstatic 127 | woven-admin.py deploy woven@example.com 128 | 129 | Deploy sets up a virtual environment on the server and deploys your sqlite3 database, django, and your project and all your dependencies into it. Sqlite3 is the default but again there's nothing stopping you dumping to a file and importing into Postgres or Mysql. 130 | 131 | Everything in a deployment is versioned right down to the web configuration files. The only thing that isn't versioned is your database and MEDIA_ROOT. If you get errors, from misconfiguration or package installs, you can just fix your issue and run it again until it completes and activates your environment. 132 | 133 | .. Note:: 134 | 135 | Versions are critical to woven, and how woven differs from most deployment tools. Woven deploys a separate virtualenv just like the one we created earlier for *each* version of your distribution. This means you don't destroy an existing working environment when you deploy a new version. You could use this feature to test different features, or simply to rollback from a failed release. Not that you'll ever have a failed release. Ever. 136 | 137 | You'll also notice woven has created a pip ``requirements.txt`` file and a ``sitesettings`` folder with a setting file inside. Requirements are your pip requirements for the project. The sitesettings.settings.py will import and override your local settings file on the node. 138 | 139 | Patch 140 | ------ 141 | 142 | Of course mistakes are made, but to avoid stupidity and overwriting a working installation you cannot re-deploy the same version of your project with deploy (though the ``--overwrite`` option will do the trick if you're desperate). To get around having to deploy a new version for small changes you can run: 143 | 144 | .. code-block:: bash 145 | 146 | woven-admin.py patch woven@example.com 147 | 148 | This will update existing files in your project, media and webserver configurations. It won't delete any files or update any dependencies. To update dependencies to a new library version you should increase your setup.py version and re-run deploy. 149 | 150 | Patch can also just upload a specific part of your project using a subcommand. For example to just patch your webconf files: 151 | 152 | .. code-block:: bash 153 | 154 | woven-admin.py patch webconf woven@example.com 155 | 156 | The different subcommands are ``project|static|media|templates|webconf`` 157 | 158 | Where to now 159 | ------------ 160 | 161 | If you want to work directly on the server (perhaps you need to debug something in staging) you can ``ssh woven@example.com -p10022`` into your host and type:: 162 | 163 | workon helloadmin 164 | 165 | This will use virtualenvwrapper to activate your current virtualenv and drop you into the project sitesettings manage.py directory. A convenience manage.py is provided to run ./manage.py from there on the first site. 166 | 167 | Of course installing packages from a requirements file can be problematic if pypi or a particular site is down . Make use of the ``woven-admin.py bundle`` command. This will use pip to bundle all the requirements into a dist folder in the distribution for deploy command to use. 168 | 169 | We also haven't covered in this tutorial features such as roles, integrated South migrations and multi-site creation with ``startsites``. Have a read of the woven django management :doc:`commands` for more. 170 | -------------------------------------------------------------------------------- /docs/woven/deployment.rst: -------------------------------------------------------------------------------- 1 | woven.deployment 2 | ================ 3 | 4 | .. automodule:: woven.deployment 5 | :members: -------------------------------------------------------------------------------- /docs/woven/linux.rst: -------------------------------------------------------------------------------- 1 | woven.linux 2 | ============ 3 | 4 | .. automodule:: woven.linux 5 | :members: -------------------------------------------------------------------------------- /docs/woven/main.rst: -------------------------------------------------------------------------------- 1 | woven.main 2 | =========== 3 | 4 | .. automodule:: woven.main 5 | :members: -------------------------------------------------------------------------------- /docs/woven/project.rst: -------------------------------------------------------------------------------- 1 | woven.project 2 | ============= 3 | 4 | .. automodule:: woven.project 5 | :members: -------------------------------------------------------------------------------- /docs/woven/virtualenv.rst: -------------------------------------------------------------------------------- 1 | woven.virtualenv 2 | ================ 3 | 4 | .. automodule:: woven.virtualenv 5 | :members: -------------------------------------------------------------------------------- /docs/woven/webservers.rst: -------------------------------------------------------------------------------- 1 | woven.webservers 2 | ================ 3 | 4 | .. automodule:: woven.webservers 5 | :members: -------------------------------------------------------------------------------- /docs/woven/woven.rst: -------------------------------------------------------------------------------- 1 | woven 2 | ===== 3 | 4 | .. automodule:: woven 5 | 6 | 7 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [build_sphinx] 2 | source-dir = docs 3 | build-dir = docs/_build 4 | all_files = 1 5 | 6 | [upload_sphinx] 7 | upload-dir = docs/_build/html -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from setuptools import setup, find_packages 3 | from woven import get_version 4 | 5 | setup(name='woven', 6 | version=get_version(), 7 | description='A deployment tool for Django built on Fabric', 8 | author='Brett Haydon', 9 | author_email='brett@haydon.id.au', 10 | url='http://github.com/bretth/woven', 11 | download_url='http://github.com/bretth/woven/downloads/', 12 | package_dir={'woven': 'woven'}, 13 | packages=find_packages(), 14 | scripts=['bin/woven-admin.py'], 15 | include_package_data = True, 16 | package_data={'woven': ['templates/woven/*.txt', 17 | 'templates/woven/nginx/*.conf', 18 | 'templates/woven/apache2/*.conf', 19 | 'templates/woven/ssh/sshd_config', 20 | 'templates/etc/*', 21 | 'templates/*.py', 22 | 'templates/distributions_template/*', 23 | ]}, 24 | classifiers=['Development Status :: 3 - Alpha', 25 | 'Environment :: Web Environment', 26 | 'Framework :: Django', 27 | 'Intended Audience :: Developers', 28 | 'License :: OSI Approved :: BSD License', 29 | 'Operating System :: OS Independent', 30 | 'Programming Language :: Python', 31 | 'Topic :: Software Development :: Libraries :: Python Modules', 32 | 'Topic :: Utilities'], 33 | long_description="See http://packages.python.org/woven/ for documentation", 34 | install_requires=['Fabric'], 35 | ) -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /tests/dec.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests the decorators.py module 3 | """ 4 | 5 | from fabric.api import settings, sudo 6 | 7 | from woven.decorators import run_once_per_node, run_once_per_version 8 | 9 | H = '192.168.188.10' 10 | HS = 'root@192.168.188.10:22' 11 | R = 'root' 12 | 13 | def teardown(): 14 | with settings(host_string=HS,user=R,password=R,project_fullname='example-0.2'): 15 | sudo('rm -rf /var/local/woven') 16 | 17 | def test_dec_run_once_per_node(): 18 | teardown() 19 | 20 | @run_once_per_node 21 | def test_func(): 22 | return 'some' 23 | 24 | with settings(host=H, host_string=HS,user=R,password=R,project_fullname='example-0.1'): 25 | assert test_func() == 'some' 26 | r = test_func() 27 | assert not r 28 | with settings(host=H,host_string=HS,user=R,password=R,project_fullname='example-0.2'): 29 | assert test_func() 30 | 31 | teardown() 32 | 33 | def test_dec_run_once_per_version(): 34 | teardown() 35 | 36 | @run_once_per_version 37 | def test_func(): 38 | return 'some' 39 | 40 | with settings(host=H,host_string=HS,user=R,password=R,project_fullname='example-0.1'): 41 | assert test_func() == 'some' 42 | with settings(host=H,host_string=HS,user=R,password=R,project_fullname='example-0.2'): 43 | r=test_func() 44 | assert r 45 | #run a second time 46 | assert not test_func() 47 | 48 | teardown() 49 | -------------------------------------------------------------------------------- /tests/dep.py: -------------------------------------------------------------------------------- 1 | from fabric.contrib.files import exists 2 | from fabric.api import sudo, settings 3 | 4 | from woven.deployment import _backup_file, _restore_file 5 | 6 | H = '192.168.188.10' 7 | HS = 'root@192.168.188.10:22' 8 | R = 'root' 9 | 10 | def test_dep_backup_file(): 11 | with settings(hosts=[H],host_string=HS,user=R,password=R): 12 | sudo('rm -rf /var/local/woven-backup') 13 | _backup_file('/etc/ssh/sshd_config') 14 | assert exists('/var/local/woven-backup/etc/ssh/sshd_config') 15 | sudo('rm -rf /var/local/woven-backup') 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /tests/env.py: -------------------------------------------------------------------------------- 1 | 2 | from fabric.api import * 3 | from fabric.state import env 4 | 5 | from woven.environment import _root_domain, _parse_project_version 6 | from woven.environment import set_env, server_state, set_server_state 7 | from woven.environment import version_state, set_version_state 8 | H = '192.168.188.10' 9 | HS = 'root@192.168.188.10:22' 10 | R = 'root' 11 | 12 | def setup(): 13 | sudo('rm -rf /var/local/woven') 14 | 15 | def teardown(): 16 | sudo('rm -rf /var/local/woven') 17 | 18 | def test_env_set_env(): 19 | print "TEST SET ENV" 20 | set_env() 21 | 22 | def test_env_server_state(): 23 | with settings(host_string=HS,user=R,password=R): 24 | setup() 25 | env.project_fullname = 'example_project-0.1' 26 | sudo('rm -rf /var/local/woven') 27 | #test 28 | set_server_state('example',delete=True) 29 | set_server_state('example') 30 | assert server_state('example') 31 | set_server_state('example',object=['something']) 32 | state = server_state('example') 33 | assert state == ['something'] 34 | set_server_state('example',delete=True) 35 | state = server_state('example') 36 | assert not state 37 | 38 | teardown() 39 | 40 | def test_env_version_state(): 41 | with settings(host_string=HS,user=R,password=R): 42 | setup() 43 | env.project_fullname = 'example_project-0.1' 44 | sudo('rm -rf /var/local/woven') 45 | #test 46 | set_version_state('example',delete=True) 47 | set_version_state('example') 48 | assert version_state('example') 49 | set_version_state('example',object=['something']) 50 | state = version_state('example') 51 | assert state == ['something'] 52 | state = version_state('example', prefix=True) 53 | assert state 54 | 55 | set_version_state('example',delete=True) 56 | state = version_state('example') 57 | assert not state 58 | teardown() 59 | 60 | 61 | def test_env_parse_project_version(): 62 | v = _parse_project_version('0.1') 63 | env.project_version = '' 64 | assert v == '0.1' 65 | v = _parse_project_version('0.1.0.1') 66 | env.project_version = '' 67 | assert v == '0.1' 68 | v = _parse_project_version('0.1 alpha') 69 | env.project_version = '' 70 | assert v =='0.1-alpha' 71 | v = _parse_project_version('0.1a 1234') 72 | env.project_version = '' 73 | assert v == '0.1a' 74 | v = _parse_project_version('0.1-alpha') 75 | env.project_version = '' 76 | assert v == '0.1-alpha' 77 | v = _parse_project_version('0.1 rc1 1234') 78 | env.project_version = '' 79 | assert v == '0.1-rc1' 80 | v = _parse_project_version('0.1.0rc1') 81 | env.project_version = '' 82 | assert v == '0.1.0rc1' 83 | v = _parse_project_version('0.1.1 rc2') 84 | env.project_version = '' 85 | assert v == '0.1.1-rc2' 86 | v = _parse_project_version('0.1.1.rc2.1234') 87 | env.project_version = '' 88 | assert v == '0.1.1.rc2' 89 | v = _parse_project_version('0.1.1-rc2.1234') 90 | env.project_version = '' 91 | assert v == '0.1.1-rc2' 92 | v = _parse_project_version('0.1.1-rc2-1234') 93 | env.project_version = '' 94 | assert v == '0.1.1-rc2' 95 | v = _parse_project_version('0.1.1 rc2 1234') 96 | assert v == '0.1.1-rc2' 97 | v = _parse_project_version('d.og') 98 | assert v == 'd.og' 99 | v = _parse_project_version('dog') 100 | assert v == 'dog' 101 | 102 | def test_env_root_domain(): 103 | with settings(hosts=[H],host_string=HS,user=R,password=R): 104 | #In the event of noinput, the domain will default to example.com 105 | domain = _root_domain() 106 | print domain 107 | assert domain == 'example.com' -------------------------------------------------------------------------------- /tests/fabfile.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Test runner for woven. 4 | 5 | Unit Tests don't appear to work with Fabric so all tests are run as fabfiles. 6 | In absence of a better alternative the tests are split into separate files 7 | with a specific naming strategy to make sure they run in groups. 8 | 9 | ``fab test`` will run all tests 10 | 11 | To run individual tests: 12 | 13 | ``fab test_[test name]`` 14 | 15 | Tests are prefixed with an abbreviated name of the module they are testing so 16 | that they run in groups, then alpha order. 17 | 18 | Test functions defined in this file should be less than 10 characters in length. 19 | 20 | """ 21 | import os, string, sys 22 | 23 | from django.utils import importlib 24 | from fabric.state import commands, env 25 | from woven.environment import set_env 26 | 27 | #import tests 28 | from env import test_env_set_env, test_env_server_state, test_env_parse_project_version, test_env_root_domain 29 | from env import test_env_version_state 30 | 31 | #from ubu import test_ubu_disable_root, test_ubu_change_ssh_port, test_ubu_port_is_open 32 | #from ubu import test_ubu_setup_ufw, test_ubu_post_install_package, test_ubu_post_setupnode 33 | 34 | from web import test_web_site_users 35 | from lin import test_lin_add_repositories, test_lin_uninstall_packages 36 | from lin import test_lin_setup_ufw_rules, test_lin_disable_root 37 | from dec import test_dec_run_once_per_node, test_dec_run_once_per_version 38 | from dep import test_dep_backup_file 39 | 40 | #Set the environ for Django 41 | settings_module = os.environ['DJANGO_SETTINGS_MODULE'] = 'example_project.setting' 42 | 43 | env.INTERACTIVE = False 44 | #Normally you would run fab or manage.py under the setup.py path 45 | #since we are testing outside the normal working directory we need to pass it in 46 | setup_dir = os.path.join(os.path.split(os.path.realpath(__file__))[0],'simplest_example') 47 | sys.path.insert(0,setup_dir) 48 | 49 | env.verbosity = 2 50 | #Most high level api functions require set_env to set the necessary woven environment 51 | set_env(setup_dir=setup_dir) 52 | 53 | def _run_tests(key=''): 54 | #Get a list of functions from fabric 55 | tests = commands.keys() 56 | for t in tests: 57 | if key: 58 | test_prefix = 'test_'+key+'_' 59 | else: 60 | test_prefix = 'test_' 61 | if test_prefix in t and len(t)>10: 62 | print string.upper(t) 63 | commands[t]() 64 | print string.upper(t), 'COMPLETE' 65 | def test(): 66 | """ 67 | Run all tests (in alpha order) 68 | """ 69 | _run_tests() 70 | 71 | def test_env(): 72 | """ 73 | Run all environment tests 74 | """ 75 | _run_tests('env') 76 | 77 | def test_lin(): 78 | """ 79 | Run all linux tests 80 | """ 81 | _run_tests('lin') 82 | 83 | def test_vir(): 84 | """ 85 | Run all virtualenv tests 86 | """ 87 | _run_tests('vir') 88 | 89 | def test_web(): 90 | """ 91 | Run all virtualenv tests 92 | """ 93 | _run_tests('web') 94 | 95 | def test_dec(): 96 | """ 97 | Run all decorator tests 98 | """ 99 | _run_tests('dec') 100 | 101 | def test_dep(): 102 | """ 103 | Run all deployment tests 104 | """ 105 | _run_tests('dep') 106 | 107 | 108 | 109 | 110 | -------------------------------------------------------------------------------- /tests/lin.py: -------------------------------------------------------------------------------- 1 | 2 | import os 3 | 4 | from fabric.api import * 5 | from fabric.contrib.files import uncomment, exists, comment, contains, append, sed 6 | from fabric.state import connections, env 7 | from fabric.network import join_host_strings, normalize 8 | 9 | from woven.linux import disable_root, change_ssh_port, port_is_open, setup_ufw 10 | from woven.linux import setup_ufw_rules 11 | from woven.linux import uninstall_packages 12 | from woven.linux import add_repositories 13 | 14 | from woven.environment import server_state, set_server_state 15 | 16 | H = '192.168.188.10' 17 | HS = 'woven@192.168.188.10:10022' 18 | R = 'woven' 19 | 20 | 21 | def test_lin_add_repositories(): 22 | add_repositories() 23 | #Step 1 in Server setup process 24 | 25 | def teardown_disable_root(): 26 | local('rm -rf .woven') 27 | with settings(host_string='woven@192.168.188.10:22',user='woven',password='woven'): 28 | run('echo %s:%s > /tmp/root_user.txt'% ('root','root')) 29 | sudo('chpasswd < /tmp/root_user.txt') 30 | sudo('rm -rf /tmp/root_user.txt') 31 | print "Closing connection %s"% env.host_string 32 | #print connections 33 | connections[env.host_string].close() 34 | try: 35 | connections['woven@example.com:22'].close() 36 | except: pass 37 | original_username = 'woven' 38 | (olduser,host,port) = normalize(env.host_string) 39 | host_string=join_host_strings('root',host,'22') 40 | with settings(host_string=host_string, password='root'): 41 | sudo('deluser --remove-home '+original_username) 42 | 43 | 44 | def test_lin_disable_root(): 45 | 46 | #automate 47 | env.DISABLE_ROOT = True 48 | env.INTERACTIVE = False 49 | env.HOST_PASSWORD = 'woven' 50 | env.ROOT_PASSWORD = 'root' 51 | 52 | #test 53 | with settings(host_string='woven@192.168.188.10:22',user='woven',password='woven'): 54 | disable_root() 55 | assert exists('/home/woven') 56 | 57 | #cleanup - re-enable root 58 | #teardown_disable_root() 59 | 60 | def test_lin_change_ssh_port(): 61 | 62 | #automate 63 | env.ROOT_PASSWORD = 'root' 64 | 65 | #setup 66 | host_state_dir = os.path.join(os.getcwd(),'.woven') 67 | host_state_path = os.path.join(host_state_dir,'example.com') 68 | if not os.path.exists(host_state_dir): 69 | os.mkdir(host_state_dir) 70 | open(host_state_path,'w').close() 71 | #test 72 | print "test_change_ssh_port" 73 | with settings(user='root',password=env.ROOT_PASSWORD): 74 | change_ssh_port() 75 | print "test logging in on the new port" 76 | 77 | with settings(host_string='root@192.168.188.10:10022',user='root',password=env.ROOT_PASSWORD): 78 | try: 79 | run('echo') 80 | except: 81 | print "\nTEST: change_ssh_port FAILED" 82 | return 83 | print 'CHANGE PASSED' 84 | with settings(user='root',password=env.ROOT_PASSWORD): 85 | result = change_ssh_port() 86 | print result 87 | assert result 88 | #teardown 89 | with settings(host_string='root@192.168.188.10:10022', user='root',password=env.ROOT_PASSWORD): 90 | sed('/etc/ssh/sshd_config','Port 10022','Port 22',use_sudo=True) 91 | sudo('/etc/init.d/ssh restart') 92 | local('rm -rf .woven') 93 | return 94 | 95 | def test_lin_port_is_open(): 96 | with settings(host_string='root@192.168.188.10:22', user='root',password=env.ROOT_PASSWORD): 97 | result = port_is_open() 98 | assert result 99 | 100 | sudo("echo 'Debian vers \n \l'> /etc/issue.new") 101 | sudo('cp -f /etc/issue /tmp/issue.bak') 102 | sudo('mv -f /etc/issue.new /etc/issue') 103 | 104 | result = port_is_open() 105 | 106 | sudo ('cp -f /tmp/issue.bak /etc/issue') 107 | 108 | 109 | #def test_lin_post_install_package(): 110 | # env.installed_packages = ['postgresql','somepackage'] 111 | # post_install_packages() 112 | 113 | #def test_lin_post_setupnode(): 114 | # post_setupnode() 115 | 116 | def test_lin_setup_ufw_rules(): 117 | #first define some rules that was in the settings 118 | UFW_RULES = ['allow from 127.0.0.1 to any app apache2', 'allow 5432/tcp'] 119 | 120 | with settings(packages=p,UFW_RULES=UFW_RULES, host_string=HS,user=R,password=R): 121 | setup_ufw_rules() 122 | 123 | 124 | def test_lin_setup_ufw(): 125 | with settings(host_string='root@192.168.188.10', user='root',password='root'): 126 | 127 | #tests 128 | env.HOST_SSH_PORT = '22' 129 | setup_ufw() 130 | r = sudo('ufw status').strip() 131 | assert 'woven' in r 132 | assert 'ALLOW' in r 133 | 134 | with settings(warn_only=True): 135 | 136 | sudo('ufw disable') 137 | sudo('rm -f /etc/ufw/applications.d/woven') 138 | sudo('rm -f /etc/ufw/applications.d/woven_project') 139 | apt_get_purge('ufw') 140 | set_server_state('ufw_installed',delete=True) 141 | 142 | #test change port 143 | print "CHANGE PORT to add 10022" 144 | env.HOST_SSH_PORT='22,10022' 145 | setup_ufw() 146 | r = sudo('ufw status verbose') 147 | assert '22,10022' in r 148 | assert '80,443' in r 149 | 150 | #test add an allow 151 | env.UFW_RULES = ['allow 5432/tcp'] 152 | setup_ufw() 153 | r = sudo('ufw status verbose') 154 | assert '5432' in r 155 | 156 | #teardown 157 | sudo('ufw disable') 158 | sudo('rm -f /etc/ufw/applications.d/woven') 159 | apt_get_purge('ufw') 160 | set_server_state('ufw_installed',delete=True) 161 | 162 | def test_lin_uninstall_packages(): 163 | uninstall_packages() 164 | 165 | -------------------------------------------------------------------------------- /tests/vir.py: -------------------------------------------------------------------------------- 1 | 2 | from fabric.state import env 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /tests/web.py: -------------------------------------------------------------------------------- 1 | from fabric.api import * 2 | 3 | from woven.webservers import _site_users 4 | from woven.linux import add_user 5 | 6 | def test_web_site_users(): 7 | with settings(host_string='root@192.168.188.10:22', user='root',password='root'): 8 | sudo('userdel site_1') 9 | users = _site_users() 10 | assert not users 11 | #now add a user 12 | add_user(username='site_1',group='www-data',site_user=True) 13 | users = _site_users() 14 | assert users[0] == 'site_1' 15 | -------------------------------------------------------------------------------- /woven/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | VERSION = (0, 9, 0,'alpha', 0) 4 | 5 | def get_version(): 6 | version = '%s.%s' % (VERSION[0], VERSION[1]) 7 | if VERSION[2]: 8 | version = '%s.%s' % (version, VERSION[2]) 9 | if VERSION[3:] == ('alpha', 0): 10 | version = '%s pre-alpha' % version 11 | else: 12 | if VERSION[3] != 'final': 13 | version = '%s %s %s' % (version, VERSION[3], VERSION[4]) 14 | return version 15 | -------------------------------------------------------------------------------- /woven/api.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | The full public woven api 4 | """ 5 | from fabric.state import env 6 | 7 | from woven.decorators import run_once_per_node, run_once_per_version 8 | 9 | from woven.deployment import deploy_files, mkdirs 10 | from woven.deployment import upload_template 11 | 12 | from woven.environment import check_settings, deployment_root, set_env, patch_project 13 | from woven.environment import get_project_version, server_state, set_server_state 14 | from woven.environment import set_version_state, version_state, get_packages 15 | from woven.environment import post_install_package, post_exec_hook 16 | 17 | from woven.project import deploy_static, deploy_media, deploy_project, deploy_db, deploy_templates 18 | 19 | from woven.linux import add_user, install_package, port_is_open, skip_disable_root 20 | from woven.linux import install_packages, uninstall_packages 21 | from woven.linux import upgrade_packages, setup_ufw, setup_ufw_rules, disable_root 22 | from woven.linux import add_repositories, restrict_ssh, upload_ssh_key 23 | from woven.linux import change_ssh_port, set_timezone, lsb_release, upload_etc 24 | 25 | from woven.virtualenv import activate, active_version 26 | from woven.virtualenv import mkvirtualenv, rmvirtualenv, pip_install_requirements 27 | 28 | 29 | from woven.webservers import deploy_wsgi, deploy_webconf, start_webserver, stop_webserver, reload_webservers 30 | from woven.webservers import webserver_list 31 | 32 | def deploy(overwrite=False): 33 | """ 34 | deploy a versioned project on the host 35 | """ 36 | check_settings() 37 | if overwrite: 38 | rmvirtualenv() 39 | deploy_funcs = [deploy_project,deploy_templates, deploy_static, deploy_media, deploy_webconf, deploy_wsgi] 40 | if not patch_project() or overwrite: 41 | deploy_funcs = [deploy_db,mkvirtualenv,pip_install_requirements] + deploy_funcs 42 | for func in deploy_funcs: func() 43 | 44 | 45 | def setupnode(overwrite=False): 46 | """ 47 | Install a baseline host. Can be run multiple times 48 | 49 | """ 50 | if not port_is_open(): 51 | if not skip_disable_root(): 52 | disable_root() 53 | port_changed = change_ssh_port() 54 | #avoid trying to take shortcuts if setupnode did not finish 55 | #on previous execution 56 | if server_state('setupnode-incomplete'): 57 | env.overwrite=True 58 | else: set_server_state('setupnode-incomplete') 59 | upload_ssh_key() 60 | restrict_ssh() 61 | add_repositories() 62 | upgrade_packages() 63 | setup_ufw() 64 | uninstall_packages() 65 | install_packages() 66 | 67 | upload_etc() 68 | post_install_package() 69 | setup_ufw_rules() 70 | set_timezone() 71 | set_server_state('setupnode-incomplete',delete=True) 72 | #stop and start webservers - and reload nginx 73 | for s in webserver_list(): 74 | stop_webserver(s) 75 | start_webserver(s) 76 | 77 | 78 | -------------------------------------------------------------------------------- /woven/decorators.py: -------------------------------------------------------------------------------- 1 | from functools import wraps 2 | 3 | from fabric.api import env 4 | from woven.environment import server_state, set_server_state 5 | from woven.environment import version_state, set_version_state 6 | 7 | def run_once_per_node(func): 8 | """ 9 | Decorator preventing wrapped function from running more than 10 | once per host (not just interpreter session). 11 | 12 | Using env.patch = True will allow the wrapped function to be run 13 | if it has been previously executed, but not otherwise 14 | 15 | Stores the result of a function as server state 16 | """ 17 | @wraps(func) 18 | def decorated(*args, **kwargs): 19 | if not hasattr(env,'patch'): env.patch = False 20 | state = version_state(func.__name__) 21 | if not env.patch and state: 22 | verbose = " ".join([env.host,func.__name__,"completed. Skipping..."]) 23 | elif env.patch and not state: 24 | verbose = " ".join([env.host,func.__name__,"not previously completed. Skipping..."]) 25 | else: 26 | results = func(*args, **kwargs) 27 | verbose ='' 28 | if results: set_version_state(func.__name__,object=results) 29 | else: set_version_state(func.__name__) 30 | return results 31 | if env.verbosity and verbose: print verbose 32 | return 33 | 34 | return decorated 35 | 36 | def run_once_per_version(func): 37 | """ 38 | Decorator preventing wrapped function from running more than 39 | once per host and env.project_fullname (not just interpreter session). 40 | 41 | Using env.patch = True will allow the function to be run 42 | 43 | Stores the result of a function as server state 44 | """ 45 | @wraps(func) 46 | def decorated(*args, **kwargs): 47 | if not hasattr(env,'patch'): env.patch = False 48 | state = version_state(func.__name__) 49 | if not env.patch and state: 50 | verbose = " ".join([env.host,func.__name__,"completed. Skipping..."]) 51 | elif env.patch and not state: 52 | verbose = " ".join([env.host,func.__name__,"not previously completed. Skipping..."]) 53 | else: 54 | results = func(*args, **kwargs) 55 | verbose ='' 56 | if results: set_version_state(func.__name__,object=results) 57 | else: set_version_state(func.__name__) 58 | return results 59 | if env.verbosity and verbose: print verbose 60 | return 61 | 62 | return decorated -------------------------------------------------------------------------------- /woven/deploy.py: -------------------------------------------------------------------------------- 1 | from fabric.state import env 2 | from fabric.api import sudo, settings 3 | 4 | def post_install_postgresql(): 5 | """ 6 | example default hook for installing postgresql 7 | """ 8 | from django.conf import settings as s 9 | with settings(warn_only=True): 10 | sudo('/etc/init.d/postgresql-8.4 restart') 11 | sudo("""psql template1 -c "ALTER USER postgres with encrypted password '%s';" """% env.password, user='postgres') 12 | sudo("psql -f /usr/share/postgresql/8.4/contrib/adminpack.sql", user='postgres') 13 | if s.DATABASES['default']['ENGINE']=='django.db.backends.postgresql_psycopg2': 14 | sudo("""psql template1 -c "CREATE ROLE %s LOGIN with encrypted password '%s';" """% (s.DATABASES['default']['USER'],s.DATABASES['default']['PASSWORD']), user='postgres') 15 | sudo('createdb -T template0 -O %s %s'% (s.DATABASES['default']['USER'],s.DATABASES['default']['NAME']), user='postgres') 16 | 17 | print "* setup postgres user password with your '%s' password"% env.user 18 | print "* imported the adminpack" 19 | print "Post install setup of Postgresql complete!" 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /woven/deployment.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from functools import wraps 3 | from glob import glob 4 | from hashlib import sha1 5 | import os, shutil, sys, tempfile 6 | 7 | from django.template.loader import render_to_string 8 | 9 | from fabric.state import env 10 | from fabric.operations import run, sudo, put 11 | from fabric.context_managers import cd, settings, hide 12 | from fabric.contrib.files import exists 13 | from fabric.contrib.project import rsync_project 14 | 15 | def _backup_file(path): 16 | """ 17 | Backup a file but never overwrite an existing backup file 18 | """ 19 | backup_base = '/var/local/woven-backup' 20 | backup_path = ''.join([backup_base,path]) 21 | if not exists(backup_path): 22 | directory = ''.join([backup_base,os.path.split(path)[0]]) 23 | sudo('mkdir -p %s'% directory) 24 | sudo('cp %s %s'% (path,backup_path)) 25 | 26 | def _restore_file(path, delete_backup=True): 27 | """ 28 | Restore a file if it exists and remove the backup 29 | """ 30 | backup_base = '/var/local/woven-backup' 31 | backup_path = ''.join([backup_base,path]) 32 | if exists(backup_path): 33 | if delete_backup: 34 | sudo('mv -f %s %s'% (backup_path,path)) 35 | else: 36 | sudo('cp -f %s %s'% (backup_path,path)) 37 | 38 | 39 | def _get_local_files(local_dir, pattern=''): 40 | """ 41 | Returns a dictionary with directories as keys, and filenames as values 42 | for filenames matching the glob ``pattern`` under the ``local_dir`` 43 | ``pattern can contain the Boolean OR | to evaluated multiple patterns into 44 | a combined set. 45 | """ 46 | local_files = {} 47 | 48 | if pattern: 49 | cwd = os.getcwd() 50 | os.chdir(local_dir) 51 | patterns = pattern.split('|') 52 | local_list = set([]) 53 | for p in patterns: local_list = local_list | set(glob(p)) 54 | for path in local_list: 55 | dir, file = os.path.split(path) 56 | if os.path.isfile(path): 57 | local_files[dir] = local_files.get(dir,[])+[file] 58 | elif os.path.isdir(path): 59 | local_files[file] = local_files.get(dir,[]) 60 | os.chdir(cwd) 61 | return local_files 62 | 63 | def _stage_local_files(local_dir, local_files={}): 64 | """ 65 | Either ``local_files`` and/or ``context`` should be supplied. 66 | 67 | Will stage a ``local_files`` dictionary of path:filename pairs where path 68 | is relative to ``local_dir`` into a local tmp staging directory. 69 | 70 | Returns a path to the temporary local staging directory 71 | 72 | """ 73 | staging_dir = os.path.join(tempfile.mkdtemp(),os.path.basename(local_dir)) 74 | os.mkdir(staging_dir) 75 | for root, dirs, files in os.walk(local_dir): 76 | relative_tree = root.replace(local_dir,'') 77 | if relative_tree: 78 | relative_tree = relative_tree[1:] 79 | if local_files: 80 | files = local_files.get(relative_tree,[]) 81 | for file in files: 82 | if relative_tree: 83 | filepath = os.path.join(relative_tree,file) 84 | if not os.path.exists(os.path.join(staging_dir,relative_tree)): 85 | os.mkdir(os.path.join(staging_dir,relative_tree)) 86 | else: filepath = file 87 | shutil.copy2(os.path.join(root,file),os.path.join(staging_dir,filepath)) 88 | return staging_dir 89 | 90 | def deploy_files(local_dir, remote_dir, pattern = '',rsync_exclude=['*.pyc','.*'], use_sudo=False): 91 | """ 92 | Generic deploy function for cases where one or more files are being deployed to a host. 93 | Wraps around ``rsync_project`` and stages files locally and/or remotely 94 | for network efficiency. 95 | 96 | ``local_dir`` is the directory that will be deployed. 97 | 98 | ``remote_dir`` is the directory the files will be deployed to. 99 | Directories will be created if necessary. 100 | 101 | Note: Unlike other ways of deploying files, all files under local_dir will be 102 | deployed into remote_dir. This is the equivalent to cp -R local_dir/* remote_dir. 103 | 104 | ``pattern`` finds all the pathnames matching a specified glob pattern relative 105 | to the local_dir according to the rules used by the Unix shell. 106 | ``pattern`` enhances the basic functionality by allowing the python | to include 107 | multiple patterns. eg '*.txt|Django*' 108 | 109 | ``rsync_exclude`` as per ``rsync_project`` 110 | 111 | Returns a list of directories and files created on the host. 112 | 113 | """ 114 | #normalise paths 115 | if local_dir[-1] == os.sep: local_dir = local_dir[:-1] 116 | if remote_dir[-1] == '/': remote_dir = remote_dir[:-1] 117 | created_list = [] 118 | staging_dir = local_dir 119 | 120 | #resolve pattern into a dir:filename dict 121 | local_files = _get_local_files(local_dir,pattern) 122 | #If we are only copying specific files or rendering templates we need to stage locally 123 | if local_files: staging_dir = _stage_local_files(local_dir, local_files) 124 | remote_staging_dir = '/home/%s/.staging'% env.user 125 | if not exists(remote_staging_dir): 126 | run(' '.join(['mkdir -pv',remote_staging_dir])).split('\n') 127 | created_list = [remote_staging_dir] 128 | 129 | #upload into remote staging 130 | rsync_project(local_dir=staging_dir,remote_dir=remote_staging_dir,exclude=rsync_exclude,delete=True) 131 | 132 | #create the final destination 133 | created_dir_list = mkdirs(remote_dir, use_sudo) 134 | 135 | if not os.listdir(staging_dir): return created_list 136 | 137 | func = use_sudo and sudo or run 138 | #cp recursively -R from the staging to the destination and keep a list 139 | remote_base_path = '/'.join([remote_staging_dir,os.path.basename(local_dir),'*']) 140 | copy_file_list = func(' '.join(['cp -Ruv',remote_base_path,remote_dir])).split('\n') 141 | if copy_file_list[0]: created_list += [file.split(' ')[2][1:-1] for file in copy_file_list if file] 142 | 143 | #cleanup any tmp staging dir 144 | if staging_dir <> local_dir: 145 | shutil.rmtree(staging_dir,ignore_errors=True) 146 | 147 | return created_list 148 | 149 | def mkdirs(remote_dir, use_sudo=False): 150 | """ 151 | Wrapper around mkdir -pv 152 | 153 | Returns a list of directories created 154 | """ 155 | func = use_sudo and sudo or run 156 | result = func(' '.join(['mkdir -pv',remote_dir])).split('\n') 157 | #extract dir list from ["mkdir: created directory `example.com/some/dir'"] 158 | if result[0]: result = [dir.split(' ')[3][1:-1] for dir in result if dir] 159 | return result 160 | 161 | def upload_template(filename, destination, context={}, use_sudo=False, backup=True, modified_only=False): 162 | """ 163 | Render and upload a template text file to a remote host using the Django 164 | template api. 165 | 166 | ``filename`` should be the Django template name. 167 | 168 | ``context`` is the Django template dictionary context to use. 169 | 170 | The resulting rendered file will be uploaded to the remote file path 171 | ``destination`` (which should include the desired remote filename.) If the 172 | destination file already exists, it will be renamed with a ``.bak`` 173 | extension. 174 | 175 | By default, the file will be copied to ``destination`` as the logged-in 176 | user; specify ``use_sudo=True`` to use `sudo` instead. 177 | """ 178 | #Replaces the default fabric.contrib.files.upload_template 179 | basename = os.path.basename(filename) 180 | text = render_to_string(filename,context) 181 | 182 | func = use_sudo and sudo or run 183 | 184 | #check hashed template on server first 185 | if modified_only: 186 | hashfile_dir, hashfile = os.path.split(destination) 187 | hashfile_dir = ''.join(['/var/local/woven-backup',hashfile_dir]) 188 | hashfile = '%s.hashfile'% hashfile 189 | hashfile_path = os.path.join(hashfile_dir, hashfile) 190 | hashed = sha1(text).hexdigest() 191 | if hashfile: 192 | if not exists(hashfile_dir): sudo('mkdir -p %s'% hashfile_dir) 193 | sudo('touch %s'% hashfile_path) #store the hash near the template 194 | previous_hashed = sudo('cat %s'% hashfile_path).strip() 195 | if previous_hashed == hashed: 196 | return False 197 | else: sudo('echo %s > %s'% (hashed, hashfile_path)) 198 | 199 | temp_destination = '/tmp/' + basename 200 | 201 | # This temporary file should not be automatically deleted on close, as we 202 | # need it there to upload it (Windows locks the file for reading while open). 203 | tempfile_fd, tempfile_name = tempfile.mkstemp() 204 | output = open(tempfile_name, "w+b") 205 | 206 | output.write(text) 207 | output.close() 208 | 209 | # Upload the file. 210 | put(tempfile_name, temp_destination) 211 | os.close(tempfile_fd) 212 | os.remove(tempfile_name) 213 | 214 | 215 | # Back up any original file (need to do figure out ultimate destination) 216 | if backup: 217 | to_backup = destination 218 | with settings(hide('everything'), warn_only=True): 219 | # Is destination a directory? 220 | if func('test -f %s' % to_backup).failed: 221 | # If so, tack on the filename to get "real" destination 222 | to_backup = destination + '/' + basename 223 | if exists(to_backup): 224 | _backup_file(to_backup) 225 | # Actually move uploaded template to destination 226 | func("mv %s %s" % (temp_destination, destination)) 227 | return True -------------------------------------------------------------------------------- /woven/management/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | -------------------------------------------------------------------------------- /woven/management/base.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | 4 | from optparse import make_option 5 | 6 | from django.conf import settings 7 | from django.core.management.base import BaseCommand 8 | from django.core.management.color import no_style 9 | 10 | from fabric import state 11 | from fabric.network import normalize 12 | from fabric.context_managers import hide,show 13 | 14 | from woven.environment import set_env 15 | 16 | class WovenCommand(BaseCommand): 17 | option_list = BaseCommand.option_list + ( 18 | make_option('--noinput', action='store_false', dest='interactive', default=True, 19 | help='Do NOT prompt for input (except password entry if required)'), 20 | make_option('-r', '--reject-unknown-hosts', 21 | action='store_true', 22 | default=False, 23 | help="reject unknown hosts" 24 | ), 25 | 26 | make_option('-D', '--disable-known-hosts', 27 | action='store_true', 28 | default=False, 29 | help="do not load user known_hosts file" 30 | ), 31 | make_option('-i', 32 | action='append', 33 | dest='key_filename', 34 | default=None, 35 | help="path to SSH private key file." 36 | ), 37 | 38 | make_option('-u', '--user', 39 | default=state._get_system_username(), 40 | help="username to use when connecting to remote hosts" 41 | ), 42 | 43 | make_option('-p', '--password', 44 | default=None, 45 | help="password for use with authentication and/or sudo" 46 | ), 47 | 48 | make_option('--setup', 49 | help='The /path/to/dir containing the setup.py module. The command will execute from this directory. Only required if you are not executing the command from below the setup.py directory', 50 | ), 51 | 52 | ) 53 | help = "" 54 | args = "host1 [host2 ...] or user@host1 ..." 55 | requires_model_validation = False 56 | 57 | 58 | def handle_host(self, *args, **options): 59 | """ 60 | This will be executed per host - override in subclass 61 | """ 62 | def parse_host_args(self, *args): 63 | """ 64 | Returns a comma separated string of hosts 65 | """ 66 | return ','.join(args) 67 | 68 | def handle(self, *args, **options): 69 | """ 70 | Initializes the fabric environment 71 | """ 72 | self.style = no_style() 73 | #manage.py execution specific variables 74 | #verbosity 0 = No output at all, 1 = woven output only, 2 = Fabric outputlevel = everything except debug 75 | state.env.verbosity = int(options.get('verbosity', 1)) 76 | 77 | #show_traceback = options.get('traceback', False) 78 | state.env.INTERACTIVE = options.get('interactive') 79 | 80 | #Fabric options 81 | #Django passes in a dictionary instead of the optparse options objects 82 | for option in options: 83 | state.env[option] = options[option] 84 | 85 | #args will be tuple. We convert it to a comma separated string for fabric 86 | all_role_hosts = [] 87 | 88 | if args: 89 | #subclasses can implement parse_host_args to strip out subcommands 90 | comma_hosts = self.parse_host_args(*args) 91 | normalized_host_list = comma_hosts.split(',') 92 | for r in normalized_host_list: 93 | #define a list of hosts for given roles 94 | if hasattr(settings,'ROLEDEFS') and settings.ROLEDEFS.get(r): 95 | all_role_hosts+=settings.ROLEDEFS[r] 96 | state.env['roles'] = state.env['roles'] + [r] 97 | #simple single host 98 | else: 99 | all_role_hosts.append(r) 100 | 101 | #if no args are given we'll use either a 'default' roledef/role_node 102 | #or as last resort we'll use a simple HOSTS list 103 | elif hasattr(settings, 'ROLEDEFS') and settings.ROLEDEFS.get('default'): 104 | all_role_hosts = settings.ROLEDEFS['default'] 105 | state.env['roles'] = ['default'] 106 | elif hasattr(settings,'HOSTS') and settings.HOSTS: 107 | all_role_hosts = settings.HOSTS 108 | else: 109 | print "Error: You must include a host or role in the command line or set HOSTS or ROLEDEFS in your settings file" 110 | sys.exit(1) 111 | state.env['hosts'] = all_role_hosts 112 | 113 | #This next section is taken pretty much verbatim from fabric.main 114 | #so we follow an almost identical but more limited execution strategy 115 | 116 | #We now need to load django project woven settings into env 117 | #This is the equivalent to module level execution of the fabfile.py. 118 | #If we were using a fabfile.py then we would include set_env() 119 | 120 | if int(state.env.verbosity) < 2: 121 | with hide('warnings', 'running', 'stdout', 'stderr'): 122 | set_env(settings,state.env.setup) 123 | else: set_env(settings,state.env.setup) 124 | 125 | #Back to the standard execution strategy 126 | # Set host list (also copy to env) 127 | state.env.all_hosts = hosts = state.env.hosts 128 | # If hosts found, execute the function on each host in turn 129 | for host in hosts: 130 | # Preserve user 131 | prev_user = state.env.user 132 | # Split host string and apply to env dict 133 | #TODO - This section is replaced by network.interpret_host_string in Fabric 1.0 134 | username, hostname, port = normalize(host) 135 | state.env.host_string = host 136 | state.env.host = hostname 137 | state.env.user = username 138 | state.env.port = port 139 | 140 | # Actually run command 141 | if int(state.env.verbosity) < 2: 142 | with hide('warnings', 'running', 'stdout', 'stderr'): 143 | self.handle_host(*args, **options) 144 | else: 145 | self.handle_host(*args, **options) 146 | # Put old user back 147 | state.env.user = prev_user 148 | -------------------------------------------------------------------------------- /woven/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | -------------------------------------------------------------------------------- /woven/management/commands/activate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from optparse import make_option 3 | 4 | from fabric.state import env 5 | 6 | from woven.environment import project_version 7 | from woven.virtualenv import activate 8 | from woven.management.base import WovenCommand 9 | 10 | class Command(WovenCommand): 11 | """ 12 | Active a project version 13 | 14 | e.g. python manage.py activate 0.1 15 | """ 16 | 17 | help = "Activate a version of your project" 18 | requires_model_validation = False 19 | args = "version user@ipaddress [host2...]" 20 | 21 | def parse_host_args(self, *args): 22 | """ 23 | Returns a comma separated string of hosts 24 | """ 25 | return ','.join(args[1:]) 26 | 27 | def handle_host(self,*args, **options): 28 | vers = args[0] 29 | env.nomigration = True 30 | with project_version(vers): 31 | activate() 32 | 33 | return -------------------------------------------------------------------------------- /woven/management/commands/bundle.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from optparse import make_option 3 | from glob import glob 4 | import os 5 | 6 | from django.core.management.base import BaseCommand 7 | from django.core.management.color import no_style 8 | 9 | from fabric import state 10 | from fabric.operations import local 11 | from fabric.context_managers import hide 12 | 13 | from woven.environment import set_env 14 | 15 | class Command(BaseCommand): 16 | """ 17 | Pip bundle your requirements into .pybundles for efficient deployment 18 | 19 | python manage.py bundle 20 | """ 21 | help = "Pip bundle your requirements into .pybundles for efficient deployment" 22 | args = "" 23 | requires_model_validation = False 24 | 25 | def handle(self, *args, **options): 26 | 27 | self.style = no_style() 28 | #manage.py execution specific variables 29 | #verbosity 0 = No output at all, 1 = woven output only, 2 = Fabric outputlevel = everything except debug 30 | state.env.verbosity = int(options.get('verbosity', 1)) 31 | 32 | #show_traceback = options.get('traceback', False) 33 | set_env.no_domain = True 34 | state.env.INTERACTIVE = options.get('interactive') 35 | if int(state.env.verbosity) < 2: 36 | with hide('warnings', 'running', 'stdout', 'stderr'): 37 | set_env() 38 | else: 39 | set_env() 40 | if not state.env.PIP_REQUIREMENTS: req_files = glob('req*') 41 | else: req_files = state.env.PIP_REQUIREMENTS 42 | dist_dir = os.path.join(os.getcwd(),'dist') 43 | if not os.path.exists(dist_dir): 44 | os.mkdir(dist_dir) 45 | for r in req_files: 46 | bundle = ''.join([r.split('.')[0],'.zip']) 47 | command = 'pip bundle -r %s %s/%s'% (r,dist_dir,bundle) 48 | if state.env.verbosity: print command 49 | if int(state.env.verbosity) < 2: 50 | with hide('warnings', 'running', 'stdout', 'stderr'): 51 | local(command) 52 | else: 53 | local(command) 54 | 55 | -------------------------------------------------------------------------------- /woven/management/commands/deploy.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from optparse import make_option 3 | 4 | from fabric.context_managers import settings 5 | 6 | from woven.api import deploy 7 | from woven.virtualenv import activate 8 | from woven.management.base import WovenCommand 9 | 10 | 11 | class Command(WovenCommand): 12 | """ 13 | Deploy your project to a host and activate 14 | 15 | Basic Usage: 16 | ``python manage.py deploy [user]@[hoststring]`` 17 | 18 | Examples: 19 | ``python manage.py deploy woven@192.168.188.10`` 20 | ``python manage.py deploy woven@host.example.com`` 21 | 22 | For just the current user 23 | ``python manage.py deploy host.example.com`` 24 | 25 | """ 26 | option_list = WovenCommand.option_list + ( 27 | make_option('-m', '--migration', 28 | default='', #use south default run all migrations 29 | help="Specify a specific migration to run" 30 | ), 31 | make_option('--fake', 32 | action='store_true', 33 | default=False, 34 | help="Fake the south migration. Useful when converting an app" 35 | ), 36 | make_option('--nomigration', 37 | action='store_true', 38 | default=False, 39 | help="Do not run any migration" 40 | ), 41 | make_option('--manualmigration', 42 | action='store_true', 43 | default=False, 44 | help="Manage the database migration manually" 45 | ), 46 | make_option('--overwrite', 47 | action='store_true', 48 | default=False, 49 | help="Overwrite an existing installation" 50 | ), 51 | 52 | ) 53 | help = "Deploy the current version of your project" 54 | requires_model_validation = False 55 | 56 | def handle_host(self,*args, **options): 57 | self.validate() 58 | deploy(overwrite=options.get('overwrite')) 59 | 60 | with settings(nomigration = options.get('nomigration'), 61 | migration = options.get('migration'), 62 | manualmigration = options.get('manualmigration')): 63 | activate() 64 | 65 | -------------------------------------------------------------------------------- /woven/management/commands/node.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Node command to execute arbitrary commands on a host. 4 | """ 5 | from optparse import make_option 6 | from fabric.state import env 7 | from fabric.context_managers import cd 8 | from fabric.operations import run 9 | 10 | from woven.management.base import WovenCommand 11 | 12 | class Command(WovenCommand): 13 | """ 14 | Run a management command on a host 15 | 16 | Basic Usage: 17 | ``python manage.py node [user]@[hoststring] --options="[option ...]"`` 18 | 19 | """ 20 | option_list = WovenCommand.option_list + ( 21 | make_option('--options', 22 | help='Store all the management command options in a string. ie --options="--[opt]=[value] ..."'), 23 | ) 24 | help = """Execute a management command on one or more hosts"""\ 25 | """ Must not require user input""" 26 | requires_model_validation = False 27 | 28 | args = "command user@ipaddress ..." 29 | 30 | def parse_host_args(self, *args): 31 | """ 32 | Splits out the management command and returns a comma separated list of host_strings 33 | """ 34 | #This overrides the base command 35 | return ','.join(args[1:]) 36 | 37 | def handle_host(self,*args, **options): 38 | opts = options.get('options') 39 | command = args[0] 40 | path = '/home/%s/%s/env/%s/project/%s/'% (env.user,root_domain(),env.project_fullname,env.project_name) 41 | pythonpath = '/home/%s/%s/env/%s/bin/python'% (env.user,env.root_domain,env.project_fullname) 42 | with cd(path): 43 | result = run(' '.join([pythonpath,'manage.py',command,opts])) 44 | if env.verbosity: 45 | print result 46 | -------------------------------------------------------------------------------- /woven/management/commands/patch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from optparse import make_option 3 | 4 | from fabric.context_managers import settings 5 | 6 | from woven.api import deploy, activate 7 | from woven.api import deploy_project, deploy_templates, deploy_static, deploy_media 8 | from woven.api import deploy_wsgi, deploy_webconf 9 | 10 | from woven.management.base import WovenCommand 11 | 12 | class Command(WovenCommand): 13 | """ 14 | Patch the current version of your project on hosts and restart webservices 15 | Includes project, web configuration, media, and wsgi but does not pip install 16 | 17 | Basic Usage: 18 | ``python manage.py patch [user]@[hoststring]`` 19 | 20 | Examples: 21 | ``python manage.py patch woven@192.168.188.10`` 22 | ``python manage.py patch woven@host.example.com`` 23 | 24 | For just the current user 25 | ``python manage.py patch host.example.com`` 26 | 27 | """ 28 | 29 | help = "Patch all parts of the current version of your project, or patch part of the project" 30 | args = "[project|templates|static|media|wsgi|webconf] [user@hoststring ...]" 31 | requires_model_validation = False 32 | 33 | def parse_host_args(self, *args): 34 | """ 35 | Splits out the patch subcommand and returns a comma separated list of host_strings 36 | """ 37 | self.subcommand = None 38 | new_args = args 39 | try: 40 | sub = args[0] 41 | if sub in ['project','templates','static','media','wsgi','webconf']: 42 | self.subcommand = args[0] 43 | new_args = args[1:] 44 | except IndexError: 45 | pass 46 | 47 | return ','.join(new_args) 48 | 49 | def handle_host(self,*args, **options): 50 | with settings(patch=True): 51 | if not self.subcommand: 52 | deploy() 53 | activate() 54 | else: 55 | eval(''.join(['deploy_',self.subcommand,'()'])) 56 | activate() 57 | 58 | 59 | -------------------------------------------------------------------------------- /woven/management/commands/setupnode.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from optparse import make_option 3 | 4 | from django.utils import importlib 5 | 6 | from fabric import state 7 | from fabric.context_managers import settings 8 | 9 | from woven.api import setupnode, post_exec_hook 10 | from woven.management.base import WovenCommand 11 | 12 | class Command(WovenCommand): 13 | """ 14 | Setup a baseline linux server ready for deployment 15 | 16 | Basic Usage: 17 | ``python manage.py setupnode [user]@[hoststring]`` 18 | 19 | Examples: 20 | ``python manage.py setupnode woven@192.168.188.10`` 21 | ``python manage.py setupnode woven@host.example.com`` 22 | 23 | """ 24 | option_list = WovenCommand.option_list + ( 25 | make_option('--root_disabled', 26 | action='store_true', 27 | default=False, 28 | help="Skip user creation and root disable" 29 | ), 30 | 31 | ) 32 | help = "Setup a baseline linux host" 33 | requires_model_validation = False 34 | 35 | def handle_host(self,*args, **options): 36 | state.env.root_disabled = options.get('root_disabled') 37 | setupnode() 38 | post_exec_hook('post_setupnode') 39 | 40 | 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /woven/management/commands/startsites.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from optparse import make_option 3 | import os 4 | 5 | from fabric import state 6 | from fabric.decorators import runs_once 7 | from fabric.context_managers import settings 8 | from fabric.operations import sudo 9 | from fabric.contrib.files import exists 10 | 11 | from woven.management.base import WovenCommand 12 | from woven.webservers import _get_django_sites, deploy_wsgi, deploy_webconf, domain_sites, reload_webservers 13 | from woven.project import deploy_sitesettings 14 | 15 | class Command(WovenCommand): 16 | """ 17 | Create sitesetting files for new django.contrib.sites. 18 | 19 | In django site creation is through the production database. The startsite command 20 | creates the sitesetting files for each of the new sites, and deploys them. 21 | 22 | Basic Usage: 23 | ``python manage.py startsite [hoststring|role]`` 24 | 25 | """ 26 | help = "Create a sitesetting file for new django sites" 27 | requires_model_validation = False 28 | 29 | def handle_host(self,*args, **options): 30 | with settings(patch=True): 31 | deploy_wsgi() 32 | deploy_webconf() 33 | 34 | activate_sites = [''.join([d.name.replace('.','_'),'-',state.env.project_version,'.conf']) for d in domain_sites()] 35 | site_paths = ['/etc/apache2','/etc/nginx'] 36 | 37 | #activate new sites 38 | for path in site_paths: 39 | for site in activate_sites: 40 | if not exists('/'.join([path,'sites-enabled',site])): 41 | sudo("chmod 644 %s" % '/'.join([path,'sites-available',site])) 42 | sudo("ln -s %s/sites-available/%s %s/sites-enabled/%s"% (path,site,path,site)) 43 | if state.env.verbosity: 44 | print " * enabled", "%s/sites-enabled/%s"% (path,site) 45 | reload_webservers() 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /woven/project.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Anything related to deploying your project modules, media, and data 4 | """ 5 | import os, shutil, sys 6 | 7 | from django.template.loader import render_to_string 8 | 9 | from fabric.state import env 10 | from fabric.operations import local, run, put, sudo 11 | from fabric.decorators import runs_once 12 | from fabric.contrib.files import exists 13 | from fabric.contrib.console import confirm 14 | #Required for a bug in 0.9 15 | from fabric.version import get_version 16 | 17 | from woven.decorators import run_once_per_version 18 | from woven.deployment import deploy_files 19 | from woven.environment import deployment_root, _root_domain 20 | 21 | @runs_once 22 | def _make_local_sitesettings(overwrite=False): 23 | local_settings_dir = os.path.join(os.getcwd(),env.project_package_name,'sitesettings') 24 | if not os.path.exists(local_settings_dir) or overwrite: 25 | if overwrite: 26 | shutil.rmtree(local_settings_dir,ignore_errors=True) 27 | os.mkdir(local_settings_dir) 28 | f = open(os.path.join(local_settings_dir,'__init__.py'),"w") 29 | f.close() 30 | 31 | settings_file_path = os.path.join(local_settings_dir,'settings.py') 32 | if not os.path.exists(settings_file_path): 33 | root_domain = _root_domain() 34 | u_domain = root_domain.replace('.','_') 35 | output = render_to_string('woven/sitesettings.txt', 36 | {"deployment_root":deployment_root(), 37 | "site_id":"1", 38 | "project_name": env.project_name, 39 | "project_fullname": env.project_fullname, 40 | "project_package_name": env.project_package_name, 41 | "u_domain":u_domain, 42 | "domain":root_domain, 43 | "user":env, 44 | "MEDIA_URL":env.MEDIA_URL, 45 | "STATIC_URL":env.STATIC_URL} 46 | ) 47 | 48 | f = open(settings_file_path,"w+") 49 | f.writelines(output) 50 | f.close() 51 | if env.verbosity: 52 | print "Created local sitesettings folder and default settings file" 53 | #copy manage.py into that directory 54 | manage_path = os.path.join(os.getcwd(),env.project_package_name,'manage.py') 55 | dest_manage_path = os.path.join(os.getcwd(),env.project_package_name,'sitesettings','manage.py') 56 | shutil.copy(manage_path, dest_manage_path) 57 | 58 | return 59 | 60 | @run_once_per_version 61 | def deploy_project(): 62 | """ 63 | Deploy to the project directory in the virtualenv 64 | """ 65 | 66 | project_root = '/'.join([deployment_root(),'env',env.project_fullname,'project']) 67 | local_dir = os.getcwd() 68 | 69 | if env.verbosity: 70 | print env.host,"DEPLOYING project", env.project_fullname 71 | #Exclude a few things that we don't want deployed as part of the project folder 72 | rsync_exclude = ['local_settings*','*.pyc','*.log','.*','/build','/dist','/media*','/static*','/www','/public','/template*'] 73 | 74 | #make site local settings if they don't already exist 75 | _make_local_sitesettings() 76 | created = deploy_files(local_dir, project_root, rsync_exclude=rsync_exclude) 77 | if not env.patch: 78 | #hook the project into sys.path 79 | pyvers = run('python -V').split(' ')[1].split('.')[0:2] #Python x.x.x 80 | sitepackages = ''.join(['lib/python',pyvers[0],'.',pyvers[1],'/site-packages']) 81 | link_name = '/'.join([deployment_root(),'env',env.project_fullname,sitepackages,env.project_package_name]) 82 | target = '/'.join([project_root,env.project_package_name]) 83 | run(' '.join(['ln -s',target,link_name])) 84 | 85 | #make sure manage.py has exec permissions 86 | managepy = '/'.join([target,'sitesettings','manage.py']) 87 | if exists(managepy): 88 | sudo('chmod ugo+x %s'% managepy) 89 | 90 | return created 91 | 92 | def deploy_sitesettings(): 93 | """ 94 | Deploy to the project directory in the virtualenv 95 | """ 96 | 97 | sitesettings = '/'.join([deployment_root(),'env',env.project_fullname,'project',env.project_package_name,'sitesettings']) 98 | local_dir = os.path.join(os.getcwd(),env.project_package_name,'sitesettings') 99 | 100 | created = deploy_files(local_dir, sitesettings) 101 | if env.verbosity and created: 102 | print env.host,"DEPLOYING sitesettings" 103 | for path in created: 104 | tail = path.split('/')[-1] 105 | print ' * uploaded',tail 106 | 107 | @run_once_per_version 108 | def deploy_templates(): 109 | """ 110 | Deploy any templates from your shortest TEMPLATE_DIRS setting 111 | """ 112 | 113 | deployed = None 114 | if not hasattr(env, 'project_template_dir'): 115 | #the normal pattern would mean the shortest path is the main one. 116 | #its probably the last listed 117 | length = 1000 118 | for dir in env.TEMPLATE_DIRS: 119 | if dir: 120 | len_dir = len(dir) 121 | if len_dir < length: 122 | length = len_dir 123 | env.project_template_dir = dir 124 | 125 | if hasattr(env,'project_template_dir'): 126 | remote_dir = '/'.join([deployment_root(),'env',env.project_fullname,'templates']) 127 | if env.verbosity: 128 | print env.host,"DEPLOYING templates", remote_dir 129 | deployed = deploy_files(env.project_template_dir,remote_dir) 130 | return deployed 131 | 132 | @run_once_per_version 133 | def deploy_static(): 134 | """ 135 | Deploy static (application) versioned media 136 | """ 137 | 138 | if not env.STATIC_URL or 'http://' in env.STATIC_URL: return 139 | from django.core.servers.basehttp import AdminMediaHandler 140 | remote_dir = '/'.join([deployment_root(),'env',env.project_fullname,'static']) 141 | m_prefix = len(env.MEDIA_URL) 142 | #if app media is not handled by django-staticfiles we can install admin media by default 143 | if 'django.contrib.admin' in env.INSTALLED_APPS and not 'django.contrib.staticfiles' in env.INSTALLED_APPS: 144 | 145 | if env.MEDIA_URL and env.MEDIA_URL == env.ADMIN_MEDIA_PREFIX[:m_prefix]: 146 | print "ERROR: Your ADMIN_MEDIA_PREFIX (Application media) must not be on the same path as your MEDIA_URL (User media)" 147 | sys.exit(1) 148 | admin = AdminMediaHandler('DummyApp') 149 | local_dir = admin.base_dir 150 | remote_dir = ''.join([remote_dir,env.ADMIN_MEDIA_PREFIX]) 151 | else: 152 | if env.MEDIA_URL and env.MEDIA_URL == env.STATIC_URL[:m_prefix]: 153 | print "ERROR: Your STATIC_URL (Application media) must not be on the same path as your MEDIA_URL (User media)" 154 | sys.exit(1) 155 | elif env.STATIC_ROOT: 156 | local_dir = env.STATIC_ROOT 157 | static_url = env.STATIC_URL[1:] 158 | if static_url: 159 | remote_dir = '/'.join([remote_dir,static_url]) 160 | else: return 161 | if env.verbosity: 162 | print env.host,"DEPLOYING static",remote_dir 163 | return deploy_files(local_dir,remote_dir) 164 | 165 | @run_once_per_version 166 | def deploy_media(): 167 | """ 168 | Deploy MEDIA_ROOT unversioned on host 169 | """ 170 | if not env.MEDIA_URL or not env.MEDIA_ROOT or 'http://' in env.MEDIA_URL: return 171 | local_dir = env.MEDIA_ROOT 172 | 173 | remote_dir = '/'.join([deployment_root(),'public']) 174 | media_url = env.MEDIA_URL[1:] 175 | if media_url: 176 | remote_dir = '/'.join([remote_dir,media_url]) 177 | if env.verbosity: 178 | print env.host,"DEPLOYING media",remote_dir 179 | deployed = deploy_files(local_dir,remote_dir) 180 | 181 | #make writable for www-data for file uploads 182 | sudo("chown -R www-data:sudo %s" % remote_dir) 183 | sudo("chmod -R ug+w %s"% remote_dir) 184 | return deployed 185 | 186 | @runs_once 187 | def deploy_db(rollback=False): 188 | """ 189 | Deploy a sqlite database from development 190 | """ 191 | if not rollback: 192 | 193 | if env.DEFAULT_DATABASE_ENGINE=='django.db.backends.sqlite3': 194 | db_dir = '/'.join([deployment_root(),'database']) 195 | db_name = ''.join([env.project_name,'_','site_1','.db']) 196 | dest_db_path = '/'.join([db_dir,db_name]) 197 | if exists(dest_db_path): return 198 | if env.verbosity: 199 | print env.host,"DEPLOYING DEFAULT SQLITE DATABASE" 200 | if not env.DEFAULT_DATABASE_NAME: 201 | print "ERROR: A database name has not been defined in your Django settings file" 202 | sys.exit(1) 203 | 204 | if env.DEFAULT_DATABASE_NAME[0] not in [os.path.sep,'.']: #relative path 205 | db_path = os.path.join(os.getcwd(),env.project_package_name,env.DEFAULT_DATABASE_NAME) 206 | 207 | elif env.DEFAULT_DATABASE_NAME[:2] == '..': 208 | print "ERROR: Use a full expanded path to the database in your Django settings" 209 | sys.exit(1) 210 | else: 211 | db_path = env.DEFAULT_DATABASE_NAME 212 | 213 | if not db_path or not os.path.exists(db_path): 214 | print "ERROR: the database %s does not exist. \nRun python manage.py syncdb to create your database locally first, or check your settings."% db_path 215 | sys.exit(1) 216 | 217 | db_name = os.path.split(db_path)[1] 218 | run('mkdir -p '+db_dir) 219 | put(db_path,dest_db_path) 220 | #directory and file must be writable by webserver 221 | sudo("chown -R %s:www-data %s"% (env.user,db_dir)) 222 | sudo("chmod -R ug+w %s"% db_dir) 223 | 224 | elif env.DEFAULT_DATABASE_ENGINE=='django.db.backends.': 225 | print "ERROR: The default database engine has not been defined in your Django settings file" 226 | print "At a minimum you must define an sqlite3 database for woven to deploy, or define a database that is managed outside of woven." 227 | sys.exit(1) 228 | elif rollback and env.DEFAULT_DATABASE_ENGINE=='django.db.backends.sqlite3': 229 | if env.INTERACTIVE: 230 | delete = confirm('DELETE the database on the host?',default=False) 231 | if delete: 232 | run('rm -f '+db_name) 233 | return 234 | -------------------------------------------------------------------------------- /woven/templates/distribution_template/project_name/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bretth/woven/ec1da7b401a335f43129e7115fe7a4d145649f1e/woven/templates/distribution_template/project_name/__init__.py -------------------------------------------------------------------------------- /woven/templates/distribution_template/project_name/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from django.core.management import execute_manager 3 | try: 4 | import settings # Assumed to be in the same directory. 5 | except ImportError: 6 | import sys 7 | sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__) 8 | sys.exit(1) 9 | 10 | if __name__ == "__main__": 11 | execute_manager(settings) 12 | -------------------------------------------------------------------------------- /woven/templates/distribution_template/project_name/settings.py: -------------------------------------------------------------------------------- 1 | # Woven settings for {{ project_name }} Django 1.3 project. 2 | 3 | 4 | from os import path 5 | PROJECT_ROOT = path.dirname(path.realpath(__file__)) 6 | DISTRIBUTION_ROOT = path.split(PROJECT_ROOT)[0] 7 | 8 | DEBUG = True 9 | TEMPLATE_DEBUG = DEBUG 10 | 11 | ADMINS = ( 12 | # ('Your Name', 'your_email@example.com'), 13 | ) 14 | 15 | MANAGERS = ADMINS 16 | 17 | DATABASES = { 18 | 'default': { 19 | 'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'. 20 | 'NAME': path.join(DISTRIBUTION_ROOT,'database','default.db') , # Or path to database file if using sqlite3. 21 | 'USER': '', # Not used with sqlite3. 22 | 'PASSWORD': '', # Not used with sqlite3. 23 | 'HOST': '', # Set to empty string for localhost. Not used with sqlite3. 24 | 'PORT': '', # Set to empty string for default. Not used with sqlite3. 25 | } 26 | } 27 | 28 | # Local time zone for this installation. Choices can be found here: 29 | # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name 30 | # although not all choices may be available on all operating systems. 31 | # On Unix systems, a value of None will cause Django to use the same 32 | # timezone as the operating system. 33 | # If running in a Windows environment this must be set to the same as your 34 | # system time zone. 35 | TIME_ZONE = 'America/Chicago' 36 | 37 | # Language code for this installation. All choices can be found here: 38 | # http://www.i18nguy.com/unicode/language-identifiers.html 39 | LANGUAGE_CODE = 'en-us' 40 | 41 | SITE_ID = 1 42 | 43 | # If you set this to False, Django will make some optimizations so as not 44 | # to load the internationalization machinery. 45 | USE_I18N = True 46 | 47 | # If you set this to False, Django will not format dates, numbers and 48 | # calendars according to the current locale 49 | USE_L10N = True 50 | 51 | # Absolute filesystem path to the directory that will hold user-uploaded files. 52 | # Example: "/home/media/media.lawrence.com/media/" 53 | MEDIA_ROOT = DISTRIBUTION_ROOT + '/media/' 54 | 55 | # URL that handles the media served from MEDIA_ROOT. Make sure to use a 56 | # trailing slash. 57 | # Examples: "http://media.lawrence.com/media/", "http://example.com/media/" 58 | MEDIA_URL = '/media/' 59 | 60 | # Absolute path to the directory static files should be collected to. 61 | # Don't put anything in this directory yourself; store your static files 62 | # in apps' "static/" subdirectories and in STATICFILES_DIRS. 63 | # Example: "/home/media/media.lawrence.com/static/" 64 | STATIC_ROOT = DISTRIBUTION_ROOT + '/static/' 65 | 66 | # URL that handles the static files served from STATIC_ROOT. 67 | # Example: "http://media.lawrence.com/static/" 68 | STATIC_URL = '/static/' 69 | 70 | # URL prefix for admin media -- CSS, JavaScript and images. 71 | # Make sure to use a trailing slash. 72 | # Examples: "http://foo.com/static/admin/", "/static/admin/". 73 | ADMIN_MEDIA_PREFIX = '/static/admin/' 74 | 75 | # A list of locations of additional static files 76 | STATICFILES_DIRS = ( 77 | # Put strings here, like "/home/html/static" or "C:/www/django/static". 78 | # Always use forward slashes, even on Windows. 79 | # Don't forget to use absolute paths, not relative paths. 80 | ) 81 | 82 | # List of finder classes that know how to find static files in 83 | # various locations. 84 | STATICFILES_FINDERS = ( 85 | 'django.contrib.staticfiles.finders.FileSystemFinder', 86 | 'django.contrib.staticfiles.finders.AppDirectoriesFinder', 87 | # 'django.contrib.staticfiles.finders.DefaultStorageFinder', 88 | ) 89 | 90 | # Make this unique, and don't share it with anybody. 91 | SECRET_KEY = '' 92 | 93 | # List of callables that know how to import templates from various sources. 94 | TEMPLATE_LOADERS = ( 95 | 'django.template.loaders.filesystem.Loader', 96 | 'django.template.loaders.app_directories.Loader', 97 | # 'django.template.loaders.eggs.Loader', 98 | ) 99 | 100 | MIDDLEWARE_CLASSES = ( 101 | 'django.middleware.common.CommonMiddleware', 102 | 'django.contrib.sessions.middleware.SessionMiddleware', 103 | 'django.middleware.csrf.CsrfViewMiddleware', 104 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 105 | 'django.contrib.messages.middleware.MessageMiddleware', 106 | # Uncomment the next line for simple clickjacking protection: 107 | # 'django.middleware.clickjacking.XFrameOptionsMiddleware', 108 | ) 109 | 110 | ROOT_URLCONF = '{{ project_name }}.urls' 111 | 112 | TEMPLATE_DIRS = ( 113 | DISTRIBUTION_ROOT + '/templates/', 114 | # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates". 115 | # Always use forward slashes, even on Windows. 116 | # Don't forget to use absolute paths, not relative paths. 117 | ) 118 | 119 | INSTALLED_APPS = ( 120 | 'django.contrib.auth', 121 | 'django.contrib.contenttypes', 122 | 'django.contrib.sessions', 123 | 'django.contrib.sites', 124 | 'django.contrib.messages', 125 | 'django.contrib.staticfiles', 126 | # Uncomment the next line to enable the admin: 127 | ## 'django.contrib.admin', 128 | # Uncomment the next line to enable admin documentation: 129 | ## 'django.contrib.admindocs', 130 | ) 131 | 132 | # A sample logging configuration. The only tangible logging 133 | # performed by this configuration is to send an email to 134 | # the site admins on every HTTP 500 error when DEBUG=False. 135 | # See http://docs.djangoproject.com/en/dev/topics/logging for 136 | # more details on how to customize your logging configuration. 137 | LOGGING = { 138 | 'version': 1, 139 | 'disable_existing_loggers': False, 140 | 'handlers': { 141 | 'mail_admins': { 142 | 'level': 'ERROR', 143 | 'class': 'django.utils.log.AdminEmailHandler' 144 | } 145 | }, 146 | 'loggers': { 147 | 'django.request': { 148 | 'handlers': ['mail_admins'], 149 | 'level': 'ERROR', 150 | 'propagate': True, 151 | }, 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /woven/templates/distribution_template/project_name/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls.defaults import patterns, include, url 2 | 3 | # Uncomment the next two lines to enable the admin: 4 | # from django.contrib import admin 5 | # admin.autodiscover() 6 | 7 | urlpatterns = patterns('', 8 | # Examples: 9 | # url(r'^$', '{{ project_name }}.views.home', name='home'), 10 | # url(r'^{{ project_name }}/', include('{{ project_name }}.foo.urls')), 11 | 12 | # Uncomment the admin/doc line below to enable admin documentation: 13 | # url(r'^admin/doc/', include('django.contrib.admindocs.urls')), 14 | 15 | # Uncomment the next line to enable the admin: 16 | # url(r'^admin/', include(admin.site.urls)), 17 | ) 18 | -------------------------------------------------------------------------------- /woven/templates/distribution_template/setup.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup 2 | 3 | setup(name='{{ project_name }}', 4 | version='0.1', 5 | packages=['{{ project_name }}'], 6 | ) -------------------------------------------------------------------------------- /woven/templates/setup.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup 2 | 3 | setup(name='{{ project_name }}', 4 | version='0.1', 5 | packages=['{{ project_name }}'], 6 | ) -------------------------------------------------------------------------------- /woven/templates/woven/django-apache-template.txt: -------------------------------------------------------------------------------- 1 | 2 | 3 | #ServerAdmin 4 | 5 | ServerName {{ domain }} 6 | 7 | #Stack size reduces the amount of virtual memory available per thread to a much more sensible limit than 8MB 8 | #This setting is especially useful for vps servers 9 | WSGIDaemonProcess {{ domain }} user={{ site_user}} processes=2 inactivity-timeout=300 maximum-requests=10000 threads=15 stack-size=524288 display-name={{ site_user }} 10 | WSGIProcessGroup {{ domain }} 11 | WSGIScriptAlias / {{ deployment_root }}/env/{{ project_name }}/wsgi/{{ wsgi_filename }} 12 | 13 | 14 | 15 | Order deny,allow 16 | Allow from all 17 | 18 | 19 | #LogLevels: debug, info, notice, warn, error, crit, alert, emerg 20 | LogLevel warn 21 | CustomLog /var/log/apache2/{{ u_domain }}_apache_access.log combined 22 | ErrorLog /var/log/apache2/{{ u_domain }}_apache_error.log 23 | 24 | -------------------------------------------------------------------------------- /woven/templates/woven/django-wsgi-template.txt: -------------------------------------------------------------------------------- 1 | #!{{ deployment_root }}/env/{{ project_name }}/bin/python 2 | import site 3 | site.addsitedir('{{ deployment_root }}/env/{{ project_name }}/lib/python2.6/site-packages') 4 | 5 | # Add in any optional local apps directly into the site-packages 6 | {% if project_apps_path %} 7 | site.addsitedir('{{ deployment_root }}/env/{{ project_name }}/project/{{ project_package_name }}/{{ project_apps_path }}') 8 | {% endif %} 9 | 10 | #uncomment to allow print statements output to error logs for debugging 11 | #import sys 12 | #sys.stdout = sys.stderr 13 | 14 | import os 15 | os.environ['PATH'] = '{{ deployment_root }}/env/{{ project_name }}/bin:%s' % os.environ.get('PATH', '/bin:/usr/bin') 16 | os.environ['VIRTUAL_ENV'] = '{{ deployment_root }}/env/{{ project_name }}' 17 | os.environ['PYTHON_EGG_CACHE'] = '{{ deployment_root }}/env/{{ project_name }}/egg_cache' 18 | 19 | activate_this = '{{ deployment_root }}/env/{{ project_name }}/bin/activate_this.py' 20 | execfile(activate_this, dict(__file__=activate_this)) 21 | 22 | # Set the DJANGO_SETTINGS_MODULE environment variable. 23 | # The project package itself is symlinked into site-packages 24 | os.environ['DJANGO_SETTINGS_MODULE'] = "{{ project_package_name }}.sitesettings.{{ settings }}" 25 | 26 | import django.core.handlers.wsgi 27 | application = django.core.handlers.wsgi.WSGIHandler() -------------------------------------------------------------------------------- /woven/templates/woven/etc/apache2/ports.conf: -------------------------------------------------------------------------------- 1 | # If you just change the port or add more ports here, you will likely also 2 | # have to change the VirtualHost statement in 3 | # /etc/apache2/sites-enabled/000-default 4 | # This is also true if you have upgraded from before 2.2.9-3 (i.e. from 5 | # Debian etch). See /usr/share/doc/apache2.2-common/NEWS.Debian.gz and 6 | # README.Debian.gz 7 | 8 | NameVirtualHost *:10080 9 | Listen 10080 10 | 11 | 12 | # If you add NameVirtualHost *:443 here, you will also have to change 13 | # the VirtualHost statement in /etc/apache2/sites-available/default-ssl 14 | # to 15 | # Server Name Indication for SSL named virtual hosts is currently not 16 | # supported by MSIE on Windows XP. 17 | Listen 443 18 | 19 | 20 | 21 | Listen 443 22 | 23 | 24 | ThreadStackSize 524288 25 | 26 | -------------------------------------------------------------------------------- /woven/templates/woven/etc/init.d/nginx: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | ### BEGIN INIT INFO 4 | # Provides: nginx 5 | # Required-Start: $local_fs $remote_fs $network $syslog 6 | # Required-Stop: $local_fs $remote_fs $network $syslog 7 | # Default-Start: 2 3 4 5 8 | # Default-Stop: 0 1 6 9 | # Short-Description: starts the nginx web server 10 | # Description: starts nginx using start-stop-daemon 11 | ### END INIT INFO 12 | 13 | #Woven init.d script for nginx - just adds sleep to ensure consistent start after reboot 14 | 15 | PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin 16 | DAEMON=/usr/sbin/nginx 17 | NAME=nginx 18 | DESC=nginx 19 | 20 | test -x $DAEMON || exit 0 21 | 22 | # Include nginx defaults if available 23 | if [ -f /etc/default/nginx ] ; then 24 | . /etc/default/nginx 25 | fi 26 | 27 | set -e 28 | 29 | . /lib/lsb/init-functions 30 | 31 | test_nginx_config() { 32 | if nginx -t $DAEMON_OPTS 33 | then 34 | return 0 35 | else 36 | return $? 37 | fi 38 | } 39 | 40 | case "$1" in 41 | start) 42 | echo -n "Starting $DESC: " 43 | sleep 1 44 | test_nginx_config 45 | sleep 1 46 | start-stop-daemon --start --quiet --pidfile /var/run/$NAME.pid \ 47 | --exec $DAEMON -- $DAEMON_OPTS || true 48 | echo "$NAME." 49 | ;; 50 | stop) 51 | echo -n "Stopping $DESC: " 52 | start-stop-daemon --stop --quiet --pidfile /var/run/$NAME.pid \ 53 | --exec $DAEMON || true 54 | echo "$NAME." 55 | ;; 56 | restart|force-reload) 57 | echo -n "Restarting $DESC: " 58 | start-stop-daemon --stop --quiet --pidfile \ 59 | /var/run/$NAME.pid --exec $DAEMON || true 60 | sleep 1 61 | test_nginx_config 62 | start-stop-daemon --start --quiet --pidfile \ 63 | /var/run/$NAME.pid --exec $DAEMON -- $DAEMON_OPTS || true 64 | echo "$NAME." 65 | ;; 66 | reload) 67 | echo -n "Reloading $DESC configuration: " 68 | test_nginx_config 69 | start-stop-daemon --stop --signal HUP --quiet --pidfile /var/run/$NAME.pid \ 70 | --exec $DAEMON || true 71 | echo "$NAME." 72 | ;; 73 | configtest) 74 | echo -n "Testing $DESC configuration: " 75 | if test_nginx_config 76 | then 77 | echo "$NAME." 78 | else 79 | exit $? 80 | fi 81 | ;; 82 | status) 83 | status_of_proc -p /var/run/$NAME.pid "$DAEMON" nginx && exit 0 || exit $? 84 | ;; 85 | *) 86 | echo "Usage: $NAME {start|stop|restart|reload|force-reload|status|configtest}" >&2 87 | exit 1 88 | ;; 89 | esac 90 | 91 | exit 0 -------------------------------------------------------------------------------- /woven/templates/woven/etc/nginx/nginx.conf: -------------------------------------------------------------------------------- 1 | user www-data; 2 | 3 | worker_processes 2; 4 | 5 | error_log /var/log/nginx/error.log; 6 | pid /var/run/nginx.pid; 7 | 8 | events { 9 | worker_connections 1024; 10 | # multi_accept on; 11 | } 12 | 13 | http { 14 | include /etc/nginx/mime.types; 15 | 16 | access_log /var/log/nginx/access.log; 17 | 18 | sendfile on; 19 | #This can be enabled on linux 20 | tcp_nopush on; 21 | 22 | #keepalive_timeout 0; 23 | 24 | #We'll leave the default keepalive for static media 25 | keepalive_timeout 65; 26 | tcp_nodelay on; 27 | 28 | gzip on; 29 | #compression defaults to 1 for fastest 30 | gzip_comp_level 2; 31 | gzip_disable "MSIE [1-6]\.(?!.*SV1)"; 32 | #add some additional static files to compress text/html is always compressed 33 | gzip_types text/plain text/css application/x-javascript text/xml application/xml 34 | application/xml+rss text/javascript; 35 | #You may want to disable this for some django applications 36 | gzip_proxied any; 37 | 38 | include /etc/nginx/conf.d/*.conf; 39 | include /etc/nginx/sites-enabled/*; 40 | } 41 | 42 | # mail { 43 | # # See sample authentication script at: 44 | # # http://wiki.nginx.org/NginxImapAuthenticateWithApachePhpScript 45 | # 46 | # # auth_http localhost/auth.php; 47 | # # pop3_capabilities "TOP" "USER"; 48 | # # imap_capabilities "IMAP4rev1" "UIDPLUS"; 49 | # 50 | # server { 51 | # listen localhost:110; 52 | # protocol pop3; 53 | # proxy on; 54 | # } 55 | # 56 | # server { 57 | # listen localhost:143; 58 | # protocol imap; 59 | # proxy on; 60 | # } 61 | # } 62 | -------------------------------------------------------------------------------- /woven/templates/woven/etc/nginx/proxy.conf: -------------------------------------------------------------------------------- 1 | #Woven proxy settings 2 | proxy_redirect off; 3 | proxy_set_header Host $host; 4 | proxy_set_header X-Real-IP $remote_addr; 5 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 6 | 7 | #File upload file size - alter if your users are uploading large files 8 | client_max_body_size 10m; 9 | client_body_buffer_size 128k; 10 | 11 | #You want proxy buffering to be off for long-polling applications 12 | #For django proxying however proxy_buffering on is the best option 13 | proxy_buffering on; 14 | proxy_connect_timeout 90; 15 | proxy_send_timeout 90; 16 | proxy_read_timeout 90; 17 | 18 | #should match the filesystem block size which is normally 4k for ext3 19 | proxy_buffer_size 4k; 20 | 21 | #Per connection potential memory usage would be num x size 22 | #total potential memory usage would be worker_processes x worker_connections x num x size 23 | #Better to have a larger number of small buffers since Django is likely only returning text 24 | #and even if the page + headers is less than 4k, 4k is used, so larger sizes can be wasteful 25 | #For very small servers (< 512MB) you may want to reduce the number of buffers to 8 26 | proxy_buffers 32 4k; 27 | 28 | #proxy_buffer_size x 2 29 | proxy_busy_buffers_size 8k; 30 | proxy_temp_file_write_size 8k; -------------------------------------------------------------------------------- /woven/templates/woven/etc/nginx/sites-available/default: -------------------------------------------------------------------------------- 1 | server { 2 | #default conf that denies any domains other than the virtual hosts 3 | #that are configured elsewhere 4 | 5 | listen 80 default; 6 | server_name _; 7 | deny all; 8 | } 9 | 10 | -------------------------------------------------------------------------------- /woven/templates/woven/etc/postgresql/8.4/main/pg_hba.conf: -------------------------------------------------------------------------------- 1 | # Woven default template from Ubuntu 10.10 2 | # PostgreSQL Client Authentication Configuration File 3 | # =================================================== 4 | # 5 | # Refer to the "Client Authentication" section in the 6 | # PostgreSQL documentation for a complete description 7 | # of this file. A short synopsis follows. 8 | # 9 | # This file controls: which hosts are allowed to connect, how clients 10 | # are authenticated, which PostgreSQL user names they can use, which 11 | # databases they can access. Records take one of these forms: 12 | # 13 | # local DATABASE USER METHOD [OPTIONS] 14 | # host DATABASE USER CIDR-ADDRESS METHOD [OPTIONS] 15 | # hostssl DATABASE USER CIDR-ADDRESS METHOD [OPTIONS] 16 | # hostnossl DATABASE USER CIDR-ADDRESS METHOD [OPTIONS] 17 | # 18 | # (The uppercase items must be replaced by actual values.) 19 | # 20 | # The first field is the connection type: "local" is a Unix-domain socket, 21 | # "host" is either a plain or SSL-encrypted TCP/IP socket, "hostssl" is an 22 | # SSL-encrypted TCP/IP socket, and "hostnossl" is a plain TCP/IP socket. 23 | # 24 | # DATABASE can be "all", "sameuser", "samerole", a database name, or 25 | # a comma-separated list thereof. 26 | # 27 | # USER can be "all", a user name, a group name prefixed with "+", or 28 | # a comma-separated list thereof. In both the DATABASE and USER fields 29 | # you can also write a file name prefixed with "@" to include names from 30 | # a separate file. 31 | # 32 | # CIDR-ADDRESS specifies the set of hosts the record matches. 33 | # It is made up of an IP address and a CIDR mask that is an integer 34 | # (between 0 and 32 (IPv4) or 128 (IPv6) inclusive) that specifies 35 | # the number of significant bits in the mask. Alternatively, you can write 36 | # an IP address and netmask in separate columns to specify the set of hosts. 37 | # 38 | # METHOD can be "trust", "reject", "md5", "password", "gss", "sspi", "krb5", 39 | # "ident", "pam", "ldap" or "cert". Note that "password" sends passwords 40 | # in clear text; "md5" is preferred since it sends encrypted passwords. 41 | # 42 | # OPTIONS are a set of options for the authentication in the format 43 | # NAME=VALUE. The available options depend on the different authentication 44 | # methods - refer to the "Client Authentication" section in the documentation 45 | # for a list of which options are available for which authentication methods. 46 | # 47 | # Database and user names containing spaces, commas, quotes and other special 48 | # characters must be quoted. Quoting one of the keywords "all", "sameuser" or 49 | # "samerole" makes the name lose its special character, and just match a 50 | # database or username with that name. 51 | # 52 | # This file is read on server startup and when the postmaster receives 53 | # a SIGHUP signal. If you edit the file on a running system, you have 54 | # to SIGHUP the postmaster for the changes to take effect. You can use 55 | # "pg_ctl reload" to do that. 56 | 57 | # Put your actual configuration here 58 | # ---------------------------------- 59 | # 60 | # If you want to allow non-local connections, you need to add more 61 | # "host" records. In that case you will also need to make PostgreSQL listen 62 | # on a non-local interface via the listen_addresses configuration parameter, 63 | # or via the -i or -h command line switches. 64 | # 65 | 66 | 67 | 68 | 69 | # DO NOT DISABLE! 70 | # If you change this first entry you will need to make sure that the 71 | # database 72 | # super user can access the database using some other method. 73 | # Noninteractive 74 | # access to all databases is required during automatic maintenance 75 | # (custom daily cronjobs, replication, and similar tasks). 76 | # 77 | # Database administrative login by UNIX sockets 78 | local all postgres ident 79 | 80 | # TYPE DATABASE USER CIDR-ADDRESS METHOD 81 | 82 | # "local" is for Unix domain socket connections only 83 | local all all ident 84 | # IPv4 local connections: 85 | host all all 127.0.0.1/32 md5 86 | # IPv6 local connections: 87 | host all all ::1/128 md5 88 | # Woven default 89 | host all all 0.0.0.0/0 md5 90 | 91 | -------------------------------------------------------------------------------- /woven/templates/woven/etc/postgresql/8.4/main/postgresql.conf: -------------------------------------------------------------------------------- 1 | # Default Woven PostgreSQL taken from Ubuntu 10.10 2 | # ----------------------------- 3 | # PostgreSQL configuration file 4 | # ----------------------------- 5 | # 6 | # This file consists of lines of the form: 7 | # 8 | # name = value 9 | # 10 | # (The "=" is optional.) Whitespace may be used. Comments are introduced with 11 | # "#" anywhere on a line. The complete list of parameter names and allowed 12 | # values can be found in the PostgreSQL documentation. 13 | # 14 | # The commented-out settings shown in this file represent the default values. 15 | # Re-commenting a setting is NOT sufficient to revert it to the default value; 16 | # you need to reload the server. 17 | # 18 | # This file is read on server startup and when the server receives a SIGHUP 19 | # signal. If you edit the file on a running system, you have to SIGHUP the 20 | # server for the changes to take effect, or use "pg_ctl reload". Some 21 | # parameters, which are marked below, require a server shutdown and restart to 22 | # take effect. 23 | # 24 | # Any parameter can also be given as a command-line option to the server, e.g., 25 | # "postgres -c log_connections=on". Some parameters can be changed at run time 26 | # with the "SET" SQL command. 27 | # 28 | # Memory units: kB = kilobytes Time units: ms = milliseconds 29 | # MB = megabytes s = seconds 30 | # GB = gigabytes min = minutes 31 | # h = hours 32 | # d = days 33 | 34 | 35 | #------------------------------------------------------------------------------ 36 | # FILE LOCATIONS 37 | #------------------------------------------------------------------------------ 38 | 39 | # The default values of these variables are driven from the -D command-line 40 | # option or PGDATA environment variable, represented here as ConfigDir. 41 | 42 | data_directory = '/var/lib/postgresql/8.4/main' # use data in another directory 43 | # (change requires restart) 44 | hba_file = '/etc/postgresql/8.4/main/pg_hba.conf' # host-based authentication file 45 | # (change requires restart) 46 | ident_file = '/etc/postgresql/8.4/main/pg_ident.conf' # ident configuration file 47 | # (change requires restart) 48 | 49 | # If external_pid_file is not explicitly set, no extra PID file is written. 50 | external_pid_file = '/var/run/postgresql/8.4-main.pid' # write an extra PID file 51 | # (change requires restart) 52 | 53 | 54 | #------------------------------------------------------------------------------ 55 | # CONNECTIONS AND AUTHENTICATION 56 | #------------------------------------------------------------------------------ 57 | 58 | # - Connection Settings - 59 | #Woven default 60 | listen_addresses = '*' # what IP address(es) to listen on; 61 | # comma-separated list of addresses; 62 | # defaults to 'localhost', '*' = all 63 | # (change requires restart) 64 | 65 | port = 5432 # (change requires restart) 66 | max_connections = 50 # (change requires restart) 67 | # Note: Increasing max_connections costs ~400 bytes of shared memory per 68 | # connection slot, plus lock space (see max_locks_per_transaction). 69 | #superuser_reserved_connections = 3 # (change requires restart) 70 | unix_socket_directory = '/var/run/postgresql' # (change requires restart) 71 | #unix_socket_group = '' # (change requires restart) 72 | #unix_socket_permissions = 0777 # begin with 0 to use octal notation 73 | # (change requires restart) 74 | #bonjour_name = '' # defaults to the computer name 75 | # (change requires restart) 76 | 77 | # - Security and Authentication - 78 | 79 | #authentication_timeout = 1min # 1s-600s 80 | ssl = false # (change requires restart) 81 | #ssl_ciphers = 'ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH' # allowed SSL ciphers 82 | # (change requires restart) 83 | #ssl_renegotiation_limit = 512MB # amount of data between renegotiations 84 | #password_encryption = on 85 | #db_user_namespace = off 86 | 87 | # Kerberos and GSSAPI 88 | #krb_server_keyfile = '' 89 | #krb_srvname = 'postgres' # (Kerberos only) 90 | #krb_caseins_users = off 91 | 92 | # - TCP Keepalives - 93 | # see "man 7 tcp" for details 94 | 95 | #tcp_keepalives_idle = 0 # TCP_KEEPIDLE, in seconds; 96 | # 0 selects the system default 97 | #tcp_keepalives_interval = 0 # TCP_KEEPINTVL, in seconds; 98 | # 0 selects the system default 99 | #tcp_keepalives_count = 0 # TCP_KEEPCNT; 100 | # 0 selects the system default 101 | 102 | 103 | #------------------------------------------------------------------------------ 104 | # RESOURCE USAGE (except WAL) 105 | #------------------------------------------------------------------------------ 106 | 107 | # - Memory - 108 | 109 | shared_buffers = 28MB # min 128kB 110 | # (change requires restart) 111 | #temp_buffers = 8MB # min 800kB 112 | #max_prepared_transactions = 0 # zero disables the feature 113 | # (change requires restart) 114 | # Note: Increasing max_prepared_transactions costs ~600 bytes of shared memory 115 | # per transaction slot, plus lock space (see max_locks_per_transaction). 116 | # It is not advisable to set max_prepared_transactions nonzero unless you 117 | # actively intend to use prepared transactions. 118 | #work_mem = 1MB # min 64kB 119 | #maintenance_work_mem = 16MB # min 1MB 120 | #max_stack_depth = 2MB # min 100kB 121 | 122 | # - Kernel Resource Usage - 123 | 124 | #max_files_per_process = 1000 # min 25 125 | # (change requires restart) 126 | #shared_preload_libraries = '' # (change requires restart) 127 | 128 | # - Cost-Based Vacuum Delay - 129 | 130 | #vacuum_cost_delay = 0ms # 0-100 milliseconds 131 | #vacuum_cost_page_hit = 1 # 0-10000 credits 132 | #vacuum_cost_page_miss = 10 # 0-10000 credits 133 | #vacuum_cost_page_dirty = 20 # 0-10000 credits 134 | #vacuum_cost_limit = 200 # 1-10000 credits 135 | 136 | # - Background Writer - 137 | 138 | #bgwriter_delay = 200ms # 10-10000ms between rounds 139 | #bgwriter_lru_maxpages = 100 # 0-1000 max buffers written/round 140 | #bgwriter_lru_multiplier = 2.0 # 0-10.0 multipler on buffers scanned/round 141 | 142 | # - Asynchronous Behavior - 143 | 144 | #effective_io_concurrency = 1 # 1-1000. 0 disables prefetching 145 | 146 | 147 | #------------------------------------------------------------------------------ 148 | # WRITE AHEAD LOG 149 | #------------------------------------------------------------------------------ 150 | 151 | # - Settings - 152 | 153 | #fsync = on # turns forced synchronization on or off 154 | #synchronous_commit = on # immediate fsync at commit 155 | #wal_sync_method = fsync # the default is the first option 156 | # supported by the operating system: 157 | # open_datasync 158 | # fdatasync 159 | # fsync 160 | # fsync_writethrough 161 | # open_sync 162 | #full_page_writes = on # recover from partial page writes 163 | #wal_buffers = 64kB # min 32kB 164 | # (change requires restart) 165 | #wal_writer_delay = 200ms # 1-10000 milliseconds 166 | 167 | #commit_delay = 0 # range 0-100000, in microseconds 168 | #commit_siblings = 5 # range 1-1000 169 | 170 | # - Checkpoints - 171 | 172 | #checkpoint_segments = 3 # in logfile segments, min 1, 16MB each 173 | #checkpoint_timeout = 5min # range 30s-1h 174 | #checkpoint_completion_target = 0.5 # checkpoint target duration, 0.0 - 1.0 175 | #checkpoint_warning = 30s # 0 disables 176 | 177 | # - Archiving - 178 | 179 | #archive_mode = off # allows archiving to be done 180 | # (change requires restart) 181 | #archive_command = '' # command to use to archive a logfile segment 182 | #archive_timeout = 0 # force a logfile segment switch after this 183 | # number of seconds; 0 disables 184 | 185 | 186 | #------------------------------------------------------------------------------ 187 | # QUERY TUNING 188 | #------------------------------------------------------------------------------ 189 | 190 | # - Planner Method Configuration - 191 | 192 | #enable_bitmapscan = on 193 | #enable_hashagg = on 194 | #enable_hashjoin = on 195 | #enable_indexscan = on 196 | #enable_mergejoin = on 197 | #enable_nestloop = on 198 | #enable_seqscan = on 199 | #enable_sort = on 200 | #enable_tidscan = on 201 | 202 | # - Planner Cost Constants - 203 | 204 | #seq_page_cost = 1.0 # measured on an arbitrary scale 205 | #random_page_cost = 4.0 # same scale as above 206 | #cpu_tuple_cost = 0.01 # same scale as above 207 | #cpu_index_tuple_cost = 0.005 # same scale as above 208 | #cpu_operator_cost = 0.0025 # same scale as above 209 | #effective_cache_size = 128MB 210 | 211 | # - Genetic Query Optimizer - 212 | 213 | #geqo = on 214 | #geqo_threshold = 12 215 | #geqo_effort = 5 # range 1-10 216 | #geqo_pool_size = 0 # selects default based on effort 217 | #geqo_generations = 0 # selects default based on effort 218 | #geqo_selection_bias = 2.0 # range 1.5-2.0 219 | 220 | # - Other Planner Options - 221 | 222 | #default_statistics_target = 100 # range 1-10000 223 | #constraint_exclusion = partition # on, off, or partition 224 | #cursor_tuple_fraction = 0.1 # range 0.0-1.0 225 | #from_collapse_limit = 8 226 | #join_collapse_limit = 8 # 1 disables collapsing of explicit 227 | # JOIN clauses 228 | 229 | 230 | #------------------------------------------------------------------------------ 231 | # ERROR REPORTING AND LOGGING 232 | #------------------------------------------------------------------------------ 233 | 234 | # - Where to Log - 235 | 236 | #log_destination = 'stderr' # Valid values are combinations of 237 | # stderr, csvlog, syslog and eventlog, 238 | # depending on platform. csvlog 239 | # requires logging_collector to be on. 240 | 241 | # This is used when logging to stderr: 242 | #logging_collector = off # Enable capturing of stderr and csvlog 243 | # into log files. Required to be on for 244 | # csvlogs. 245 | # (change requires restart) 246 | 247 | # These are only used if logging_collector is on: 248 | #log_directory = 'pg_log' # directory where log files are written, 249 | # can be absolute or relative to PGDATA 250 | #log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log' # log file name pattern, 251 | # can include strftime() escapes 252 | #log_truncate_on_rotation = off # If on, an existing log file of the 253 | # same name as the new log file will be 254 | # truncated rather than appended to. 255 | # But such truncation only occurs on 256 | # time-driven rotation, not on restarts 257 | # or size-driven rotation. Default is 258 | # off, meaning append to existing files 259 | # in all cases. 260 | #log_rotation_age = 1d # Automatic rotation of logfiles will 261 | # happen after that time. 0 disables. 262 | #log_rotation_size = 10MB # Automatic rotation of logfiles will 263 | # happen after that much log output. 264 | # 0 disables. 265 | 266 | # These are relevant when logging to syslog: 267 | #syslog_facility = 'LOCAL0' 268 | #syslog_ident = 'postgres' 269 | 270 | #silent_mode = off # Run server silently. 271 | # DO NOT USE without syslog or 272 | # logging_collector 273 | # (change requires restart) 274 | 275 | 276 | # - When to Log - 277 | 278 | #client_min_messages = notice # values in order of decreasing detail: 279 | # debug5 280 | # debug4 281 | # debug3 282 | # debug2 283 | # debug1 284 | # log 285 | # notice 286 | # warning 287 | # error 288 | 289 | #log_min_messages = warning # values in order of decreasing detail: 290 | # debug5 291 | # debug4 292 | # debug3 293 | # debug2 294 | # debug1 295 | # info 296 | # notice 297 | # warning 298 | # error 299 | # log 300 | # fatal 301 | # panic 302 | 303 | #log_error_verbosity = default # terse, default, or verbose messages 304 | 305 | #log_min_error_statement = error # values in order of decreasing detail: 306 | # debug5 307 | # debug4 308 | # debug3 309 | # debug2 310 | # debug1 311 | # info 312 | # notice 313 | # warning 314 | # error 315 | # log 316 | # fatal 317 | # panic (effectively off) 318 | 319 | #log_min_duration_statement = -1 # -1 is disabled, 0 logs all statements 320 | # and their durations, > 0 logs only 321 | # statements running at least this number 322 | # of milliseconds 323 | 324 | 325 | # - What to Log - 326 | 327 | #debug_print_parse = off 328 | #debug_print_rewritten = off 329 | #debug_print_plan = off 330 | #debug_pretty_print = on 331 | #log_checkpoints = off 332 | #log_connections = off 333 | #log_disconnections = off 334 | #log_duration = off 335 | #log_hostname = off 336 | log_line_prefix = '%t ' # special values: 337 | # %u = user name 338 | # %d = database name 339 | # %r = remote host and port 340 | # %h = remote host 341 | # %p = process ID 342 | # %t = timestamp without milliseconds 343 | # %m = timestamp with milliseconds 344 | # %i = command tag 345 | # %c = session ID 346 | # %l = session line number 347 | # %s = session start timestamp 348 | # %v = virtual transaction ID 349 | # %x = transaction ID (0 if none) 350 | # %q = stop here in non-session 351 | # processes 352 | # %% = '%' 353 | # e.g. '<%u%%%d> ' 354 | #log_lock_waits = off # log lock waits >= deadlock_timeout 355 | #log_statement = 'none' # none, ddl, mod, all 356 | #log_temp_files = -1 # log temporary files equal or larger 357 | # than the specified size in kilobytes; 358 | # -1 disables, 0 logs all temp files 359 | #log_timezone = unknown # actually, defaults to TZ environment 360 | # setting 361 | 362 | 363 | #------------------------------------------------------------------------------ 364 | # RUNTIME STATISTICS 365 | #------------------------------------------------------------------------------ 366 | 367 | # - Query/Index Statistics Collector - 368 | 369 | #track_activities = on 370 | #Woven default 371 | track_counts = on 372 | #track_functions = none # none, pl, all 373 | #track_activity_query_size = 1024 374 | #update_process_title = on 375 | #stats_temp_directory = 'pg_stat_tmp' 376 | 377 | 378 | # - Statistics Monitoring - 379 | 380 | #log_parser_stats = off 381 | #log_planner_stats = off 382 | #log_executor_stats = off 383 | #log_statement_stats = off 384 | 385 | 386 | #------------------------------------------------------------------------------ 387 | # AUTOVACUUM PARAMETERS 388 | #------------------------------------------------------------------------------ 389 | # Woven default 390 | autovacuum = on # Enable autovacuum subprocess? 'on' 391 | # requires track_counts to also be on. 392 | #log_autovacuum_min_duration = -1 # -1 disables, 0 logs all actions and 393 | # their durations, > 0 logs only 394 | # actions running at least this number 395 | # of milliseconds. 396 | #autovacuum_max_workers = 3 # max number of autovacuum subprocesses 397 | #autovacuum_naptime = 1min # time between autovacuum runs 398 | #autovacuum_vacuum_threshold = 50 # min number of row updates before 399 | # vacuum 400 | #autovacuum_analyze_threshold = 50 # min number of row updates before 401 | # analyze 402 | #autovacuum_vacuum_scale_factor = 0.2 # fraction of table size before vacuum 403 | #autovacuum_analyze_scale_factor = 0.1 # fraction of table size before analyze 404 | #autovacuum_freeze_max_age = 200000000 # maximum XID age before forced vacuum 405 | # (change requires restart) 406 | #autovacuum_vacuum_cost_delay = 20ms # default vacuum cost delay for 407 | # autovacuum, in milliseconds; 408 | # -1 means use vacuum_cost_delay 409 | #autovacuum_vacuum_cost_limit = -1 # default vacuum cost limit for 410 | # autovacuum, -1 means use 411 | # vacuum_cost_limit 412 | 413 | 414 | #------------------------------------------------------------------------------ 415 | # CLIENT CONNECTION DEFAULTS 416 | #------------------------------------------------------------------------------ 417 | 418 | # - Statement Behavior - 419 | 420 | #search_path = '"$user",public' # schema names 421 | #default_tablespace = '' # a tablespace name, '' uses the default 422 | #temp_tablespaces = '' # a list of tablespace names, '' uses 423 | # only default tablespace 424 | #check_function_bodies = on 425 | #default_transaction_isolation = 'read committed' 426 | #default_transaction_read_only = off 427 | #session_replication_role = 'origin' 428 | #statement_timeout = 0 # in milliseconds, 0 is disabled 429 | #vacuum_freeze_min_age = 50000000 430 | #vacuum_freeze_table_age = 150000000 431 | #xmlbinary = 'base64' 432 | #xmloption = 'content' 433 | 434 | # - Locale and Formatting - 435 | 436 | datestyle = 'iso, mdy' 437 | #intervalstyle = 'postgres' 438 | #timezone = unknown # actually, defaults to TZ environment 439 | # setting 440 | #timezone_abbreviations = 'Default' # Select the set of available time zone 441 | # abbreviations. Currently, there are 442 | # Default 443 | # Australia 444 | # India 445 | # You can create your own file in 446 | # share/timezonesets/. 447 | #extra_float_digits = 0 # min -15, max 2 448 | #client_encoding = sql_ascii # actually, defaults to database 449 | # encoding 450 | 451 | # These settings are initialized by initdb, but they can be changed. 452 | lc_messages = 'C' # locale for system error message 453 | # strings 454 | lc_monetary = 'C' # locale for monetary formatting 455 | lc_numeric = 'C' # locale for number formatting 456 | lc_time = 'C' # locale for time formatting 457 | 458 | # default configuration for text search 459 | default_text_search_config = 'pg_catalog.english' 460 | 461 | # - Other Defaults - 462 | 463 | #dynamic_library_path = '$libdir' 464 | #local_preload_libraries = '' 465 | 466 | 467 | #------------------------------------------------------------------------------ 468 | # LOCK MANAGEMENT 469 | #------------------------------------------------------------------------------ 470 | 471 | #deadlock_timeout = 1s 472 | #max_locks_per_transaction = 64 # min 10 473 | # (change requires restart) 474 | # Note: Each lock table slot uses ~270 bytes of shared memory, and there are 475 | # max_locks_per_transaction * (max_connections + max_prepared_transactions) 476 | # lock table slots. 477 | 478 | 479 | #------------------------------------------------------------------------------ 480 | # VERSION/PLATFORM COMPATIBILITY 481 | #------------------------------------------------------------------------------ 482 | 483 | # - Previous PostgreSQL Versions - 484 | 485 | #add_missing_from = off 486 | #array_nulls = on 487 | #backslash_quote = safe_encoding # on, off, or safe_encoding 488 | #default_with_oids = off 489 | #escape_string_warning = on 490 | #regex_flavor = advanced # advanced, extended, or basic 491 | #sql_inheritance = on 492 | #standard_conforming_strings = off 493 | #synchronize_seqscans = on 494 | 495 | # - Other Platforms and Clients - 496 | 497 | #transform_null_equals = off 498 | 499 | 500 | #------------------------------------------------------------------------------ 501 | # CUSTOMIZED OPTIONS 502 | #------------------------------------------------------------------------------ 503 | 504 | #custom_variable_classes = '' # list of custom variable class names 505 | -------------------------------------------------------------------------------- /woven/templates/woven/gunicorn.conf: -------------------------------------------------------------------------------- 1 | description "Gunicorn" 2 | #basic gunicorn config 3 | 4 | start on runlevel [2345] 5 | stop on runlevel [!2345] 6 | #Send KILL after 5 seconds 7 | kill timeout 5 8 | respawn 9 | 10 | env VENV="{{ deployment_root }}/env/{{ project_name }}" 11 | 12 | #TODO add in -u site_1 and -g www-data when the PPA is version 0.11.3 13 | #It should launch as root and spawn workers as non-root user, but fails atm 14 | #also need and add in proper logrotate conf.. 15 | 16 | script 17 | exec $VENV/bin/python /usr/bin/gunicorn_django --log-level=info --log-file=/tmp/gunicorn.log -p /tmp/gunicorn.pid -b 127.0.0.1:10081 $VENV/project/{{ project_package_name }}/sitesettings/{{ settings }}.py 18 | end script -------------------------------------------------------------------------------- /woven/templates/woven/maintenance.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | The site is temporarily unavailable 4 | 7 | 8 | 9 | 10 | 11 | 14 | 15 |
12 | The site is temporarily down for maintenance. Please try again later. 13 |
16 | 17 | 18 | -------------------------------------------------------------------------------- /woven/templates/woven/nginx-gunicorn-template.txt: -------------------------------------------------------------------------------- 1 | #basic template for nginx with gunicorn backend 2 | server { 3 | listen 80; 4 | server_name {{ domain }}; 5 | 6 | error_log /var/log/nginx/{{ u_domain }}_nginx_error.log; 7 | access_log off; 8 | #access_log /var/log/nginx/{{ u_domain }}_nginx_access.log; 9 | 10 | 11 | location / { 12 | allow all; 13 | proxy_pass http://127.0.0.1:10081/; 14 | include /etc/nginx/proxy.conf; 15 | 16 | } 17 | 18 | error_page 502 503 504 /maintenance.html; 19 | location /maintenance.html { 20 | root /var/www/nginx-default/; 21 | } 22 | 23 | 24 | {% if MEDIA_URL %} 25 | location {{ MEDIA_URL }} { 26 | root {{ deployment_root }}/public/; 27 | 28 | } 29 | {% endif %} 30 | 31 | {% if STATIC_URL %} 32 | location {{ STATIC_URL }} { 33 | root {{ deployment_root }}/env/{{ project_name }}/static/; 34 | 35 | } 36 | {% endif %} 37 | 38 | 39 | 40 | } -------------------------------------------------------------------------------- /woven/templates/woven/nginx-template.txt: -------------------------------------------------------------------------------- 1 | server { 2 | listen 80; 3 | server_name {{ domain }}; 4 | 5 | error_log /var/log/nginx/{{ u_domain }}_nginx_error.log; 6 | access_log off; 7 | #access_log /var/log/nginx/{{ u_domain }}_nginx_access.log; 8 | 9 | location / { 10 | allow all; 11 | proxy_pass http://127.0.0.1:10080/; 12 | include /etc/nginx/proxy.conf; 13 | 14 | } 15 | 16 | error_page 502 503 504 /maintenance.html; 17 | location /maintenance.html { 18 | root /var/www/nginx-default/; 19 | } 20 | 21 | 22 | {% if MEDIA_URL %} 23 | location {{ MEDIA_URL }} { 24 | root {{ deployment_root }}/public/; 25 | 26 | } 27 | {% endif %} 28 | 29 | {% if STATIC_URL %} 30 | location {{ STATIC_URL }} { 31 | root {{ deployment_root }}/env/{{ project_name }}/static/; 32 | 33 | } 34 | {% endif %} 35 | 36 | 37 | 38 | } -------------------------------------------------------------------------------- /woven/templates/woven/requirements.txt: -------------------------------------------------------------------------------- 1 | #Set your project requirements here 2 | 3 | #By default the current local ./manage.py --version of Django is installed automatically before any requirements 4 | 5 | #Requirements files can be named in order of installation. Anything with req* will be treated as a requirement file 6 | #eg requirements, req1, requirements2.txt ... 7 | 8 | #Bundles with the same name as the requirements files and .zip extension will be used 9 | #if they exist in the dist directory. See also the bundle management command. 10 | 11 | {{ django }} -------------------------------------------------------------------------------- /woven/templates/woven/sitesettings.txt: -------------------------------------------------------------------------------- 1 | #Created from the sitesettings.txt template 2 | #This module is the base for all production/staging sites 3 | 4 | #Import global project settings 5 | from {{ project_package_name }}.settings import * 6 | 7 | #operating system USER determines site_id for multi-site capability 8 | from getpass import getuser 9 | 10 | USER = getuser() 11 | if 'site' not in USER: USER = 'site_1' 12 | try: 13 | SITE_ID = int(USER[-1]) 14 | except ValueError: 15 | SITE_ID = 1 16 | 17 | 18 | DEBUG = False 19 | TEMPLATE_DEBUG = DEBUG 20 | 21 | DATABASES = { 22 | 'default': { 23 | 'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'. 24 | 'NAME': ''.join(['{{ deployment_root }}/database/','{{ project_name }}','_',USER,'.db']), # Or path to database file if using sqlite3. 25 | 'USER': '', # Not used with sqlite3. 26 | 'PASSWORD': '', # Not used with sqlite3. 27 | 'HOST': '', # Set to empty string for localhost. Not used with sqlite3. 28 | 'PORT': '', # Set to empty string for default. Not used with sqlite3. 29 | } 30 | } 31 | 32 | 33 | #Normally you shouldn't amend these settings, unless you have changed your local media settings 34 | MEDIA_ROOT = '{{ deployment_root }}/public/{{ MEDIA_URL|slice:"1:" }}' 35 | STATIC_ROOT = '{{ deployment_root }}/env/{{ project_name }}/static/{{ STATIC_URL|slice:"1:" }}' 36 | 37 | TEMPLATE_DIRS = ( 38 | '{{ deployment_root }}/env/{{ project_name }}/templates/', 39 | ) -------------------------------------------------------------------------------- /woven/templates/woven/ssh/sshd_config: -------------------------------------------------------------------------------- 1 | # Woven sshd_config template - from Ubuntu 10.04 2 | 3 | # WARNING: if you alter this file you can potentially lock yourself out of your 4 | # instance, if you make incorrect settings. 5 | 6 | # Package generated configuration file 7 | # See the sshd_config(5) manpage for details 8 | 9 | # What ports, IPs and protocols we listen for 10 | Port {{ HOST_SSH_PORT }} 11 | # Use these options to restrict which interfaces/protocols sshd will bind to 12 | #ListenAddress :: 13 | #ListenAddress 0.0.0.0 14 | Protocol 2 15 | # HostKeys for protocol version 2 16 | HostKey /etc/ssh/ssh_host_rsa_key 17 | HostKey /etc/ssh/ssh_host_dsa_key 18 | #Privilege Separation is turned on for security 19 | UsePrivilegeSeparation yes 20 | 21 | # Lifetime and size of ephemeral version 1 server key 22 | KeyRegenerationInterval 3600 23 | ServerKeyBits 768 24 | 25 | # Logging 26 | SyslogFacility AUTH 27 | LogLevel INFO 28 | 29 | # Authentication: 30 | LoginGraceTime 120 31 | PermitRootLogin yes 32 | StrictModes yes 33 | 34 | RSAAuthentication yes 35 | PubkeyAuthentication yes 36 | 37 | #Woven uncomments the AuthorizedKeysFile 38 | AuthorizedKeysFile %h/.ssh/authorized_keys 39 | 40 | # Don't read the user's ~/.rhosts and ~/.shosts files 41 | IgnoreRhosts yes 42 | # For this to work you will also need host keys in /etc/ssh_known_hosts 43 | RhostsRSAAuthentication no 44 | # similar for protocol version 2 45 | HostbasedAuthentication no 46 | # Uncomment if you don't trust ~/.ssh/known_hosts for RhostsRSAAuthentication 47 | #IgnoreUserKnownHosts yes 48 | 49 | # To enable empty passwords, change to yes (NOT RECOMMENDED) 50 | PermitEmptyPasswords no 51 | 52 | # Change to yes to enable challenge-response passwords (beware issues with 53 | # some PAM modules and threads) 54 | ChallengeResponseAuthentication no 55 | 56 | # Change to no to disable tunnelled clear text passwords 57 | # Woven uncomments this on demand 58 | #PasswordAuthentication no 59 | 60 | # Kerberos options 61 | #KerberosAuthentication no 62 | #KerberosGetAFSToken no 63 | #KerberosOrLocalPasswd yes 64 | #KerberosTicketCleanup yes 65 | 66 | # GSSAPI options 67 | #GSSAPIAuthentication no 68 | #GSSAPICleanupCredentials yes 69 | 70 | #Woven changes the default to no 71 | X11Forwarding no 72 | X11DisplayOffset 10 73 | PrintMotd no 74 | PrintLastLog yes 75 | TCPKeepAlive yes 76 | #UseLogin no 77 | 78 | #MaxStartups 10:30:60 79 | #Banner /etc/issue.net 80 | 81 | # Allow client to pass locale environment variables 82 | AcceptEnv LANG LC_* 83 | 84 | Subsystem sftp /usr/lib/openssh/sftp-server 85 | 86 | # Set this to 'yes' to enable PAM authentication, account processing, 87 | # and session processing. If this is enabled, PAM authentication will 88 | # be allowed through the ChallengeResponseAuthentication and 89 | # PasswordAuthentication. Depending on your PAM configuration, 90 | # PAM authentication via ChallengeResponseAuthentication may bypass 91 | # the setting of "PermitRootLogin without-password". 92 | # If you just want the PAM account and session checks to run without 93 | # PAM authentication, then enable this but set PasswordAuthentication 94 | # and ChallengeResponseAuthentication to 'no'. 95 | UsePAM yes 96 | 97 | #Woven adds the UseDNS setting 98 | UseDNS no 99 | -------------------------------------------------------------------------------- /woven/templates/woven/ufw.txt: -------------------------------------------------------------------------------- 1 | [woven] 2 | title=Woven 3 | description=Django deployment app 4 | ports={{ HOST_SSH_PORT }}/tcp -------------------------------------------------------------------------------- /woven/virtualenv.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from glob import glob 3 | import os, sys 4 | import site 5 | 6 | from django import get_version 7 | from django.template.loader import render_to_string 8 | 9 | 10 | from fabric.decorators import runs_once 11 | from fabric.state import env 12 | from fabric.operations import run, sudo 13 | from fabric.context_managers import cd, settings 14 | from fabric.contrib.files import exists 15 | from fabric.contrib.console import confirm 16 | 17 | from woven.decorators import run_once_per_version 18 | from woven.deployment import mkdirs, deploy_files 19 | from woven.environment import deployment_root,set_version_state, version_state, get_packages 20 | from woven.environment import post_exec_hook, State 21 | from woven.webservers import _get_django_sites, _ls_sites, _sitesettings_files, stop_webserver, start_webserver, webserver_list, domain_sites 22 | from fabric.contrib.files import append 23 | 24 | def active_version(): 25 | """ 26 | Determine the current active version on the server 27 | 28 | Just examine the which environment is symlinked 29 | """ 30 | 31 | link = '/'.join([deployment_root(),'env',env.project_name]) 32 | if not exists(link): return None 33 | active = os.path.split(run('ls -al '+link).split(' -> ')[1])[1] 34 | return active 35 | 36 | def activate(): 37 | """ 38 | Activates the version specified in ``env.project_version`` if it is different 39 | from the current active version. 40 | 41 | An active version is just the version that is symlinked. 42 | """ 43 | 44 | env_path = '/'.join([deployment_root(),'env',env.project_fullname]) 45 | 46 | if not exists(env_path): 47 | print env.host,"ERROR: The version",env.project_version,"does not exist at" 48 | print env_path 49 | sys.exit(1) 50 | 51 | active = active_version() 52 | servers = webserver_list() 53 | 54 | if env.patch or active <> env.project_fullname: 55 | for s in servers: 56 | stop_webserver(s) 57 | 58 | if not env.patch and active <> env.project_fullname: 59 | 60 | if env.verbosity: 61 | print env.host, "ACTIVATING version", env_path 62 | 63 | if not env.nomigration: 64 | sync_db() 65 | 66 | #south migration 67 | if 'south' in env.INSTALLED_APPS and not env.nomigration and not env.manualmigration: 68 | migration() 69 | 70 | if env.manualmigration or env.MANUAL_MIGRATION: manual_migration() 71 | 72 | #activate sites 73 | activate_sites = [''.join([d.name.replace('.','_'),'-',env.project_version,'.conf']) for d in domain_sites()] 74 | if 'apache2' in get_packages(): 75 | site_paths = ['/etc/apache2','/etc/nginx'] 76 | else: 77 | site_paths = ['/etc/nginx'] 78 | 79 | #disable existing sites 80 | for path in site_paths: 81 | for site in _ls_sites('/'.join([path,'sites-enabled'])): 82 | if site not in activate_sites: 83 | sudo("rm %s/sites-enabled/%s"% (path,site)) 84 | 85 | #activate new sites 86 | for path in site_paths: 87 | for site in activate_sites: 88 | if not exists('/'.join([path,'sites-enabled',site])): 89 | sudo("chmod 644 %s" % '/'.join([path,'sites-available',site])) 90 | sudo("ln -s %s/sites-available/%s %s/sites-enabled/%s"% (path,site,path,site)) 91 | if env.verbosity: 92 | print " * enabled", "%s/sites-enabled/%s"% (path,site) 93 | 94 | #delete existing symlink 95 | ln_path = '/'.join([deployment_root(),'env',env.project_name]) 96 | run('rm -f '+ln_path) 97 | #run post deploy hooks 98 | post_exec_hook('post_deploy') 99 | #activate 100 | run('ln -s %s %s'% (env_path,ln_path)) 101 | 102 | 103 | if env.verbosity: 104 | print env.host,env.project_fullname, "ACTIVATED" 105 | else: 106 | if env.verbosity and not env.patch: 107 | print env.project_fullname,"is the active version" 108 | 109 | if env.patch or active <> env.project_fullname: 110 | for s in servers: 111 | start_webserver(s) 112 | print 113 | return 114 | 115 | @runs_once 116 | def sync_db(): 117 | """ 118 | Runs the django syncdb command 119 | """ 120 | with cd('/'.join([deployment_root(),'env',env.project_fullname,'project',env.project_package_name,'sitesettings'])): 121 | venv = '/'.join([deployment_root(),'env',env.project_fullname,'bin','activate']) 122 | sites = _get_django_sites() 123 | site_ids = sites.keys() 124 | site_ids.sort() 125 | for site in site_ids: 126 | for settings_file in _sitesettings_files(): 127 | site_settings = '.'.join([env.project_package_name,'sitesettings',settings_file.replace('.py','')]) 128 | if env.verbosity: 129 | print " * django-admin.py syncdb --noinput --settings=%s"% site_settings 130 | output = sudo(' '.join(['source',venv,'&&',"django-admin.py syncdb --noinput --settings=%s"% site_settings]), 131 | user='site_%s'% site) 132 | if env.verbosity: 133 | print output 134 | 135 | @runs_once 136 | def manual_migration(): 137 | """ 138 | Simple interactive function to pause the deployment. 139 | A manual migration can be done two different ways: 140 | Option 1: Enter y to exit the current deployment. When migration is completed run deploy again. 141 | Option 2: run the migration in a separate shell 142 | """ 143 | if env.INTERACTIVITY: 144 | print "A manual migration can be done two different ways:" 145 | print "Option 1: Enter y to exit the current deployment. When migration is completed run deploy again." 146 | print "Option 2: run the migration in a separate shell" 147 | exit = confirm("Enter y to exit or accept default to complete deployment and activate the new version",default=False) 148 | else: 149 | exit = True 150 | if exit: 151 | print "Login to your node and run 'workon %s'"% env.project_fullname 152 | sys.exit(0) 153 | 154 | @runs_once 155 | def migration(): 156 | """ 157 | Integrate with south schema migration 158 | """ 159 | 160 | #activate env 161 | with cd('/'.join([deployment_root(),'env',env.project_fullname,'project',env.project_package_name,'sitesettings'])): 162 | #migrates all or specific env.migration 163 | venv = '/'.join([deployment_root(),'env',env.project_fullname,'bin','activate']) 164 | cmdpt1 = ' '.join(['source',venv,'&&']) 165 | 166 | sites = _get_django_sites() 167 | site_ids = sites.keys() 168 | site_ids.sort() 169 | for site in site_ids: 170 | for settings_file in _sitesettings_files(): 171 | site_settings = '.'.join([env.project_package_name,'sitesettings',settings_file.replace('.py','')]) 172 | cmdpt2 = ' '.join(["django-admin.py migrate",env.migration]) 173 | if hasattr(env,"fakemigration"): 174 | cmdpt2 = ' '.join([cmdpt2,'--fake']) 175 | cmdpt2 = ''.join([cmdpt2,'--settings=',site_settings]) 176 | if env.verbosity: 177 | print " *", cmdpt2 178 | output = sudo(' '.join([cmdpt1,cmdpt2]),user='site_%s'% site) 179 | if env.verbosity: 180 | print output 181 | return 182 | 183 | @run_once_per_version 184 | def mkvirtualenv(): 185 | """ 186 | Create the virtualenv project environment 187 | """ 188 | root = '/'.join([deployment_root(),'env']) 189 | path = '/'.join([root,env.project_fullname]) 190 | dirs_created = [] 191 | if env.verbosity: 192 | print env.host,'CREATING VIRTUALENV', path 193 | if not exists(root): dirs_created += mkdirs(root) 194 | with cd(root): 195 | run(' '.join(["virtualenv",env.project_fullname])) 196 | with cd(path): 197 | dirs_created += mkdirs('egg_cache') 198 | sudo('chown -R %s:www-data egg_cache'% env.user) 199 | sudo('chmod -R g+w egg_cache') 200 | run(''.join(["echo 'cd ",path,'/','project','/',env.project_package_name,'/sitesettings',"' > bin/postactivate"])) 201 | sudo('chmod ugo+rwx bin/postactivate') 202 | 203 | #Create a state 204 | out = State(' '.join([env.host,'virtualenv',path,'created'])) 205 | out.object = dirs_created + ['bin','lib','include'] 206 | out.failed = False 207 | return out 208 | 209 | def rmvirtualenv(): 210 | """ 211 | Remove the current or ``env.project_version`` environment and all content in it 212 | """ 213 | path = '/'.join([deployment_root(),'env',env.project_fullname]) 214 | link = '/'.join([deployment_root(),'env',env.project_name]) 215 | if version_state('mkvirtualenv'): 216 | sudo(' '.join(['rm -rf',path])) 217 | sudo(' '.join(['rm -f',link])) 218 | sudo('rm -f /var/local/woven/%s*'% env.project_fullname) 219 | set_version_state('mkvirtualenv',delete=True) 220 | 221 | 222 | @run_once_per_version 223 | def pip_install_requirements(): 224 | """ 225 | Install on current installed virtualenv version from a pip bundle [dist/project name-version].zip or pip ``req.txt``|``requirements.txt`` 226 | or a env.pip_requirements list. 227 | 228 | By default it will look for a zip bundle in the dist directory first then a requirements file. 229 | 230 | 231 | The limitations of installing requirements are that you cannot point directly to packages 232 | in your local filesystem. In this case you would bundle instead. 233 | """ 234 | if not version_state('mkvirtualenv'): 235 | print env.host,'Error: Cannot run pip_install_requirements. A virtualenv is not created for this version. Run mkvirtualenv first' 236 | return 237 | if env.verbosity: 238 | print env.host, 'PIP INSTALLING REQUIREMENTS:' 239 | 240 | #Remove any pre-existing pip-log from any previous failed installation 241 | pip_log_dir = '/'.join(['/home',env.user,'.pip']) 242 | if exists(pip_log_dir): run('rm -f %s/*.txt'% pip_log_dir) 243 | 244 | #determine what req files or bundle files we need to deploy 245 | if not env.PIP_REQUIREMENTS: 246 | req_files = {}.fromkeys(glob('req*')) 247 | else: 248 | req_files = {}.fromkeys(env.PIP_REQUIREMENTS) 249 | 250 | for key in req_files: 251 | bundle = ''.join([key.split('.')[0],'.zip']) 252 | if os.path.exists(os.path.join('dist',bundle)): 253 | req_files[key] = bundle 254 | 255 | #determine the django version 256 | file_patterns ='' 257 | django_version = get_version() 258 | svn_version = django_version.find('SVN') 259 | if svn_version > -1: 260 | django_version = django_version[svn_version+4:] 261 | django_req = ''.join(['-e svn+http://code.djangoproject.com/svn/django/trunk@',django_version,'#egg=Django']) 262 | else: 263 | other_builds = ['alpha','beta','rc'] 264 | for b in other_builds: 265 | if b in django_version: 266 | print "ERROR: Unsupported Django version", django_version 267 | print "Define a DJANGO_REQUIREMENT pointing to the tar.gz for",django_version 268 | print "and re-deploy, or use the official or SVN release of Django." 269 | sys.exit(1) 270 | django_req = ''.join(['Django==',django_version]) 271 | 272 | #if no requirements file exists create one 273 | if not req_files: 274 | f = open("requirements.txt","w+") 275 | text = render_to_string('woven/requirements.txt', {'django':django_req}) 276 | f.write(text) 277 | f.close() 278 | if env.verbosity: 279 | print "Created local requirements.txt" 280 | req_files["requirements.txt"]='' 281 | 282 | req_files_list = req_files.keys() 283 | req_files_list.sort() 284 | 285 | #patterns for bundles 286 | if req_files: file_patterns = '|'.join([file_patterns,'req*.zip']) 287 | 288 | #create a pip cache & src directory 289 | cache = '/'.join([deployment_root(),'.pip','cache']) 290 | src = '/'.join([deployment_root(),'.pip','src']) 291 | deployed = mkdirs(cache) 292 | deployed += mkdirs(src) 293 | #deploy bundles and any local copy of django 294 | local_dir = os.path.join(os.getcwd(),'dist') 295 | remote_dir = '/'.join([deployment_root(),'env',env.project_fullname,'dist']) 296 | if os.path.exists(local_dir): 297 | if file_patterns: deployed += deploy_files(local_dir, remote_dir, pattern=file_patterns) 298 | 299 | #deploy any requirement files 300 | deployed += deploy_files(os.getcwd(), remote_dir, pattern = 'req*') 301 | 302 | #install in the env 303 | out = State(' '.join([env.host,'pip install requirements'])) 304 | python_path = '/'.join([deployment_root(),'env',env.project_fullname,'bin','python']) 305 | with settings(warn_only=True): 306 | with cd(remote_dir): 307 | for req in req_files_list: 308 | bundle = req_files[req] 309 | if bundle: req=bundle 310 | if env.verbosity: 311 | print ' * installing',req 312 | if '.zip' in req.lower(): 313 | install = run('pip install %s -q --environment=%s --log=/home/%s/.pip/%s_pip_log.txt'% 314 | (req, python_path, env.user, req.replace('.','_'))) 315 | 316 | else: 317 | install = run('pip install -q --environment=%s --src=%s --download-cache=%s --requirement=%s --log=/home/%s/.pip/%s_pip_log.txt'% 318 | (python_path,src,cache,req, env.user,req.replace('.','_'))) 319 | if install.failed: 320 | out.failed =True 321 | out.stderr += ' '.join([env.host, "ERROR INSTALLING",req,'\n']) 322 | 323 | out.object = deployed 324 | 325 | if out.failed: 326 | print out.stderr 327 | print "Review the pip install logs at %s/.pip and re-deploy"% deployment_root() 328 | sys.exit(1) 329 | return out 330 | -------------------------------------------------------------------------------- /woven/webservers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os,socket, sys 3 | import json 4 | 5 | from fabric.state import _AttributeDict, env 6 | from fabric.operations import run, sudo 7 | from fabric.context_managers import cd, settings 8 | from fabric.contrib.files import append, contains, exists 9 | from fabric.decorators import runs_once 10 | 11 | from woven.decorators import run_once_per_version 12 | from woven.deployment import deploy_files, mkdirs, upload_template 13 | from woven.environment import deployment_root, version_state, _root_domain, get_packages 14 | from woven.linux import add_user 15 | 16 | def _activate_sites(path, filenames): 17 | enabled_sites = _ls_sites(path) 18 | for site in enabled_sites: 19 | if env.verbosity: 20 | print env.host,'Disabling', site 21 | if site not in filenames: 22 | sudo("rm %s/%s"% (path,site)) 23 | 24 | sudo("chmod 644 %s" % site) 25 | if not exists('/etc/apache2/sites-enabled'+ filename): 26 | sudo("ln -s %s%s %s%s"% (self.deploy_root,filename,self.enabled_path,filename)) 27 | 28 | def _deploy_webconf(remote_dir,template): 29 | 30 | if not 'http:' in env.MEDIA_URL: media_url = env.MEDIA_URL 31 | else: media_url = '' 32 | if not 'http:' in env.STATIC_URL: static_url = env.STATIC_URL 33 | else: static_url = '' 34 | if not static_url: static_url = env.ADMIN_MEDIA_PREFIX 35 | log_dir = '/'.join([deployment_root(),'log']) 36 | deployed = [] 37 | users_added = [] 38 | 39 | domains = domain_sites() 40 | for d in domains: 41 | u_domain = d.name.replace('.','_') 42 | wsgi_filename = d.settings.replace('.py','.wsgi') 43 | site_user = ''.join(['site_',str(d.site_id)]) 44 | filename = ''.join([remote_dir,'/',u_domain,'-',env.project_version,'.conf']) 45 | context = {"project_name": env.project_name, 46 | "deployment_root":deployment_root(), 47 | "u_domain":u_domain, 48 | "domain":d.name, 49 | "root_domain":env.root_domain, 50 | "user":env.user, 51 | "site_user":site_user, 52 | "SITE_ID":d.site_id, 53 | "host_ip":socket.gethostbyname(env.host), 54 | "wsgi_filename":wsgi_filename, 55 | "MEDIA_URL":media_url, 56 | "STATIC_URL":static_url, 57 | } 58 | 59 | upload_template('/'.join(['woven',template]), 60 | filename, 61 | context, 62 | use_sudo=True) 63 | if env.verbosity: 64 | print " * uploaded", filename 65 | 66 | #add site users if necessary 67 | site_users = _site_users() 68 | if site_user not in users_added and site_user not in site_users: 69 | add_user(username=site_user,group='www-data',site_user=True) 70 | users_added.append(site_user) 71 | if env.verbosity: 72 | print " * useradded",site_user 73 | 74 | return deployed 75 | 76 | def _site_users(): 77 | """ 78 | Get a list of site_n users 79 | """ 80 | userlist = sudo("cat /etc/passwd | awk '/site/'").split('\n') 81 | siteuserlist = [user.split(':')[0] for user in userlist if 'site_' in user] 82 | return siteuserlist 83 | 84 | def _ls_sites(path): 85 | """ 86 | List only sites in the domain_sites() to ensure we co-exist with other projects 87 | """ 88 | with cd(path): 89 | sites = run('ls').split('\n') 90 | doms = [d.name for d in domain_sites()] 91 | dom_sites = [] 92 | for s in sites: 93 | ds = s.split('-')[0] 94 | ds = ds.replace('_','.') 95 | if ds in doms and s not in dom_sites: 96 | dom_sites.append(s) 97 | return dom_sites 98 | 99 | 100 | 101 | def _sitesettings_files(): 102 | """ 103 | Get a list of sitesettings files 104 | 105 | settings.py can be prefixed with a subdomain and underscore so with example.com site: 106 | sitesettings/settings.py would be the example.com settings file and 107 | sitesettings/admin_settings.py would be the admin.example.com settings file 108 | """ 109 | settings_files = [] 110 | sitesettings_path = os.path.join(env.project_package_name,'sitesettings') 111 | if os.path.exists(sitesettings_path): 112 | sitesettings = os.listdir(sitesettings_path) 113 | for file in sitesettings: 114 | if file == 'settings.py': 115 | settings_files.append(file) 116 | elif len(file)>12 and file[-12:]=='_settings.py': #prefixed settings 117 | settings_files.append(file) 118 | return settings_files 119 | 120 | def _get_django_sites(): 121 | """ 122 | Get a list of sites as dictionaries {site_id:'domain.name'} 123 | 124 | """ 125 | deployed = version_state('deploy_project') 126 | if not env.sites and 'django.contrib.sites' in env.INSTALLED_APPS and deployed: 127 | with cd('/'.join([deployment_root(),'env',env.project_fullname,'project',env.project_package_name,'sitesettings'])): 128 | venv = '/'.join([deployment_root(),'env',env.project_fullname,'bin','activate']) 129 | #since this is the first time we run ./manage.py on the server it can be 130 | #a point of failure for installations 131 | with settings(warn_only=True): 132 | output = run(' '.join(['source',venv,'&&',"django-admin.py dumpdata sites --settings=%s.sitesettings.settings"% env.project_package_name])) 133 | 134 | if output.failed: 135 | print "ERROR: There was an error running ./manage.py on the node" 136 | print "See the troubleshooting docs for hints on how to diagnose deployment issues" 137 | if hasattr(output, 'stderr'): 138 | print output.stderr 139 | sys.exit(1) 140 | output = output.split('\n')[-1] #ignore any lines prior to the data being dumped 141 | sites = json.loads(output) 142 | env.sites = {} 143 | for s in sites: 144 | env.sites[s['pk']] = s['fields']['domain'] 145 | return env.sites 146 | 147 | def domain_sites(): 148 | """ 149 | Get a list of domains 150 | 151 | Each domain is an attribute dict with name, site_id and settings 152 | """ 153 | 154 | if not hasattr(env,'domains'): 155 | sites = _get_django_sites() 156 | site_ids = sites.keys() 157 | site_ids.sort() 158 | domains = [] 159 | 160 | for id in site_ids: 161 | 162 | for file in _sitesettings_files(): 163 | domain = _AttributeDict({}) 164 | 165 | if file == 'settings.py': 166 | domain.name = sites[id] 167 | else: #prefix indicates subdomain 168 | subdomain = file[:-12].replace('_','.') 169 | domain.name = ''.join([subdomain,sites[id]]) 170 | 171 | domain.settings = file 172 | domain.site_id = id 173 | domains.append(domain) 174 | 175 | env.domains = domains 176 | if env.domains: env.root_domain = env.domains[0].name 177 | else: 178 | domain.name = _root_domain(); domain.site_id = 1; domain.settings='settings.py' 179 | env.domains = [domain] 180 | 181 | return env.domains 182 | 183 | @run_once_per_version 184 | def deploy_webconf(): 185 | """ Deploy nginx and other wsgi server site configurations to the host """ 186 | deployed = [] 187 | log_dir = '/'.join([deployment_root(),'log']) 188 | #TODO - incorrect - check for actual package to confirm installation 189 | if webserver_list(): 190 | if env.verbosity: 191 | print env.host,"DEPLOYING webconf:" 192 | if not exists(log_dir): 193 | run('ln -s /var/log log') 194 | #deploys confs for each domain based on sites app 195 | if 'apache2' in get_packages(): 196 | deployed += _deploy_webconf('/etc/apache2/sites-available','django-apache-template.txt') 197 | deployed += _deploy_webconf('/etc/nginx/sites-available','nginx-template.txt') 198 | elif 'gunicorn' in get_packages(): 199 | deployed += _deploy_webconf('/etc/nginx/sites-available','nginx-gunicorn-template.txt') 200 | 201 | if not exists('/var/www/nginx-default'): 202 | sudo('mkdir /var/www/nginx-default') 203 | upload_template('woven/maintenance.html','/var/www/nginx-default/maintenance.html',use_sudo=True) 204 | sudo('chmod ugo+r /var/www/nginx-default/maintenance.html') 205 | else: 206 | print env.host,"""WARNING: Apache or Nginx not installed""" 207 | 208 | return deployed 209 | 210 | @run_once_per_version 211 | def deploy_wsgi(): 212 | """ 213 | deploy python wsgi file(s) 214 | """ 215 | if 'libapache2-mod-wsgi' in get_packages(): 216 | remote_dir = '/'.join([deployment_root(),'env',env.project_fullname,'wsgi']) 217 | wsgi = 'apache2' 218 | elif 'gunicorn' in get_packages(): 219 | remote_dir = '/etc/init' 220 | wsgi = 'gunicorn' 221 | deployed = [] 222 | 223 | #ensure project apps path is also added to environment variables as well as wsgi 224 | if env.PROJECT_APPS_PATH: 225 | pap = '/'.join([deployment_root(),'env', 226 | env.project_name,'project',env.project_package_name,env.PROJECT_APPS_PATH]) 227 | pap = ''.join(['export PYTHONPATH=$PYTHONPATH:',pap]) 228 | postactivate = '/'.join([deployment_root(),'env','postactivate']) 229 | if not exists(postactivate): 230 | append('#!/bin/bash', postactivate) 231 | run('chmod +x %s'% postactivate) 232 | if not contains('PYTHONPATH',postactivate): 233 | append(pap,postactivate) 234 | 235 | if env.verbosity: 236 | print env.host,"DEPLOYING wsgi", wsgi, remote_dir 237 | 238 | for file in _sitesettings_files(): 239 | deployed += mkdirs(remote_dir) 240 | with cd(remote_dir): 241 | settings_module = file.replace('.py','') 242 | context = {"deployment_root":deployment_root(), 243 | "user": env.user, 244 | "project_name": env.project_name, 245 | "project_package_name": env.project_package_name, 246 | "project_apps_path":env.PROJECT_APPS_PATH, 247 | "settings": settings_module, 248 | } 249 | if wsgi == 'apache2': 250 | filename = file.replace('.py','.wsgi') 251 | upload_template('/'.join(['woven','django-wsgi-template.txt']), 252 | filename, 253 | context, 254 | ) 255 | elif wsgi == 'gunicorn': 256 | filename = 'gunicorn-%s.conf'% env.project_name 257 | upload_template('/'.join(['woven','gunicorn.conf']), 258 | filename, 259 | context, 260 | backup=False, 261 | use_sudo=True 262 | ) 263 | 264 | if env.verbosity: 265 | print " * uploaded", filename 266 | #finally set the ownership/permissions 267 | #We'll use the group to allow www-data execute 268 | if wsgi == 'apache2': 269 | sudo("chown %s:www-data %s"% (env.user,filename)) 270 | run("chmod ug+xr %s"% filename) 271 | elif wsgi == 'gunicorn': 272 | sudo("chown root:root %s"% filename) 273 | sudo("chmod go+r %s"% filename) 274 | 275 | return deployed 276 | 277 | def webserver_list(): 278 | """ 279 | list of webserver packages 280 | """ 281 | p = set(get_packages()) 282 | w = set(['apache2','gunicorn','uwsgi','nginx']) 283 | installed = p & w 284 | return list(installed) 285 | 286 | def reload_webservers(): 287 | """ 288 | Reload apache2 and nginx 289 | """ 290 | if env.verbosity: 291 | print env.host, "RELOADING apache2" 292 | with settings(warn_only=True): 293 | a = sudo("/etc/init.d/apache2 reload") 294 | if env.verbosity: 295 | print '',a 296 | if env.verbosity: 297 | 298 | #Reload used to fail on Ubuntu but at least in 10.04 it works 299 | print env.host,"RELOADING nginx" 300 | with settings(warn_only=True): 301 | s = run("/etc/init.d/nginx status") 302 | if 'running' in s: 303 | n = sudo("/etc/init.d/nginx reload") 304 | else: 305 | n = sudo("/etc/init.d/nginx start") 306 | if env.verbosity: 307 | print ' *',n 308 | return True 309 | 310 | def stop_webserver(server): 311 | """ 312 | Stop server 313 | """ 314 | #TODO - distinguish between a warning and a error on apache 315 | if server == 'apache2': 316 | with settings(warn_only=True): 317 | if env.verbosity: 318 | print env.host,"STOPPING apache2" 319 | a = sudo("/etc/init.d/apache2 stop") 320 | if env.verbosity: 321 | print '',a 322 | elif server == 'gunicorn': 323 | with settings(warn_only=True): 324 | if env.verbosity: 325 | print env.host,"STOPPING","%s-%s"% (server,env.project_name) 326 | a = sudo("stop %s-%s"% (server,env.project_name)) 327 | if env.verbosity and a.strip(): 328 | print '',a 329 | return True 330 | 331 | def start_webserver(server): 332 | """ 333 | Start server 334 | """ 335 | if server == 'apache2': 336 | with settings(warn_only=True): 337 | if env.verbosity: 338 | print env.host,"STARTING apache2" 339 | #some issues with pty=True getting apache to start on ec2 340 | a = sudo("/etc/init.d/apache2 start", pty=False) 341 | if env.verbosity: 342 | print '',a 343 | 344 | if a.failed: 345 | print "ERROR: /etc/init.d/apache2 start failed" 346 | print env.host, a 347 | sys.exit(1) 348 | elif server == 'nginx': 349 | if env.verbosity: 350 | #Reload used to fail on Ubuntu but at least in 10.04 it works 351 | print env.host,"RELOADING nginx" 352 | with settings(warn_only=True): 353 | s = run("/etc/init.d/nginx status") 354 | if 'running' in s: 355 | n = sudo("/etc/init.d/nginx reload") 356 | else: 357 | n = sudo("/etc/init.d/nginx start") 358 | if env.verbosity: 359 | print ' *',n 360 | else: 361 | if env.verbosity: 362 | print env.host, "STARTING","%s-%s"% (server,env.project_name) 363 | with settings(warn_only=True): 364 | n = sudo('start %s-%s'% (server,env.project_name)) 365 | if env.verbosity and n.strip(): 366 | print ' *', n 367 | 368 | return True 369 | 370 | --------------------------------------------------------------------------------