├── __init__.py ├── essay ├── tasks │ ├── __init__.py │ ├── pypi.py │ ├── process.py │ ├── config.py │ ├── util.py │ ├── fs.py │ ├── deploy.py │ ├── validate.py │ ├── nginx.py │ ├── git.py │ ├── virtualenv.py │ ├── package.py │ ├── supervisor.py │ └── build.py ├── templates │ ├── default │ │ ├── README.tpl │ │ ├── __project__ │ │ │ ├── __init__.py.tpl │ │ │ ├── requirements.txt │ │ │ ├── settings.py.tpl │ │ │ ├── main.py.tpl │ │ │ └── log.py.tpl │ │ ├── MANIFEST.in.tpl │ │ ├── .gitignore │ │ ├── setup.py.tpl │ │ ├── fabfile │ │ │ └── __init__.py.tpl │ │ └── conf │ │ │ └── supervisord.conf.tpl │ ├── init │ │ ├── README.tpl │ │ ├── MANIFEST.in.tpl │ │ ├── setup.py.tpl │ │ ├── fabfile │ │ │ └── __init__.py.tpl │ │ └── conf │ │ │ └── supervisord.conf.tpl │ └── django │ │ ├── __project__ │ │ ├── __init__.py.tpl │ │ ├── __project__ │ │ │ ├── __init__.py.tpl │ │ │ ├── urls.py.tpl │ │ │ ├── wsgi.py.tpl │ │ │ └── settings.py.tpl │ │ ├── requirements.txt │ │ └── manage.py.tpl │ │ ├── README.tpl │ │ ├── MANIFEST.in.tpl │ │ ├── .gitignore │ │ ├── setup.py.tpl │ │ ├── fabfile │ │ └── __init__.py.tpl │ │ └── conf │ │ └── supervisord.conf.tpl ├── __init__.py ├── settings.py ├── pip_.py ├── utils.py ├── log.py ├── main.py └── project.py ├── requirements.txt ├── setup.cfg ├── MANIFEST.in ├── data ├── 1.png └── 2.png ├── sphinx ├── source │ ├── modules.rst │ ├── index.rst │ ├── essay.rst │ ├── essay.tasks.rst │ └── conf.py └── Makefile ├── .flake8 ├── .gitignore ├── release.md ├── .travis.yml ├── fabfile └── __init__.py ├── setup.py ├── conf └── supervisord.conf └── README.md /__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /essay/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Fabric 2 | Jinja2 3 | -------------------------------------------------------------------------------- /essay/templates/default/README.tpl: -------------------------------------------------------------------------------- 1 | HELLO README! -------------------------------------------------------------------------------- /essay/templates/init/README.tpl: -------------------------------------------------------------------------------- 1 | HELLO README! -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal = 1 3 | -------------------------------------------------------------------------------- /essay/templates/default/__project__/__init__.py.tpl: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /essay/templates/django/__project__/__init__.py.tpl: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /essay/templates/django/README.tpl: -------------------------------------------------------------------------------- 1 | A DJANGO PROJECT'S README! -------------------------------------------------------------------------------- /essay/templates/django/__project__/__project__/__init__.py.tpl: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | recursive-include essay *.tpl 3 | -------------------------------------------------------------------------------- /data/1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SohuTech/essay/HEAD/data/1.png -------------------------------------------------------------------------------- /data/2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SohuTech/essay/HEAD/data/2.png -------------------------------------------------------------------------------- /sphinx/source/modules.rst: -------------------------------------------------------------------------------- 1 | essay 2 | ===== 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | essay 8 | -------------------------------------------------------------------------------- /essay/templates/init/MANIFEST.in.tpl: -------------------------------------------------------------------------------- 1 | recursive-include ${project_name} *.css *.js *.jpg *.gif *.png *.html *.md -------------------------------------------------------------------------------- /essay/templates/default/MANIFEST.in.tpl: -------------------------------------------------------------------------------- 1 | recursive-include ${project_name} *.css *.js *.jpg *.gif *.png *.html *.md -------------------------------------------------------------------------------- /essay/templates/django/MANIFEST.in.tpl: -------------------------------------------------------------------------------- 1 | recursive-include ${project_name} *.css *.js *.jpg *.gif *.png *.html *.md -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | exclude = .git,.tox,__pycache__,.eggs,build,sphinx,.venv 3 | max-line-length = 110 4 | ignore = 5 | -------------------------------------------------------------------------------- /essay/templates/default/__project__/requirements.txt: -------------------------------------------------------------------------------- 1 | -f http://pypi.yourserver.com 2 | Fabric==1.4.3 3 | essay>=2.0.0.0 4 | 5 | -------------------------------------------------------------------------------- /essay/templates/django/__project__/requirements.txt: -------------------------------------------------------------------------------- 1 | -f http://pypi.yourserver.com/ 2 | Fabric==1.4.3 3 | essay>=2.0.0.0 4 | django 5 | 6 | -------------------------------------------------------------------------------- /essay/templates/default/.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *.log 3 | *.pyc 4 | .idea 5 | .project 6 | .pydevproject 7 | atlassian-ide-plugin.xml 8 | *.egg-info 9 | .env 10 | -------------------------------------------------------------------------------- /essay/templates/django/.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *.log 3 | *.pyc 4 | .idea 5 | .project 6 | .pydevproject 7 | atlassian-ide-plugin.xml 8 | *.egg-info 9 | .env 10 | -------------------------------------------------------------------------------- /essay/__init__.py: -------------------------------------------------------------------------------- 1 | try: 2 | from fabric.state import env 3 | env.PYPI_INDEX = 'http://pypi.doubanio.com/simple/' 4 | except ImportError: 5 | pass 6 | 7 | 8 | VERSION = '0.0.8' 9 | -------------------------------------------------------------------------------- /essay/settings.py: -------------------------------------------------------------------------------- 1 | # coding:utf-8 2 | 3 | __version__ = '${version}' 4 | __git_version__ = '${git_version}' 5 | __release_time__ = '${release_time}' 6 | 7 | import os 8 | from log import init_log 9 | PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__)) 10 | 11 | init_log() 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | 3 | # C extensions 4 | *.so 5 | 6 | # Packages 7 | *.egg 8 | *.egg-info 9 | dist 10 | build 11 | eggs 12 | parts 13 | bin 14 | var 15 | sdist 16 | develop-eggs 17 | .installed.cfg 18 | lib 19 | lib64 20 | __pycache__ 21 | 22 | # Installer logs 23 | pip-log.txt 24 | -------------------------------------------------------------------------------- /essay/pip_.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | import os 3 | from fabric.state import env 4 | 5 | 6 | def install(args): 7 | cmd = 'pip install -i=%s %s' % (env.PYPI_INDEX, ' '.join(args)) 8 | os.system(cmd) 9 | 10 | 11 | def default(args): 12 | cmd = 'pip ' + ' '.join(args) 13 | os.system(cmd) 14 | -------------------------------------------------------------------------------- /essay/templates/default/__project__/settings.py.tpl: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | 3 | __version__ = '${version}' 4 | __git_version__ = '${git_version}' 5 | __release_time__ = '${release_time}' 6 | 7 | #import os 8 | #PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__)) 9 | 10 | #from .log import init_log 11 | #init_log() -------------------------------------------------------------------------------- /release.md: -------------------------------------------------------------------------------- 1 | ## 版本记录 2 | 3 | * 0.0.8 4 | - 修复0.0.7必须配置deploy hook方可用的BUG 5 | - 处理不合PEP8规范的代码 6 | * 0.0.7 7 | - 去掉遗留的print用法； 8 | - 增加deploy函数前后hooks； 9 | * 0.0.6 10 | - 增加打包wheel包的逻辑； 11 | - 修复pip版本大于10时判断bug； 12 | - 支持自定义虚拟环境初始化命令； 13 | * 0.0.4 14 | - 优化supervisor启动命令; 15 | - 增加启动失败重试机制； 16 | -------------------------------------------------------------------------------- /essay/templates/django/__project__/manage.py.tpl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "${project_name}.settings") 7 | 8 | from django.core.management import execute_from_command_line 9 | 10 | execute_from_command_line(sys.argv) 11 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | sudo: false 3 | 4 | python: 5 | - 2.7 6 | - 3.5 7 | - 3.6 8 | 9 | install: 10 | - travis_retry pip install flake8 11 | - travis_retry pip install virtualenv 12 | - virtualenv ./.venv 13 | - ./.venv/bin/python -VV 14 | - ./.venv/bin/python setup.py install 15 | - ./.venv/bin/pip freeze 16 | 17 | script: 18 | - flake8 19 | -------------------------------------------------------------------------------- /essay/templates/django/__project__/__project__/urls.py.tpl: -------------------------------------------------------------------------------- 1 | from django.conf.urls import patterns, include, url 2 | 3 | from django.contrib import admin 4 | admin.autodiscover() 5 | 6 | urlpatterns = patterns('', 7 | # Examples: 8 | # url(r'^$', '${project_name}.views.home', name='home'), 9 | # url(r'^blog/', include('blog.urls')), 10 | 11 | url(r'^admin/', include(admin.site.urls)), 12 | ) 13 | -------------------------------------------------------------------------------- /essay/templates/django/__project__/__project__/wsgi.py.tpl: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for mypro project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "${project_name}.${project_name}.settings") 12 | 13 | from django.core.wsgi import get_wsgi_application 14 | application = get_wsgi_application() 15 | -------------------------------------------------------------------------------- /sphinx/source/index.rst: -------------------------------------------------------------------------------- 1 | .. essay documentation master file, created by 2 | sphinx-quickstart on Tue Sep 25 00:13:50 2012. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to essay's documentation! 7 | ================================= 8 | 9 | Contents: 10 | 11 | .. toctree:: 12 | :maxdepth: 2 13 | 14 | 15 | 16 | Indices and tables 17 | ================== 18 | 19 | * :ref:`genindex` 20 | * :ref:`modindex` 21 | * :ref:`search` 22 | 23 | -------------------------------------------------------------------------------- /essay/templates/default/__project__/main.py.tpl: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | 3 | import web 4 | import settings 5 | 6 | urls = ( 7 | '/', 'index' 8 | ) 9 | 10 | class index: 11 | def GET(self): 12 | return "Hello, world!" 13 | 14 | def main(): 15 | print('version:', settings.__version__) 16 | print('git version:', settings.__git_version__) 17 | print('release time', settings.__release_time__) 18 | 19 | app = web.application(urls, globals()) 20 | app.run() 21 | 22 | if __name__ == '__main__': 23 | main() 24 | -------------------------------------------------------------------------------- /fabfile/__init__.py: -------------------------------------------------------------------------------- 1 | # coding:utf-8 2 | 3 | import os 4 | 5 | from fabric.state import env 6 | 7 | from essay.tasks import build # noqa 8 | 9 | 10 | PROJECT_ROOT = os.path.dirname(os.path.dirname(__file__)) 11 | 12 | 13 | env.roledefs = { 14 | 'build': ['vagrant@127.0.0.1:2202'], 15 | } 16 | 17 | env.GIT_SERVER = 'https://github.com/' # ssh地址只需要填：github.com 18 | env.PROJECT = 'essay' 19 | env.BUILD_PATH = '~/buildspace' 20 | env.PROJECT_OWNER = 'SohuTech' 21 | env.DEFAULT_BRANCH = 'master' 22 | env.PYPI_INDEX = 'http://pypi.python.org/simple' 23 | -------------------------------------------------------------------------------- /essay/utils.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from __future__ import unicode_literals 3 | 4 | from importlib import import_module 5 | 6 | 7 | def import_by_path(dotted_path): 8 | """ 9 | 根据路径动态引入模块属性 10 | """ 11 | try: 12 | module_path, class_name = dotted_path.rsplit('.', 1) 13 | except ValueError: 14 | raise Exception('%s不是一个有效的模块路径' % module_path) 15 | 16 | try: 17 | module = import_module(module_path) 18 | except ImportError as e: 19 | raise Exception('模块引入错误: "%s"' % e) 20 | 21 | try: 22 | attr = getattr(module, class_name) 23 | except AttributeError as e: 24 | raise Exception('模块引入错误: "%s"' % e) 25 | 26 | return attr 27 | -------------------------------------------------------------------------------- /essay/templates/django/setup.py.tpl: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | #!/usr/bin/env python 3 | 4 | from setuptools import setup, find_packages 5 | 6 | readme = open('README').read() 7 | 8 | setup( 9 | name='${project_name}', 10 | version='${version}', 11 | description='', 12 | long_description=readme, 13 | author='', 14 | author_email='', 15 | url='http://www.sohu.com', 16 | packages=find_packages(exclude=['*.pyc']), 17 | include_package_data = True, 18 | package_data = { 19 | }, 20 | install_requires=[ 21 | 'django>1.3', 22 | 'gunicorn', 23 | ], 24 | entry_points={ 25 | 'console_scripts': [ 26 | '${project_name} = ${project_name}.main:main', 27 | ] 28 | }, 29 | ) 30 | -------------------------------------------------------------------------------- /essay/templates/default/setup.py.tpl: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | #!/usr/bin/env python 3 | 4 | from setuptools import setup, find_packages 5 | 6 | readme = open('README').read() 7 | 8 | setup( 9 | name='${project_name}', 10 | version='$${version}', 11 | description=readme.partition('\n')[0], 12 | long_description=readme, 13 | author='sohutech', 14 | author_email='youremail@example.com', 15 | url='http://www.sohu.com', 16 | packages=find_packages(exclude=['*.pyc']), 17 | include_package_data=True, 18 | package_data={ 19 | }, 20 | install_requires=[ 21 | "web.py", 22 | ], 23 | entry_points={ 24 | 'console_scripts': [ 25 | '${project_name} = ${project_name}.main:main', 26 | ] 27 | }, 28 | ) 29 | -------------------------------------------------------------------------------- /essay/tasks/pypi.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from __future__ import unicode_literals 3 | 4 | from fabric.api import run, env, task 5 | from fabric.context_managers import settings 6 | 7 | from essay.tasks import config 8 | 9 | __all__ = ['sync'] 10 | 11 | 12 | @task 13 | def sync(*packages): 14 | """从http://pypi.python.org同步包 15 | 16 | 用法: 17 | fab pypi.sync:django==1.3,tornado 18 | """ 19 | 20 | config.check('PYPI_HOST', 21 | 'PYPI_USER', 22 | 'PYPI_ROOT') 23 | 24 | with settings(host_string=env.PYPI_HOST, user=env.PYPI_USER): 25 | cmd = ["pip", "-q", "install", "--no-deps", "-i", "https://pypi.python.org/simple", 26 | "-d", env.PYPI_ROOT, 27 | ' '.join(packages)] 28 | 29 | run(" ".join(cmd)) 30 | -------------------------------------------------------------------------------- /essay/templates/init/setup.py.tpl: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | #!/usr/bin/env python 3 | 4 | from setuptools import setup, find_packages 5 | 6 | readme = open('README').read() 7 | 8 | setup( 9 | name='${project_name}', 10 | version='${version}', 11 | description='', 12 | long_description=readme, 13 | author='sohutech', 14 | author_email='youremail@example.com', 15 | url='http://m.sohu.com', 16 | packages=find_packages(exclude=['*.pyc']), 17 | include_package_data = True, 18 | package_data = { 19 | }, 20 | install_requires=[ 21 | 'essay>=2.9.1.1', 22 | 'django>=1.3', 23 | 'gunicorn', 24 | ], 25 | entry_points={ 26 | 'console_scripts': [ 27 | '${project_name} = ${project_name}.main:main', 28 | ] 29 | }, 30 | ) 31 | -------------------------------------------------------------------------------- /essay/tasks/process.py: -------------------------------------------------------------------------------- 1 | # coding:utf-8 2 | from __future__ import unicode_literals 3 | 4 | from fabric.api import run 5 | from fabric.context_managers import settings, hide 6 | from fabric.decorators import task 7 | 8 | 9 | @task 10 | def kill_by_name(name): 11 | """ 12 | 停止指定特征的进程 13 | """ 14 | 15 | with settings(warn_only=True): 16 | run("ps aux | grep '%s' | grep -v 'grep' | awk '{print $2}' | xargs kill -9" % name) 17 | 18 | 19 | @task 20 | def top(): 21 | """ 22 | 查看系统负载 23 | """ 24 | 25 | run("top -b | head -n 1") 26 | 27 | 28 | @task 29 | def ps_by_venv(venv_dir): 30 | """" 31 | 查看指定虚拟环境的进程CPU占用 32 | """ 33 | 34 | with hide('status', 'running', 'stderr'): 35 | run("""ps aux | grep -v grep | grep -v supervisor | grep %s | awk '{print $3, "|", $4}'""" % venv_dir) 36 | -------------------------------------------------------------------------------- /essay/templates/init/fabfile/__init__.py.tpl: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | 3 | import os 4 | 5 | PROJECT_ROOT = os.path.dirname(os.path.dirname(__file__)) 6 | 7 | from fabric.state import env 8 | 9 | from essay.tasks import build 10 | from essay.tasks import deploy 11 | #from essay.tasks import nginx 12 | #from essay.tasks import supervisor 13 | 14 | env.GIT_SERVER = 'https://github.com/' # ssh地址只需要填：'github.com' 15 | env.PROJECT = '${project_name}' 16 | #env.BUILD_PATH = '/opt/deploy/' 17 | #env.PROJECT_OWNER = 'EssayTech' 18 | #env.DEFAULT_BRANCH = 'master' 19 | #env.PYPI_INDEX = 'http://pypi.python.org/simple/' 20 | 21 | 22 | ###### 23 | # deploy settings: 24 | env.PROCESS_COUNT = 2 #部署时启动的进程数目 25 | env.roledefs = { 26 | 'build': ['username@buildserverip:port'], # 打包服务器配置 27 | 'dev': [''], 28 | } 29 | 30 | env.VIRTUALENV_PREFIX = '/home/django/${project_name}' 31 | env.SUPERVISOR_CONF_TEMPLATE = os.path.join(PROJECT_ROOT, 'conf', 'supervisord.conf') 32 | -------------------------------------------------------------------------------- /essay/templates/default/fabfile/__init__.py.tpl: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | 3 | import os 4 | 5 | PROJECT_ROOT = os.path.dirname(os.path.dirname(__file__)) 6 | 7 | from fabric.state import env 8 | 9 | from essay.tasks import build 10 | from essay.tasks import deploy 11 | #from essay.tasks import nginx 12 | #from essay.tasks import supervisor 13 | 14 | env.GIT_SERVER = 'https://github.com/' # ssh地址只需要填：github.com 15 | env.PROJECT = '${project_name}' 16 | #env.BUILD_PATH = '/opt/deploy/' 17 | #env.PROJECT_OWNER = 'EssayTech' 18 | #env.DEFAULT_BRANCH = 'master' 19 | #env.PYPI_INDEX = 'http://pypi.python.org/simple/' 20 | 21 | 22 | ###### 23 | # deploy settings: 24 | env.PROCESS_COUNT = 2 #部署时启动的进程数目 25 | env.roledefs = { 26 | 'build': ['username@buildserverip:port'], # 打包服务器配置 27 | 'dev': [''], 28 | } 29 | 30 | env.VIRTUALENV_PREFIX = '/home/username/${project_name}' 31 | env.SUPERVISOR_CONF_TEMPLATE = os.path.join(PROJECT_ROOT, 'conf', 'supervisord.conf') 32 | -------------------------------------------------------------------------------- /sphinx/source/essay.rst: -------------------------------------------------------------------------------- 1 | essay Package 2 | ============= 3 | 4 | :mod:`log` Module 5 | ----------------- 6 | 7 | .. automodule:: essay.log 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | 12 | :mod:`main` Module 13 | ------------------ 14 | 15 | .. automodule:: essay.main 16 | :members: 17 | :undoc-members: 18 | :show-inheritance: 19 | 20 | :mod:`pip` Module 21 | ----------------- 22 | 23 | .. automodule:: essay.pip 24 | :members: 25 | :undoc-members: 26 | :show-inheritance: 27 | 28 | :mod:`project` Module 29 | --------------------- 30 | 31 | .. automodule:: essay.project 32 | :members: 33 | :undoc-members: 34 | :show-inheritance: 35 | 36 | :mod:`settings` Module 37 | ---------------------- 38 | 39 | .. automodule:: essay.settings 40 | :members: 41 | :undoc-members: 42 | :show-inheritance: 43 | 44 | Subpackages 45 | ----------- 46 | 47 | .. toctree:: 48 | 49 | essay.tasks 50 | 51 | -------------------------------------------------------------------------------- /essay/tasks/config.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from __future__ import unicode_literals, print_function 3 | from os import path 4 | 5 | from fabric.api import task 6 | from fabric.state import env 7 | from fabric.contrib import files 8 | 9 | 10 | def check(*properties): 11 | def _check(_property): 12 | if not hasattr(env, _property): 13 | msg = 'env has not attribute [{}]'.format(_property) 14 | print(msg) 15 | raise Exception(msg) 16 | 17 | for property in properties: 18 | _check(property) 19 | 20 | 21 | @task(default=True) 22 | def upload_conf(**context): 23 | if hasattr(env, 'LOCAL_SERVER_CONF'): 24 | for local_conf, server_conf in env.LOCAL_SERVER_CONF: 25 | template_dir, filename = path.dirname(local_conf), path.basename(local_conf) 26 | venv_dir = env.CURRENT_VIRTUAL_ENV_DIR 27 | destination = path.join(venv_dir, server_conf) 28 | 29 | files.upload_template( 30 | filename, destination, context=context, 31 | use_jinja=True, template_dir=template_dir 32 | ) 33 | else: 34 | print('no local conf to upload') 35 | -------------------------------------------------------------------------------- /essay/templates/django/fabfile/__init__.py.tpl: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | 3 | import os 4 | 5 | PROJECT_ROOT = os.path.dirname(os.path.dirname(__file__)) 6 | 7 | from fabric.state import env 8 | 9 | from essay.tasks import build 10 | from essay.tasks import deploy 11 | #from essay.tasks import nginx 12 | #from essay.tasks import supervisor 13 | 14 | env.GIT_SERVER = 'https://github.com/' # ssh地址只需要填：github.com 15 | 16 | env.PROJECT = '${project_name}' 17 | #env.BUILD_PATH = '/opt/deploy/' 18 | #env.PROJECT_OWNER = 'EssayTech' 19 | #env.DEFAULT_BRANCH = 'master' 20 | #env.PYPI_INDEX = 'http://pypi.python.org/simple/' 21 | 22 | 23 | ###### 24 | # deploy settings: 25 | env.PROCESS_COUNT = 2 #部署时启动的进程数目 26 | env.roledefs = { 27 | 'build': ['username@buildserverip:port'], # 打包服务器配置 28 | 'dev': [''], 29 | } 30 | 31 | env.VIRTUALENV_PREFIX = '/home/django/${project_name}' 32 | env.SUPERVISOR_CONF_TEMPLATE = os.path.join(PROJECT_ROOT, 'conf', 'supervisord.conf') 33 | 34 | #根据工程确定项目编号, 不同环境保证监听不同的端口，通过port参数传到supervisord.conf中。 35 | PROJECT_NUM = 88 36 | env.VENV_PORT_PREFIX_MAP = { 37 | 'a': '%d0' % PROJECT_NUM, 38 | 'b': '%d1' % PROJECT_NUM, 39 | 'c': '%d2' % PROJECT_NUM, 40 | 'd': '%d3' % PROJECT_NUM, 41 | 'e': '%d4' % PROJECT_NUM, 42 | 'f': '%d5' % PROJECT_NUM, 43 | 'g': '%d6' % PROJECT_NUM, 44 | 'h': '%d7' % PROJECT_NUM, 45 | 'i': '%d8' % PROJECT_NUM, 46 | } 47 | -------------------------------------------------------------------------------- /essay/tasks/util.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | from __future__ import unicode_literals, print_function 4 | 5 | import random 6 | import string 7 | 8 | from fabric.decorators import task 9 | 10 | try: 11 | from string import lowercase 12 | except ImportError: 13 | from string import ascii_lowercase as lowercase 14 | 15 | try: 16 | from string import uppercase 17 | except ImportError: 18 | from string import ascii_uppercase as uppercase 19 | 20 | __all__ = ['random_str'] 21 | 22 | KEYS = [ 23 | lowercase, 24 | uppercase, 25 | string.digits, 26 | string.punctuation, 27 | ] 28 | 29 | 30 | @task 31 | def random_str(length=10, level=1): 32 | """ 33 | 生成随机字符串 34 | 35 | 参数: 36 | length: 字符串长度 37 | level: 使用的字符集 38 | 1 -> abcdefghijklmnopqrstuvwxyz 39 | 2 -> abcdefghijklmnopqrstuvwxyz + ABCDEFGHIJKLMNOPQRSTUVWXYZ 40 | 3 -> abcdefghijklmnopqrstuvwxyz + ABCDEFGHIJKLMNOPQRSTUVWXYZ + 0123456789 41 | 4 -> abcdefghijklmnopqrstuvwxyz + ABCDEFGHIJKLMNOPQRSTUVWXYZ + 0123456789 + !"#$%&'()*+,-./:;<=>?@[\]^_`{|}~ 42 | """ # noqa 43 | 44 | if length < 1 or 4 < level < 1: 45 | raise ValueError('无效参数') 46 | 47 | level = int(level) + 1 48 | keys = ''.join(KEYS[:level]) 49 | 50 | result = ''.join([random.choice(keys) for i in range(length)]) 51 | 52 | print(result) 53 | 54 | return result 55 | -------------------------------------------------------------------------------- /essay/tasks/fs.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from __future__ import unicode_literals 3 | import os 4 | from os import path 5 | 6 | from fabric.contrib import files 7 | from fabric.decorators import task 8 | from fabric.operations import run, local 9 | 10 | __all__ = ['rm_by_pattern'] 11 | 12 | KERNEL_NAME = os.uname()[0].lower() 13 | 14 | 15 | def ensure_dir(dir, in_local=False): 16 | """确保指定的dir被创建""" 17 | 18 | if in_local: 19 | if not path.isdir(dir): 20 | local("mkdir -p " + dir) 21 | elif not files.exists(dir): 22 | run("mkdir -p " + dir) 23 | 24 | 25 | def remove_dir(dir, in_local=False): 26 | """删除指定的文件夹""" 27 | 28 | if in_local: 29 | if not path.isdir(dir): 30 | local("rm -r " + dir) 31 | elif not files.exists(dir): 32 | run("rm -r " + dir) 33 | 34 | 35 | @task 36 | def rm_by_pattern(directory, pattern, in_local=False): 37 | """ 38 | 删除指定格式的文件 39 | 40 | 参数: 41 | directory: 目录 42 | pattern: 格式 43 | in_local: 在本地执行（默认） 44 | 45 | 示例: 46 | fab fs.rm_by_pattern:.,.pyc,True 47 | """ 48 | 49 | if in_local: 50 | local('find %s |grep %s | xargs rm -rf' % (directory, pattern)) 51 | else: 52 | run('find %s |grep %s | xargs rm -rf' % (directory, pattern)) 53 | 54 | 55 | def inplace_render(filename, params): 56 | for key, value in params.items(): 57 | files.sed(filename, '\$\{%s\}' % key, value) 58 | -------------------------------------------------------------------------------- /essay/templates/default/__project__/log.py.tpl: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from __future__ import unicode_literals 3 | 4 | from os import path 5 | import logging.config 6 | 7 | def init_log(log_dir=None): 8 | # if log_dir and not path.exists(log_dir): 9 | # msg = u'指定路径不存在:%s' % log_dir 10 | # print(msg.encode('utf-8')) 11 | # return 12 | 13 | config = { 14 | 'version': 1, 15 | 'disable_existing_loggers': False, 16 | 'formatters':{ 17 | 'default': { 18 | 'format': '%(levelname)s %(asctime)s %(module)s:%(funcName)s:%(lineno)d %(message)s' 19 | }, 20 | 'simple': { 21 | 'format': '%(levelname)s %(message)s' 22 | } 23 | }, 24 | 'handlers': { 25 | 'console': { 26 | 'level': 'DEBUG', 27 | 'class': 'logging.StreamHandler', 28 | 'formatter': 'default', 29 | }, 30 | # 'file': { 31 | # 'level': 'DEBUG', 32 | # 'class': 'logging.handlers.RotatingFileHandler', 33 | # 'filename': path.join(log_dir, 'essay.log'), 34 | # 'maxBytes': 1024 * 1024 * 50, 35 | # 'backupCount': 5, 36 | # 'formatter': 'default', 37 | # } 38 | }, 39 | 'loggers': { 40 | '':{ 41 | 'handlers': ['console'], 42 | 'level': 'INFO', 43 | 'propagate': True, 44 | }, 45 | } 46 | } 47 | 48 | logging.config.dictConfig(config) 49 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from __future__ import unicode_literals 3 | 4 | from setuptools import setup, find_packages 5 | 6 | from essay import VERSION 7 | 8 | with open('./README.md') as f: 9 | long_description = f.read() 10 | 11 | 12 | setup( 13 | name='essay', 14 | version=VERSION, 15 | description='持续部署工具', 16 | long_description=long_description, 17 | author='SohuTech', 18 | author_email='thefivefire@gmail.com', 19 | url='http://github.com/SohuTech/essay', 20 | packages=find_packages(exclude=['*.pyc', 'fabfile']), 21 | include_package_data=True, 22 | install_requires=[ 23 | 'Fabric3', 24 | 'Jinja2', 25 | 'requests', 26 | ], 27 | entry_points={ 28 | 'console_scripts': [ 29 | 'es = essay.main:main', 30 | 'ep = essay.main:pip_main', 31 | ] 32 | }, 33 | classifiers=[ 34 | 'Development Status :: 2 - Pre-Alpha', 35 | 'Environment :: Console', 36 | 'Intended Audience :: Developers', 37 | 'Intended Audience :: System Administrators', 38 | 'License :: OSI Approved :: BSD License', 39 | 'Operating System :: MacOS :: MacOS X', 40 | 'Operating System :: Unix', 41 | 'Operating System :: POSIX', 42 | 'Programming Language :: Python', 43 | 'Programming Language :: Python :: 2', 44 | 'Programming Language :: Python :: 2.7', 45 | 'Programming Language :: Python :: 3', 46 | 'Programming Language :: Python :: 3.5', 47 | 'Programming Language :: Python :: 3.6', 48 | ], 49 | ) 50 | -------------------------------------------------------------------------------- /essay/log.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from __future__ import unicode_literals 3 | 4 | import logging.config 5 | import sys 6 | from os import path 7 | 8 | 9 | def init_log(log_dir=None): 10 | if log_dir and not path.exists(log_dir): 11 | msg = '指定路径不存在:%s' % log_dir 12 | sys.stdout(msg) 13 | log_dir = None 14 | 15 | config = { 16 | 'version': 1, 17 | 'disable_existing_loggers': False, 18 | 'formatters': { 19 | 'default': { 20 | 'format': '%(levelname)s %(asctime)s %(module)s:%(funcName)s:%(lineno)d %(message)s' 21 | }, 22 | 'simple': { 23 | 'format': '%(level)s %(message)s' 24 | } 25 | }, 26 | 'handlers': { 27 | 'console': { 28 | 'level': 'DEBUG', 29 | 'class': 'logging.StreamHandler', 30 | 'formatter': 'default', 31 | }, 32 | }, 33 | 'loggers': { 34 | '': { 35 | 'handlers': ['console'], 36 | 'level': 'INFO', 37 | 'propagate': True, 38 | }, 39 | } 40 | } 41 | 42 | if log_dir: 43 | config['handles']['file'] = { 44 | 'level': 'DEBUG', 45 | 'class': 'logging.handlers.RotatingFileHandler', 46 | 'filename': path.join(log_dir, 'essay.log'), 47 | 'maxBytes': 1024 * 1024 * 50, 48 | 'backupCount': 5, 49 | 'formatter': 'default', 50 | } 51 | config['loggers']['handlers'] = ['console', 'file'] 52 | 53 | logging.config.dictConfig(config) 54 | -------------------------------------------------------------------------------- /essay/tasks/deploy.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from __future__ import unicode_literals 3 | 4 | from fabric.api import parallel, task 5 | from fabric.state import env 6 | 7 | from essay.tasks import virtualenv, supervisor, package, build 8 | 9 | __all__ = ['deploy', 'quickdeploy'] 10 | 11 | 12 | @task(default=True) 13 | @parallel(30) 14 | def deploy(version, venv_dir, profile): 15 | """ 发布指定的版本，会自动安装项目运行所需要的包 16 | 17 | version：build之后的版本 18 | venv_dir：虚拟环境名称 19 | profile：profile参数会传递到supervisord.conf中 20 | """ 21 | 22 | if not version: 23 | version = build.get_latest_version() 24 | 25 | virtualenv.ensure(venv_dir) 26 | 27 | pre_hook = getattr(env, 'DEPLOY_PRE_HOOK', None) 28 | post_hook = getattr(env, 'DEPLOY_POST_HOOK', None) 29 | 30 | with virtualenv.activate(venv_dir): 31 | if callable(pre_hook): 32 | pre_hook(version, venv_dir, profile) 33 | supervisor.ensure(project=env.PROJECT, profile=profile) 34 | package.install(env.PROJECT, version) 35 | supervisor.shutdown() 36 | supervisor.start() 37 | if callable(post_hook): 38 | post_hook(version, venv_dir, profile) 39 | 40 | 41 | @task(default=True) 42 | def quickdeploy(venv_dir, profile, branch=None): 43 | """ 44 | 快速部署 45 | 46 | $ fab -R yourroles quickdeploy:a,test,master 47 | """ 48 | 49 | deploy_host_string = env.host_string 50 | 51 | build_host = env.roledefs.get('build') 52 | env.host_string = build_host[0] if isinstance(build_host, list) else build_host 53 | build.build(branch=branch) 54 | 55 | env.host_string = deploy_host_string 56 | version = build.get_latest_version() 57 | 58 | deploy(version, venv_dir, profile) 59 | -------------------------------------------------------------------------------- /essay/tasks/validate.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from __future__ import unicode_literals, print_function 3 | 4 | from fabric.api import task 5 | from fabric.state import env 6 | from fabric.network import parse_host_string 7 | 8 | from essay.utils import import_by_path 9 | 10 | 11 | __all__ = ['validate'] 12 | 13 | 14 | class Validator(object): 15 | def __init__(self, venv, role, *args, **kwargs): 16 | self.venv = venv 17 | self.role = role 18 | self.args = args 19 | self.kwargs = kwargs 20 | 21 | def run(self): 22 | hosts = self.get_hosts() 23 | ports = self.get_ports() 24 | 25 | result = True 26 | for host in hosts: 27 | for port in ports: 28 | result = self.validate(host, port) 29 | if not result: 30 | print("验证失败: 主机[{}] 端口[{}]".format(host, port)) 31 | if result: 32 | print("验证成功") 33 | 34 | def validate(self, host, port): 35 | """ 36 | 验证服务可用性 37 | 38 | 子类重载此函数，并返回True或False 39 | """ 40 | raise NotImplementedError 41 | 42 | def get_hosts(self): 43 | return [ 44 | parse_host_string(host_string)['host'] 45 | for host_string in env.roledefs[self.role] 46 | ] 47 | 48 | def get_ports(self): 49 | port_prefix = env.VENV_PORT_PREFIX_MAP[self.venv] 50 | return [ 51 | int(str(port_prefix) + str(port_suffix)) 52 | for port_suffix in range(env.PROCESS_COUNT) 53 | ] 54 | 55 | 56 | @task 57 | def validate(venv, role='product', *args, **kwargs): 58 | """ 59 | 验证部署是否成功 60 | 61 | 参数: 62 | venv: 虚拟环境名称 63 | role: 服务器角色 64 | 65 | 用法: 66 | $ fab validate:a,product,[custom_args] 67 | 68 | """ 69 | validator_path = env.VALIDATOR_CLASS 70 | validator_class = import_by_path(validator_path) 71 | 72 | try: 73 | validator_instance = validator_class(venv, role, *args, **kwargs) 74 | except TypeError: 75 | raise Exception("{}不是一个有效的Validator".format(validator_path)) 76 | 77 | validator_instance.run() 78 | -------------------------------------------------------------------------------- /essay/main.py: -------------------------------------------------------------------------------- 1 | # coding:utf-8 2 | from __future__ import unicode_literals 3 | 4 | import re 5 | import sys 6 | import pip 7 | from optparse import OptionParser 8 | from fabric.state import env 9 | 10 | from essay.project import create_project, init_project 11 | 12 | usage = """es usage: es create/init [project_name] 13 | Commands available: 14 | create: create project with full structure 15 | pinstall: this command help you install package from our pypi server 16 | and other PIP support command 17 | """ 18 | 19 | 20 | def init_options(): 21 | parser = OptionParser(usage=usage) 22 | 23 | parser.add_option("-t", "--template", dest="template", default="default", 24 | help="project template:[default],[django]") 25 | 26 | return parser.parse_args() 27 | 28 | 29 | def main(): 30 | help_text = usage 31 | 32 | if len(sys.argv) == 2 and sys.argv[1] == 'init': 33 | init_project(None, 'init') 34 | elif len(sys.argv) >= 2 and sys.argv[1] == 'create': 35 | options, args = init_options() 36 | project_name = sys.argv[2] 37 | if re.match('^[a-zA-Z0-9_]+$', project_name): 38 | create_project(project_name, options.template) 39 | else: 40 | sys.stdout.write('无效工程名: ' + project_name) 41 | elif len(sys.argv) >= 2 and sys.argv[1] == 'init': 42 | options, args = init_options() 43 | project_name = sys.argv[2] 44 | if re.match('^[a-zA-Z0-9_]+$', project_name): 45 | init_project(project_name, options.template) 46 | else: 47 | sys.stdout.write('无效工程名: ' + project_name) 48 | elif len(sys.argv) >= 2 and sys.argv[1] == 'pinstall': 49 | if len(sys.argv) == 2 or sys.argv[2] == '-h': 50 | sys.stdout.write("es pinstall ") 51 | return 52 | 53 | args = sys.argv[1:] 54 | args[0] = 'install' 55 | args.append('-i %s' % env.PYPI_INDEX) 56 | pip.main(args) 57 | else: 58 | if len(sys.argv) == 2 and '-h' in sys.argv: 59 | sys.stdout.write(help_text) 60 | pip.main() 61 | 62 | 63 | if __name__ == '__main__': 64 | main() 65 | -------------------------------------------------------------------------------- /sphinx/source/essay.tasks.rst: -------------------------------------------------------------------------------- 1 | tasks Package 2 | ============= 3 | 4 | :mod:`build` Module 5 | ------------------- 6 | 7 | .. automodule:: essay.tasks.build 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | 12 | :mod:`cdn` Module 13 | ----------------- 14 | 15 | .. automodule:: essay.tasks.cdn 16 | :members: 17 | :undoc-members: 18 | :show-inheritance: 19 | 20 | :mod:`config` Module 21 | -------------------- 22 | 23 | .. automodule:: essay.tasks.config 24 | :members: 25 | :undoc-members: 26 | :show-inheritance: 27 | 28 | :mod:`deploy` Module 29 | -------------------- 30 | 31 | .. automodule:: essay.tasks.deploy 32 | :members: 33 | :undoc-members: 34 | :show-inheritance: 35 | 36 | :mod:`fs` Module 37 | ---------------- 38 | 39 | .. automodule:: essay.tasks.fs 40 | :members: 41 | :undoc-members: 42 | :show-inheritance: 43 | 44 | :mod:`git` Module 45 | ----------------- 46 | 47 | .. automodule:: essay.tasks.git 48 | :members: 49 | :undoc-members: 50 | :show-inheritance: 51 | 52 | :mod:`nginx` Module 53 | ------------------- 54 | 55 | .. automodule:: essay.tasks.nginx 56 | :members: 57 | :undoc-members: 58 | :show-inheritance: 59 | 60 | :mod:`package` Module 61 | --------------------- 62 | 63 | .. automodule:: essay.tasks.package 64 | :members: 65 | :undoc-members: 66 | :show-inheritance: 67 | 68 | :mod:`process` Module 69 | --------------------- 70 | 71 | .. automodule:: essay.tasks.process 72 | :members: 73 | :undoc-members: 74 | :show-inheritance: 75 | 76 | :mod:`supervisor` Module 77 | ------------------------ 78 | 79 | .. automodule:: essay.tasks.supervisor 80 | :members: 81 | :undoc-members: 82 | :show-inheritance: 83 | 84 | :mod:`util` Module 85 | ------------------ 86 | 87 | .. automodule:: essay.tasks.util 88 | :members: 89 | :undoc-members: 90 | :show-inheritance: 91 | 92 | :mod:`virtualenv` Module 93 | ------------------------ 94 | 95 | .. automodule:: essay.tasks.virtualenv 96 | :members: 97 | :undoc-members: 98 | :show-inheritance: 99 | 100 | -------------------------------------------------------------------------------- /essay/project.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | """ 4 | 创建新工程结构 5 | """ 6 | 7 | import logging 8 | import os 9 | import string 10 | from os import path 11 | 12 | from fabric.api import lcd 13 | from fabric.operations import prompt 14 | 15 | from essay import settings 16 | from essay.tasks import fs 17 | from essay.tasks import git 18 | 19 | logger = logging.getLogger(__name__) 20 | 21 | 22 | def create_project(project, template='default'): 23 | """创建本地工程""" 24 | init_project(project, template) 25 | with lcd(project): 26 | git.command('git init', in_local=True) 27 | git.add(add_all=True, in_local=True) 28 | git.commit(u'初始化工程结构', in_local=True) 29 | repos = prompt('请输入Git仓库地址:') 30 | if repos: 31 | git.command('git remote add origin %s' % repos, in_local=True) 32 | git.command('git push -u origin master', in_local=True) 33 | 34 | 35 | def init_project(project, template='default'): 36 | """初始化本地项目 37 | 38 | 此方法不需要连接git服务器 39 | """ 40 | if project is None: 41 | project_dir = path.abspath('.') 42 | template = 'init' 43 | project = '' 44 | params = { 45 | 'project_name': project 46 | } 47 | else: 48 | project_dir = path.abspath(project) 49 | fs.ensure_dir(project, in_local=True) 50 | 51 | params = { 52 | 'project_name': project 53 | } 54 | 55 | build_structure(project, project_dir, params, template) 56 | 57 | 58 | def build_structure(project, dst, params, template='default'): 59 | """ 60 | 拷贝工程打包及fab文件到工程 61 | """ 62 | dst = dst.rstrip('/') 63 | 64 | template_dir = path.join(settings.PROJECT_ROOT, 'templates', template) 65 | for root, dirs, files in os.walk(template_dir): 66 | for name in files: 67 | if name.endswith('.tpl'): 68 | src = path.join(root, name) 69 | dst_filename = src.replace(template_dir, dst).rstrip('.tpl').replace('__project__', project) 70 | dst_dir = os.path.dirname(dst_filename) 71 | 72 | fs.ensure_dir(dst_dir, in_local=True) 73 | 74 | content = open(src).read().decode('utf-8') 75 | if not name.endswith('.conf.tpl'): 76 | content = string.Template(content).safe_substitute(**params) 77 | 78 | open(dst_filename, 'w').write(content.encode('utf-8')) 79 | -------------------------------------------------------------------------------- /essay/tasks/nginx.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | from __future__ import unicode_literals 4 | 5 | from fabric.api import run, env, sudo 6 | from fabric.contrib import files 7 | from fabric.decorators import task 8 | 9 | from essay.tasks import config 10 | 11 | 12 | def _nginx_command(command, nginx_bin=None, nginx_conf=None, use_sudo=False): 13 | if not nginx_bin: 14 | config.check('NGINX_BIN') 15 | nginx_bin = env.NGINX_BIN 16 | 17 | if not nginx_conf: 18 | config.check('NGINX_CONF') 19 | nginx_conf = env.NGINX_CONF 20 | 21 | if command == 'start': 22 | cmd = '%(nginx_bin)s -c %(nginx_conf)s' % locals() 23 | else: 24 | cmd = '%(nginx_bin)s -c %(nginx_conf)s -s %(command)s' % locals() 25 | 26 | if use_sudo: 27 | sudo(cmd) 28 | else: 29 | run(cmd) 30 | 31 | 32 | @task 33 | def stop(nginx_bin=None, nginx_conf=None, use_sudo=False): 34 | """ 35 | 停止Nginx 36 | 37 | 参数: 38 | nginx_bin: nginx可执行文件路径，如果为提供则从env获取。 39 | nginx_conf: nginx配置文件路径，如果为提供则从env获取。 40 | """ 41 | 42 | _nginx_command('stop', nginx_bin, nginx_conf, use_sudo=use_sudo) 43 | 44 | 45 | @task 46 | def start(nginx_bin=None, nginx_conf=None, use_sudo=False): 47 | """ 48 | 启动Nginx 49 | 50 | 参数: 51 | nginx_bin: nginx可执行文件路径，如果为提供则从env获取。 52 | nginx_conf: nginx配置文件路径，如果为提供则从env获取。 53 | """ 54 | 55 | _nginx_command('start', nginx_bin, nginx_conf, use_sudo=use_sudo) 56 | 57 | 58 | @task 59 | def reload(nginx_bin=None, nginx_conf=None, use_sudo=False): 60 | """ 61 | 重启Nginx 62 | 63 | 参数: 64 | nginx_bin: nginx可执行文件路径，如果为提供则从env获取。 65 | nginx_conf: nginx配置文件路径，如果为提供则从env获取。 66 | """ 67 | _nginx_command('reload', nginx_bin, nginx_conf, use_sudo=use_sudo) 68 | 69 | 70 | @task 71 | def switch(src_pattern, dst_pattern, root=None, nginx_bin=None, nginx_conf=None): 72 | """ 73 | 修改配置文件并重启：源文本,目标文本,[root]（使用root) 74 | 75 | 主要用于AB环境的切换，将配置文件中的src_pattern修改为dst_pattern，并重启。 76 | 77 | 参数: 78 | src_pattern: 源模式，如upstreamA 79 | src_pattern: 目标模式，如upstreamB 80 | nginx_bin: nginx可执行文件路径，如果为提供则从env获取。 81 | nginx_conf: nginx配置文件路径，如果为提供则从env获取。 82 | """ 83 | 84 | if not nginx_conf: 85 | config.check('NGINX_CONF') 86 | nginx_conf = env.NGINX_CONF 87 | 88 | use_sudo = (root == 'root') 89 | files.sed(nginx_conf, src_pattern, dst_pattern, use_sudo=use_sudo) 90 | reload(nginx_bin, nginx_conf, use_sudo=use_sudo) 91 | -------------------------------------------------------------------------------- /essay/templates/django/__project__/__project__/settings.py.tpl: -------------------------------------------------------------------------------- 1 | __version__ = '${version}' 2 | __git_version__ = '${git_version}' 3 | __release_time__ = '${release_time}' 4 | 5 | """ 6 | Django settings for mypro project. 7 | 8 | For more information on this file, see 9 | https://docs.djangoproject.com/en/1.6/topics/settings/ 10 | 11 | For the full list of settings and their values, see 12 | https://docs.djangoproject.com/en/1.6/ref/settings/ 13 | """ 14 | 15 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 16 | import os 17 | BASE_DIR = os.path.dirname(os.path.dirname(__file__)) 18 | 19 | 20 | # Quick-start development settings - unsuitable for production 21 | # See https://docs.djangoproject.com/en/1.6/howto/deployment/checklist/ 22 | 23 | # SECURITY WARNING: keep the secret key used in production secret! 24 | SECRET_KEY = 'q@ce1am69cq4sa$1b*4t&&&*ohfpr9f33w0qr$-dtres-#m$v-' 25 | 26 | # SECURITY WARNING: don't run with debug turned on in production! 27 | DEBUG = True 28 | 29 | TEMPLATE_DEBUG = True 30 | 31 | ALLOWED_HOSTS = [] 32 | 33 | 34 | # Application definition 35 | 36 | INSTALLED_APPS = ( 37 | 'django.contrib.admin', 38 | 'django.contrib.auth', 39 | 'django.contrib.contenttypes', 40 | 'django.contrib.sessions', 41 | 'django.contrib.messages', 42 | 'django.contrib.staticfiles', 43 | ) 44 | 45 | MIDDLEWARE_CLASSES = ( 46 | 'django.contrib.sessions.middleware.SessionMiddleware', 47 | 'django.middleware.common.CommonMiddleware', 48 | 'django.middleware.csrf.CsrfViewMiddleware', 49 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 50 | 'django.contrib.messages.middleware.MessageMiddleware', 51 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 52 | ) 53 | 54 | ROOT_URLCONF = '${project_name}.${project_name}.urls' 55 | 56 | WSGI_APPLICATION = '${project_name}.${project_name}.wsgi.application' 57 | 58 | 59 | # Database 60 | # https://docs.djangoproject.com/en/1.6/ref/settings/#databases 61 | 62 | DATABASES = { 63 | 'default': { 64 | 'ENGINE': 'django.db.backends.sqlite3', 65 | 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), 66 | } 67 | } 68 | 69 | # Internationalization 70 | # https://docs.djangoproject.com/en/1.6/topics/i18n/ 71 | 72 | LANGUAGE_CODE = 'en-us' 73 | 74 | TIME_ZONE = 'UTC' 75 | 76 | USE_I18N = True 77 | 78 | USE_L10N = True 79 | 80 | USE_TZ = True 81 | 82 | 83 | # Static files (CSS, JavaScript, Images) 84 | # https://docs.djangoproject.com/en/1.6/howto/static-files/ 85 | 86 | STATIC_URL = '/static/' 87 | -------------------------------------------------------------------------------- /essay/templates/init/conf/supervisord.conf.tpl: -------------------------------------------------------------------------------- 1 | [unix_http_server] 2 | file={{ run_root }}/tmp/supervisor.sock ; (the path to the socket file) 3 | chmod=0700 ; socket file mode (default 0700) 4 | ;chown=root:root ; socket file uid:gid owner 5 | ;username={{ username }} ; (default is no username (open server)) 6 | ;password={{ password }} ; (default is no password (open server)) 7 | 8 | [supervisord] 9 | logfile={{ run_root }}/logs/supervisord.log ; (main log file;default $CWD/supervisord.log) 10 | logfile_maxbytes=50MB ; (max main logfile bytes b4 rotation;default 50MB) 11 | logfile_backups=10 ; (num of main logfile rotation backups;default 10) 12 | loglevel=info ; (log level;default info; others: debug,warn,trace) 13 | pidfile={{ run_root }}/tmp/supervisord.pid ; (supervisord pidfile;default supervisord.pid) 14 | nodaemon=false ; (start in foreground if true;default false) 15 | minfds=1024 ; (min. avail startup file descriptors;default 1024) 16 | minprocs=200 ; (min. avail process descriptors;default 200) 17 | umask=022 ; (process file creation umask;default 022) 18 | identifier=supervisor ; (supervisord identifier, default is 'supervisor') 19 | directory={{ run_root }} ; (default is not to cd during start) 20 | nocleanup=false ; (don't clean up tempfiles at start;default false) 21 | childlogdir={{ run_root }}/tmp ; ('AUTO' child log dir, default $TEMP) 22 | strip_ansi=false ; (strip ansi escape codes in logs; def. false) 23 | ;user=chrism ; (default is current user, required if root) 24 | ;environment=KEY=value ; (key value pairs to add to environment) 25 | 26 | [rpcinterface:supervisor] 27 | supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface 28 | 29 | [supervisorctl] 30 | serverurl=unix://{{ run_root }}/tmp/supervisor.sock ; use a unix:// URL for a unix socket 31 | ;username={{ username }} ; should be same as http_username if set 32 | ;password={{ supervisor_password }} ; should be same as http_password if set 33 | prompt=supervisor ; cmd line prompt (default "supervisor") 34 | history_file={{ run_root }}/.sc_history ; use readline history if available 35 | 36 | [program:{{ project }}] 37 | command={{ venv_dir}}/bin/{{ project }} --profile={{ profile }} 38 | process_name=%(program_name)s_%(process_num)d 39 | umask=022 40 | startsecs=10 41 | stopwaitsecs=0 42 | redirect_stderr=true 43 | stdout_logfile={{ run_root }}/logs/process_%(process_num)02d.log 44 | numprocs={{ process_count }} 45 | numprocs_start=0 -------------------------------------------------------------------------------- /conf/supervisord.conf: -------------------------------------------------------------------------------- 1 | [unix_http_server] 2 | file={{ run_root }}/tmp/supervisor.sock ; (the path to the socket file) 3 | chmod=0700 ; socket file mode (default 0700) 4 | ;chown=root:root ; socket file uid:gid owner 5 | ;username={{ username }} ; (default is no username (open server)) 6 | ;password={{ password }} ; (default is no password (open server)) 7 | 8 | [supervisord] 9 | logfile={{ run_root }}/logs/supervisord.log ; (main log file;default $CWD/supervisord.log) 10 | logfile_maxbytes=50MB ; (max main logfile bytes b4 rotation;default 50MB) 11 | logfile_backups=10 ; (num of main logfile rotation backups;default 10) 12 | loglevel=info ; (log level;default info; others: debug,warn,trace) 13 | pidfile={{ run_root }}/tmp/supervisord.pid ; (supervisord pidfile;default supervisord.pid) 14 | nodaemon=false ; (start in foreground if true;default false) 15 | minfds=1024 ; (min. avail startup file descriptors;default 1024) 16 | minprocs=200 ; (min. avail process descriptors;default 200) 17 | umask=022 ; (process file creation umask;default 022) 18 | identifier=supervisor ; (supervisord identifier, default is 'supervisor') 19 | directory={{ run_root }} ; (default is not to cd during start) 20 | nocleanup=false ; (don't clean up tempfiles at start;default false) 21 | childlogdir={{ run_root }}/tmp ; ('AUTO' child log dir, default $TEMP) 22 | strip_ansi=false ; (strip ansi escape codes in logs; def. false) 23 | ;user=chrism ; (default is current user, required if root) 24 | ;environment=KEY=value ; (key value pairs to add to environment) 25 | 26 | [inet_http_server] 27 | port=127.0.0.1:51250 28 | 29 | [rpcinterface:supervisor] 30 | supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface 31 | 32 | [supervisorctl] 33 | serverurl=http://127.0.0.1:51250 ; use a unix:// URL for a unix socket 34 | ;username={{ username }} ; should be same as http_username if set 35 | ;password={{ supervisor_password }} ; should be same as http_password if set 36 | prompt=supervisor ; cmd line prompt (default "supervisor") 37 | history_file={{ run_root }}/.sc_history ; use readline history if available 38 | 39 | [program:{{ project }}] 40 | command={{ run_root }}/bin/{{ project }} --profile={{ profile }} 41 | process_name=%(program_name)s_%(process_num)d 42 | umask=022 43 | startsecs=10 44 | stopwaitsecs=0 45 | redirect_stderr=true 46 | stdout_logfile={{ run_root }}/logs/process_%(process_num)02d.log 47 | numprocs={{ process_count }} 48 | numprocs_start=0 49 | 50 | -------------------------------------------------------------------------------- /essay/templates/default/conf/supervisord.conf.tpl: -------------------------------------------------------------------------------- 1 | [unix_http_server] 2 | file={{ run_root }}/tmp/supervisor.sock ; (the path to the socket file) 3 | chmod=0700 ; socket file mode (default 0700) 4 | ;chown=root:root ; socket file uid:gid owner 5 | ;username={{ username }} ; (default is no username (open server)) 6 | ;password={{ password }} ; (default is no password (open server)) 7 | 8 | [supervisord] 9 | logfile={{ run_root }}/logs/supervisord.log ; (main log file;default $CWD/supervisord.log) 10 | logfile_maxbytes=50MB ; (max main logfile bytes b4 rotation;default 50MB) 11 | logfile_backups=10 ; (num of main logfile rotation backups;default 10) 12 | loglevel=info ; (log level;default info; others: debug,warn,trace) 13 | pidfile={{ run_root }}/tmp/supervisord.pid ; (supervisord pidfile;default supervisord.pid) 14 | nodaemon=false ; (start in foreground if true;default false) 15 | minfds=1024 ; (min. avail startup file descriptors;default 1024) 16 | minprocs=200 ; (min. avail process descriptors;default 200) 17 | umask=022 ; (process file creation umask;default 022) 18 | identifier=supervisor ; (supervisord identifier, default is 'supervisor') 19 | directory={{ run_root }} ; (default is not to cd during start) 20 | nocleanup=false ; (don't clean up tempfiles at start;default false) 21 | childlogdir={{ run_root }}/tmp ; ('AUTO' child log dir, default $TEMP) 22 | strip_ansi=false ; (strip ansi escape codes in logs; def. false) 23 | ;user=chrism ; (default is current user, required if root) 24 | ;environment=KEY=value ; (key value pairs to add to environment) 25 | 26 | [rpcinterface:supervisor] 27 | supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface 28 | 29 | [supervisorctl] 30 | serverurl=unix://{{ run_root }}/tmp/supervisor.sock ; use a unix:// URL for a unix socket 31 | ;username={{ username }} ; should be same as http_username if set 32 | ;password={{ supervisor_password }} ; should be same as http_password if set 33 | prompt=supervisor ; cmd line prompt (default "supervisor") 34 | history_file={{ run_root }}/.sc_history ; use readline history if available 35 | 36 | [program:{{ project }}] 37 | command={{ venv_dir}}/bin/{{ project }} {{ port }}%(process_num)d ;--profile={{ profile }} 38 | process_name=%(program_name)s_%(process_num)d 39 | umask=022 40 | startsecs=10 41 | stopwaitsecs=0 42 | redirect_stderr=true 43 | stdout_logfile={{ run_root }}/logs/process_%(process_num)02d.log 44 | numprocs={{ process_count }} 45 | numprocs_start=0 46 | -------------------------------------------------------------------------------- /essay/tasks/git.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from __future__ import unicode_literals, print_function 3 | 4 | import logging 5 | 6 | from fabric.state import env 7 | from fabric.context_managers import lcd, cd 8 | from fabric.operations import local, run 9 | 10 | logger = logging.getLogger(__name__) 11 | 12 | 13 | def command(cmd, in_local=False, git_path=None): 14 | print(cmd, '###') 15 | 16 | if in_local: 17 | if git_path: 18 | with lcd(git_path): 19 | return local(cmd) 20 | else: 21 | return local(cmd) 22 | else: 23 | if git_path: 24 | with cd(git_path): 25 | return run(cmd) 26 | else: 27 | return run(cmd) 28 | 29 | 30 | def clone(project_name, in_local=False, git_path=None): 31 | """ 32 | 把项目clone到本地 33 | """ 34 | 35 | if env.GIT_SERVER.startswith('http'): 36 | cmd = 'git clone %s/%s' % (env.GIT_SERVER, project_name) 37 | else: 38 | cmd = 'git clone git@%s:%s' % (env.GIT_SERVER, project_name) 39 | command(cmd, in_local, git_path) 40 | 41 | 42 | def reset(in_local=False, git_path=None): 43 | """ 44 | 把项目lone到本地 45 | """ 46 | cmd = 'git reset --hard' 47 | command(cmd, in_local, git_path) 48 | 49 | 50 | def push(branch=None, in_local=False, git_path=None): 51 | """ 52 | 把项目lone到本地 53 | """ 54 | cmd = 'git push' 55 | if branch: 56 | cmd += ' origin ' + branch 57 | command(cmd, in_local, git_path) 58 | 59 | 60 | def pull(in_local=False, git_path=None): 61 | """ 62 | 把项目lone到本地 63 | """ 64 | cmd = 'git pull' 65 | 66 | command(cmd, in_local, git_path) 67 | 68 | 69 | def add(files=None, add_all=False, in_local=False, git_path=None): 70 | if not files and not add_all: 71 | raise Exception('无效参数') 72 | 73 | if add_all: 74 | cmd = 'git add .' 75 | else: 76 | if not isinstance(files, (tuple, list)): 77 | files = [files] 78 | cmd = 'git add ' + ' '.join(files) 79 | command(cmd, in_local, git_path) 80 | 81 | 82 | def commit(msg, in_local=False, git_path=None): 83 | """ 84 | 把项目lone到本地 85 | """ 86 | cmd = 'git commit -a -m "%s"' % msg 87 | command(cmd, in_local, git_path) 88 | 89 | 90 | def checkout(commit_or_branch, in_local=False, git_path=None): 91 | """ 92 | 根据commit回滚代码或者获取分支的所有代码 93 | 94 | commit据有优先权 95 | """ 96 | 97 | cmd = 'git reset --hard && git fetch && git checkout %s &&\ 98 | git pull && git submodule update --init --recursive' % commit_or_branch 99 | command(cmd, in_local, git_path) 100 | 101 | 102 | def get_version(in_local=False, git_path=None): 103 | cmd = "git rev-parse HEAD" 104 | return command(cmd, in_local, git_path) 105 | -------------------------------------------------------------------------------- /essay/templates/django/conf/supervisord.conf.tpl: -------------------------------------------------------------------------------- 1 | [unix_http_server] 2 | file={{ run_root }}/tmp/supervisor.sock ; (the path to the socket file) 3 | chmod=0700 ; socket file mode (default 0700) 4 | ;chown=root:root ; socket file uid:gid owner 5 | ;username={{ username }} ; (default is no username (open server)) 6 | ;password={{ password }} ; (default is no password (open server)) 7 | 8 | [supervisord] 9 | logfile={{ run_root }}/logs/supervisord.log ; (main log file;default $CWD/supervisord.log) 10 | logfile_maxbytes=50MB ; (max main logfile bytes b4 rotation;default 50MB) 11 | logfile_backups=10 ; (num of main logfile rotation backups;default 10) 12 | loglevel=info ; (log level;default info; others: debug,warn,trace) 13 | pidfile={{ run_root }}/tmp/supervisord.pid ; (supervisord pidfile;default supervisord.pid) 14 | nodaemon=false ; (start in foreground if true;default false) 15 | minfds=1024 ; (min. avail startup file descriptors;default 1024) 16 | minprocs=200 ; (min. avail process descriptors;default 200) 17 | umask=022 ; (process file creation umask;default 022) 18 | identifier=supervisor ; (supervisord identifier, default is 'supervisor') 19 | directory={{ run_root }} ; (default is not to cd during start) 20 | nocleanup=false ; (don't clean up tempfiles at start;default false) 21 | childlogdir={{ run_root }}/tmp ; ('AUTO' child log dir, default $TEMP) 22 | strip_ansi=false ; (strip ansi escape codes in logs; def. false) 23 | ;user=chrism ; (default is current user, required if root) 24 | ;environment=KEY=value ; (key value pairs to add to environment) 25 | 26 | [rpcinterface:supervisor] 27 | supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface 28 | 29 | [supervisorctl] 30 | serverurl=unix://{{ run_root }}/tmp/supervisor.sock ; use a unix:// URL for a unix socket 31 | ;username={{ username }} ; should be same as http_username if set 32 | ;password={{ supervisor_password }} ; should be same as http_password if set 33 | prompt=supervisor ; cmd line prompt (default "supervisor") 34 | history_file={{ run_root }}/.sc_history ; use readline history if available 35 | 36 | [program:{{ project }}] 37 | command={{ run_root }}/bin/gunicorn {{ project }}.{{ project }}.wsgi:application 38 | --workers 8 39 | --timeout 30 40 | --log-level info 41 | --error-logfile '-' 42 | --bind 0.0.0.0:{{ port }}%(process_num)1d 43 | process_name=%(program_name)s_%(process_num)d 44 | umask=022 45 | startsecs=10 46 | stopwaitsecs=0 47 | redirect_stderr=true 48 | stdout_logfile={{ run_root }}/logs/process_%(process_num)02d.log 49 | numprocs={{ process_count }} 50 | numprocs_start=0 51 | environment={{ project }}={{ profile }}=VIRTUALENV_NAME={{ venv_dir }} ; (key value pairs to add to environment) 52 | -------------------------------------------------------------------------------- /essay/tasks/virtualenv.py: -------------------------------------------------------------------------------- 1 | # coding:utf-8 2 | from __future__ import unicode_literals 3 | 4 | import posixpath 5 | from contextlib import contextmanager 6 | from os import path 7 | 8 | from fabric.api import run, prompt 9 | from fabric.contrib.files import exists 10 | from fabric.context_managers import prefix 11 | from fabric.state import env 12 | 13 | from essay.tasks import process, package, fs 14 | 15 | __all__ = [] 16 | 17 | 18 | def ensure(venv_dir, sub_dirs=None, user_mode=True): 19 | """ 20 | 确保虚拟环境存在 21 | 22 | :: 23 | .. _virtual environment: http://www.virtualenv.org/ 24 | """ 25 | 26 | if not venv_dir.startswith('/'): 27 | if 'VIRTUALENV_PREFIX' in env: 28 | venv_dir = path.join(env.VIRTUALENV_PREFIX, venv_dir) 29 | else: 30 | user_home = run('USER_HOME=$(eval echo ~${SUDO_USER}) && echo ${USER_HOME}') 31 | venv_dir = path.join(user_home, 'w', venv_dir) 32 | 33 | if is_virtualenv(venv_dir): 34 | return 35 | 36 | if env.VIRTUALENV_BIN: 37 | virtualenv_bin = env.VIRTUALENV_BIN 38 | command = '%(virtualenv_bin)s "%(venv_dir)s"' % locals() 39 | else: 40 | if package.is_virtualenv_installed_in_system(): 41 | virtualenv_bin = 'virtualenv' 42 | else: 43 | virtualenv_bin = '~/.local/bin/virtualenv' 44 | 45 | command = '%(virtualenv_bin)s --quiet "%(venv_dir)s"' % locals() 46 | 47 | run(command) 48 | 49 | if not sub_dirs: 50 | sub_dirs = ['logs', 'etc', 'tmp'] 51 | 52 | if 'VIRTUALENV_SUB_DIRS' in env: 53 | sub_dirs = list(set(sub_dirs + env.VIRTUALENV_SUB_DIRS)) 54 | 55 | for sub_dir in sub_dirs: 56 | fs.ensure_dir(path.join(venv_dir, sub_dir)) 57 | 58 | 59 | @contextmanager 60 | def activate(venv_dir, local=False): 61 | """ 62 | 用来启用VirtualEnv的上下文管理器 63 | 64 | :: 65 | with virtualenv('/path/to/virtualenv'): 66 | run('python -V') 67 | 68 | .. _virtual environment: http://www.virtualenv.org/ 69 | """ 70 | 71 | if not venv_dir.startswith('/'): 72 | if 'VIRTUALENV_PREFIX' in env: 73 | venv_dir = path.join(env.VIRTUALENV_PREFIX, venv_dir) 74 | else: 75 | user_home = run('USER_HOME=$(eval echo ~${SUDO_USER}) && echo ${USER_HOME}') 76 | venv_dir = path.join(user_home, 'w', venv_dir) 77 | 78 | if not is_virtualenv(venv_dir): 79 | raise Exception('无效虚拟环境: %s' % venv_dir) 80 | 81 | join = path.join if local else posixpath.join 82 | with prefix('. "%s"' % join(venv_dir, 'bin', 'activate')): 83 | env.CURRENT_VIRTUAL_ENV_DIR = venv_dir 84 | yield 85 | # del env['CURRENT_VIRTUAL_ENV_DIR'] 86 | 87 | 88 | def is_virtualenv(venv_dir): 89 | """判断指定的虚拟环境是否正确""" 90 | return exists(path.join(venv_dir, 'bin', 'activate')) 91 | 92 | 93 | def remove(venv_dir): 94 | """删除指定的虚拟环境""" 95 | 96 | answer = prompt(u"确定删除虚拟环境:%s (y/n)?" % venv_dir) 97 | if answer.lower() in ['y', 'yes']: 98 | if is_virtualenv(venv_dir): 99 | process.kill_by_name(path.join(venv_dir, 'bin')) 100 | run('rm -rf ' + venv_dir) 101 | -------------------------------------------------------------------------------- /essay/tasks/package.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from __future__ import unicode_literals 3 | 4 | import pip 5 | try: 6 | from urlparse import urlparse 7 | except ImportError: 8 | from urllib.parse import urlparse 9 | 10 | from fabric.api import run, env 11 | from fabric.context_managers import settings 12 | from fabric.contrib.files import exists 13 | from fabric.decorators import task 14 | 15 | __all__ = ['install'] 16 | 17 | PIP_VERSION = int(pip.__version__.split('.')[0]) 18 | 19 | 20 | def is_virtualenv_installed_in_system(): 21 | """ 22 | 检查virtualenv是否在系统目录安装 23 | """ 24 | 25 | with settings(warn_only=True): 26 | return 'no virtualenv' not in run('which virtualenv') or \ 27 | 'which' not in run('which virtualenv') 28 | 29 | 30 | def is_virtualenv_installed_in_user(): 31 | """ 32 | 检查virtualenv是否在系统目录安装 33 | """ 34 | 35 | with settings(warn_only=True): 36 | return exists('~/.local/bin/virtualenv') 37 | 38 | 39 | def is_virtualenv_installed(): 40 | """ 41 | 检查virtualenv是否已安装 42 | """ 43 | 44 | return is_virtualenv_installed_in_system() or is_virtualenv_installed_in_user() 45 | 46 | 47 | def is_installed(package): 48 | """检查Python包是否被安装 49 | 50 | 注意：只能在虚拟Python环境中执行 51 | """ 52 | 53 | if 'CURRENT_VIRTUAL_ENV_DIR' not in env: 54 | raise Exception('只可以在虚拟环境安装Python包') 55 | venv_dir = env.CURRENT_VIRTUAL_ENV_DIR 56 | 57 | with settings(warn_only=True): 58 | res = run('%(venv_dir)s/bin/pip freeze' % locals()) 59 | packages = [line.split('==')[0].lower() for line in res.splitlines()] 60 | 61 | return package.lower() in packages 62 | 63 | 64 | @task 65 | def install(package_name, version=None, private=True, user_mode=True): 66 | """ 67 | 用Pip安装Python包 68 | 69 | 参数: 70 | package: 包名，可以指定版本，如Fabric==1.4.3 71 | private: 利用私有PYPI安装 72 | user_mode: 安装在用户目录 73 | 74 | 注意：只能在虚拟Python环境中执行 75 | """ 76 | 77 | if 'CURRENT_VIRTUAL_ENV_DIR' not in env: 78 | raise Exception('只可以在虚拟环境安装Python包') 79 | 80 | venv_dir = env.CURRENT_VIRTUAL_ENV_DIR 81 | 82 | options = [] 83 | 84 | if hasattr(env, 'HTTP_PROXY'): 85 | options.append('--proxy {}'.format(env.HTTP_PROXY)) 86 | 87 | if private: 88 | options.append('-i {}'.format(env.PYPI_INDEX)) 89 | 90 | if PIP_VERSION >= 7: 91 | host = urlparse(env.PYPI_INDEX).netloc.split(':')[0] 92 | options.append('--trusted-host {}'.format(host)) 93 | 94 | options_str = ' '.join(options) 95 | if version: 96 | package_name += '==' + version 97 | 98 | command = '%(venv_dir)s/bin/pip install %(options_str)s %(package_name)s' % locals() 99 | 100 | run(command) 101 | 102 | 103 | def ensure(package, private=True, user_mode=True): 104 | """检查Python包有没有被安装，如果没有则安装 105 | 106 | 注意：只能在虚拟Python环境中执行 107 | """ 108 | 109 | if not is_installed(package): 110 | install(package, private=private, user_mode=user_mode) 111 | 112 | 113 | def uninstall(package): 114 | """卸载Python包 115 | 116 | 注意：只能在虚拟Python环境中执行 117 | """ 118 | 119 | if 'CURRENT_VIRTUAL_ENV_DIR' not in env: 120 | raise Exception('只可以在虚拟环境安装Python包') 121 | 122 | venv_dir = env.CURRENT_VIRTUAL_ENV_DIR 123 | run("%(venv_dir)s/bin/pip uninstall -y %(package)s" % locals()) 124 | -------------------------------------------------------------------------------- /essay/tasks/supervisor.py: -------------------------------------------------------------------------------- 1 | # coding:utf-8 2 | from __future__ import unicode_literals 3 | 4 | from os import path 5 | 6 | from fabric.api import run, settings 7 | from fabric.colors import green, red 8 | from fabric.context_managers import cd 9 | from fabric.contrib import files 10 | from fabric.decorators import task 11 | from fabric.state import env 12 | 13 | from essay.tasks import config, util, virtualenv, package 14 | 15 | __all__ = ['start_process', 'stop_process', 'restart_process', 'reload'] 16 | 17 | 18 | def ensure(**context): 19 | if 'CURRENT_VIRTUAL_ENV_DIR' not in env: 20 | raise Exception('只可以在虚拟环境安装Python包') 21 | venv_dir = env.CURRENT_VIRTUAL_ENV_DIR 22 | 23 | package.ensure('supervisor') 24 | 25 | context.setdefault('run_root', venv_dir) 26 | context.setdefault('username', util.random_str(10)) 27 | context.setdefault('password', util.random_str(20, True)) 28 | context.setdefault('process_count', 2) 29 | context.setdefault('venv_dir', venv_dir) 30 | context.setdefault('virtualenv_name', venv_dir[-1:]) 31 | if 'VENV_PORT_PREFIX_MAP' in env and isinstance(env.VENV_PORT_PREFIX_MAP, dict): 32 | try: 33 | context.setdefault('port', env.VENV_PORT_PREFIX_MAP[venv_dir[-1:]]) 34 | except KeyError: 35 | raise Exception('你的端口配置VENV_DIR_PORT_MAP中key[%s]不存在!' % venv_dir[-1:]) 36 | if 'PROCESS_COUNT' in env: 37 | context['process_count'] = env.PROCESS_COUNT 38 | config.check('SUPERVISOR_CONF_TEMPLATE') 39 | config_template = env.SUPERVISOR_CONF_TEMPLATE 40 | destination = path.join(venv_dir, 'etc', 'supervisord.conf') 41 | 42 | template_dir, filename = path.dirname(config_template), path.basename(config_template) 43 | 44 | files.upload_template(filename, destination, context=context, use_jinja=True, template_dir=template_dir) 45 | 46 | 47 | def _supervisor_command(command, venv_dir=None): 48 | if venv_dir: 49 | with virtualenv.activate(venv_dir): 50 | _supervisor_command(command) 51 | 52 | if 'CURRENT_VIRTUAL_ENV_DIR' not in env: 53 | raise Exception('只可以在虚拟环境安装Python包') 54 | 55 | venv_dir = env.CURRENT_VIRTUAL_ENV_DIR 56 | 57 | # 停止supervisor管理的进程 58 | with settings(warn_only=True), cd(venv_dir): 59 | run('bin/supervisorctl -c etc/supervisord.conf ' + command) 60 | 61 | 62 | @task 63 | def start(venv_dir=None, retry=0, retry_interval=2, max_retries=3): 64 | """ 65 | 重启指定虚拟环境的supervisor 66 | venv_dir 指定虚拟环境地址 67 | retry 当前重试次数 68 | retry_interval 多少秒后开始重试 69 | max_retries 最大重试次数 70 | """ 71 | 72 | if retry > max_retries: 73 | print(red('start supervisord FAIL!')) 74 | return 75 | 76 | if venv_dir: 77 | with virtualenv.activate(venv_dir): 78 | start() 79 | 80 | if 'CURRENT_VIRTUAL_ENV_DIR' not in env: 81 | raise Exception('只可以在虚拟环境安装Python包') 82 | 83 | venv_dir = env.CURRENT_VIRTUAL_ENV_DIR 84 | 85 | with settings(warn_only=True), cd(venv_dir): 86 | # 停止supervisor管理的进程 87 | result = run('bin/supervisord -c etc/supervisord.conf ') 88 | if result: 89 | retry += 1 90 | print(green('start supervisord fail, retry[{}]'.format(retry))) 91 | start(retry=retry, retry_interval=retry_interval, max_retries=max_retries) 92 | 93 | 94 | @task 95 | def shutdown(venv_dir=None): 96 | """重启指定虚拟环境的supervisor""" 97 | 98 | _supervisor_command('shutdown', venv_dir) 99 | 100 | 101 | @task 102 | def reload(venv_dir=None): 103 | """重启指定虚拟环境的supervisor""" 104 | 105 | _supervisor_command('reload', venv_dir) 106 | 107 | 108 | @task 109 | def start_process(name, venv_dir=None): 110 | """ 111 | 启动进程 112 | """ 113 | 114 | _supervisor_command(' start ' + name, venv_dir) 115 | 116 | 117 | @task 118 | def stop_process(name, venv_dir=None): 119 | """ 120 | 关闭进程 121 | """ 122 | 123 | _supervisor_command(' stop ' + name, venv_dir) 124 | 125 | 126 | @task 127 | def restart_process(name, venv_dir=None): 128 | """ 129 | 重启进程 130 | """ 131 | 132 | _supervisor_command(' restart ' + name, venv_dir) 133 | -------------------------------------------------------------------------------- /essay/tasks/build.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | from __future__ import unicode_literals, print_function 4 | 5 | import datetime 6 | import os 7 | import re 8 | import requests 9 | 10 | from fabric.api import cd, run, task, roles 11 | from fabric.state import env 12 | 13 | from essay.tasks import git, config, fs 14 | from fabric.contrib import files 15 | 16 | __all__ = ['build', 'get_latest_version', 'get_next_version'] 17 | 18 | 19 | PYPI_VERSION_RE = re.compile(r'(\d+(\.\d+)+)') 20 | A_MARKUP_RE = re.compile(r'' where is one of" 21 | @echo " html to make standalone HTML files" 22 | @echo " dirhtml to make HTML files named index.html in directories" 23 | @echo " singlehtml to make a single large HTML file" 24 | @echo " pickle to make pickle files" 25 | @echo " json to make JSON files" 26 | @echo " htmlhelp to make HTML files and a HTML help project" 27 | @echo " qthelp to make HTML files and a qthelp project" 28 | @echo " devhelp to make HTML files and a Devhelp project" 29 | @echo " epub to make an epub" 30 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 31 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 32 | @echo " text to make text files" 33 | @echo " man to make manual pages" 34 | @echo " texinfo to make Texinfo files" 35 | @echo " info to make Texinfo files and run them through makeinfo" 36 | @echo " gettext to make PO message catalogs" 37 | @echo " changes to make an overview of all changed/added/deprecated items" 38 | @echo " linkcheck to check all external links for integrity" 39 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 40 | 41 | clean: 42 | -rm -rf $(BUILDDIR)/* 43 | 44 | html: 45 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 46 | @echo 47 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 48 | 49 | dirhtml: 50 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 51 | @echo 52 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 53 | 54 | singlehtml: 55 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 56 | @echo 57 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 58 | 59 | pickle: 60 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 61 | @echo 62 | @echo "Build finished; now you can process the pickle files." 63 | 64 | json: 65 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 66 | @echo 67 | @echo "Build finished; now you can process the JSON files." 68 | 69 | htmlhelp: 70 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 71 | @echo 72 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 73 | ".hhp project file in $(BUILDDIR)/htmlhelp." 74 | 75 | qthelp: 76 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 77 | @echo 78 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 79 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 80 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/essay.qhcp" 81 | @echo "To view the help file:" 82 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/essay.qhc" 83 | 84 | devhelp: 85 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 86 | @echo 87 | @echo "Build finished." 88 | @echo "To view the help file:" 89 | @echo "# mkdir -p $$HOME/.local/share/devhelp/essay" 90 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/essay" 91 | @echo "# devhelp" 92 | 93 | epub: 94 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 95 | @echo 96 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 97 | 98 | latex: 99 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 100 | @echo 101 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 102 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 103 | "(use \`make latexpdf' here to do that automatically)." 104 | 105 | latexpdf: 106 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 107 | @echo "Running LaTeX files through pdflatex..." 108 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 109 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 110 | 111 | text: 112 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 113 | @echo 114 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 115 | 116 | man: 117 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 118 | @echo 119 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 120 | 121 | texinfo: 122 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 123 | @echo 124 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 125 | @echo "Run \`make' in that directory to run these through makeinfo" \ 126 | "(use \`make info' here to do that automatically)." 127 | 128 | info: 129 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 130 | @echo "Running Texinfo files through makeinfo..." 131 | make -C $(BUILDDIR)/texinfo info 132 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 133 | 134 | gettext: 135 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 136 | @echo 137 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 138 | 139 | changes: 140 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 141 | @echo 142 | @echo "The overview file is in $(BUILDDIR)/changes." 143 | 144 | linkcheck: 145 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 146 | @echo 147 | @echo "Link check complete; look for any errors in the above output " \ 148 | "or in $(BUILDDIR)/linkcheck/output.txt." 149 | 150 | doctest: 151 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 152 | @echo "Testing of doctests in the sources finished, look at the " \ 153 | "results in $(BUILDDIR)/doctest/output.txt." 154 | -------------------------------------------------------------------------------- /sphinx/source/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # essay documentation build configuration file, created by 4 | # sphinx-quickstart on Tue Sep 25 00:13:50 2012. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | import sys, os 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | sys.path.insert(0, os.path.abspath('../..')) 20 | 21 | # -- General configuration ----------------------------------------------------- 22 | 23 | # If your documentation needs a minimal Sphinx version, state it here. 24 | #needs_sphinx = '1.0' 25 | 26 | # Add any Sphinx extension module names here, as strings. They can be extensions 27 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 28 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode'] 29 | 30 | # Add any paths that contain templates here, relative to this directory. 31 | templates_path = ['_templates'] 32 | 33 | # The suffix of source filenames. 34 | source_suffix = '.rst' 35 | 36 | # The encoding of source files. 37 | #source_encoding = 'utf-8-sig' 38 | 39 | # The master toctree document. 40 | master_doc = 'index' 41 | 42 | # General information about the project. 43 | project = u'essay' 44 | copyright = u'2012, SOHU MPC' 45 | 46 | # The version info for the project you're documenting, acts as replacement for 47 | # |version| and |release|, also used in various other places throughout the 48 | # built documents. 49 | # 50 | # The short X.Y version. 51 | version = '2.0.0.0' 52 | # The full version, including alpha/beta/rc tags. 53 | release = '2.0.0.0' 54 | 55 | # The language for content autogenerated by Sphinx. Refer to documentation 56 | # for a list of supported languages. 57 | #language = None 58 | 59 | # There are two options for replacing |today|: either, you set today to some 60 | # non-false value, then it is used: 61 | #today = '' 62 | # Else, today_fmt is used as the format for a strftime call. 63 | #today_fmt = '%B %d, %Y' 64 | 65 | # List of patterns, relative to source directory, that match files and 66 | # directories to ignore when looking for source files. 67 | exclude_patterns = [] 68 | 69 | # The reST default role (used for this markup: `text`) to use for all documents. 70 | #default_role = None 71 | 72 | # If true, '()' will be appended to :func: etc. cross-reference text. 73 | #add_function_parentheses = True 74 | 75 | # If true, the current module name will be prepended to all description 76 | # unit titles (such as .. function::). 77 | #add_module_names = True 78 | 79 | # If true, sectionauthor and moduleauthor directives will be shown in the 80 | # output. They are ignored by default. 81 | #show_authors = False 82 | 83 | # The name of the Pygments (syntax highlighting) style to use. 84 | pygments_style = 'sphinx' 85 | 86 | # A list of ignored prefixes for module index sorting. 87 | #modindex_common_prefix = [] 88 | 89 | 90 | # -- Options for HTML output --------------------------------------------------- 91 | 92 | # The theme to use for HTML and HTML Help pages. See the documentation for 93 | # a list of builtin themes. 94 | html_theme = 'default' 95 | 96 | # Theme options are theme-specific and customize the look and feel of a theme 97 | # further. For a list of options available for each theme, see the 98 | # documentation. 99 | #html_theme_options = {} 100 | 101 | # Add any paths that contain custom themes here, relative to this directory. 102 | #html_theme_path = [] 103 | 104 | # The name for this set of Sphinx documents. If None, it defaults to 105 | # " v documentation". 106 | #html_title = None 107 | 108 | # A shorter title for the navigation bar. Default is the same as html_title. 109 | #html_short_title = None 110 | 111 | # The name of an image file (relative to this directory) to place at the top 112 | # of the sidebar. 113 | #html_logo = None 114 | 115 | # The name of an image file (within the static path) to use as favicon of the 116 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 117 | # pixels large. 118 | #html_favicon = None 119 | 120 | # Add any paths that contain custom static files (such as style sheets) here, 121 | # relative to this directory. They are copied after the builtin static files, 122 | # so a file named "default.css" will overwrite the builtin "default.css". 123 | html_static_path = ['_static'] 124 | 125 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 126 | # using the given strftime format. 127 | #html_last_updated_fmt = '%b %d, %Y' 128 | 129 | # If true, SmartyPants will be used to convert quotes and dashes to 130 | # typographically correct entities. 131 | #html_use_smartypants = True 132 | 133 | # Custom sidebar templates, maps document names to template names. 134 | #html_sidebars = {} 135 | 136 | # Additional templates that should be rendered to pages, maps page names to 137 | # template names. 138 | #html_additional_pages = {} 139 | 140 | # If false, no module index is generated. 141 | #html_domain_indices = True 142 | 143 | # If false, no index is generated. 144 | #html_use_index = True 145 | 146 | # If true, the index is split into individual pages for each letter. 147 | #html_split_index = False 148 | 149 | # If true, links to the reST sources are added to the pages. 150 | #html_show_sourcelink = True 151 | 152 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 153 | #html_show_sphinx = True 154 | 155 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 156 | #html_show_copyright = True 157 | 158 | # If true, an OpenSearch description file will be output, and all pages will 159 | # contain a tag referring to it. The value of this option must be the 160 | # base URL from which the finished HTML is served. 161 | #html_use_opensearch = '' 162 | 163 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 164 | #html_file_suffix = None 165 | 166 | # Output file base name for HTML help builder. 167 | htmlhelp_basename = 'essaydoc' 168 | 169 | 170 | # -- Options for LaTeX output -------------------------------------------------- 171 | 172 | latex_elements = { 173 | # The paper size ('letterpaper' or 'a4paper'). 174 | #'papersize': 'letterpaper', 175 | 176 | # The font size ('10pt', '11pt' or '12pt'). 177 | #'pointsize': '10pt', 178 | 179 | # Additional stuff for the LaTeX preamble. 180 | #'preamble': '', 181 | } 182 | 183 | # Grouping the document tree into LaTeX files. List of tuples 184 | # (source start file, target name, title, author, documentclass [howto/manual]). 185 | latex_documents = [ 186 | ('index', 'essay.tex', u'essay Documentation', 187 | u'SOHU MPC', 'manual'), 188 | ] 189 | 190 | # The name of an image file (relative to this directory) to place at the top of 191 | # the title page. 192 | #latex_logo = None 193 | 194 | # For "manual" documents, if this is true, then toplevel headings are parts, 195 | # not chapters. 196 | #latex_use_parts = False 197 | 198 | # If true, show page references after internal links. 199 | #latex_show_pagerefs = False 200 | 201 | # If true, show URL addresses after external links. 202 | #latex_show_urls = False 203 | 204 | # Documents to append as an appendix to all manuals. 205 | #latex_appendices = [] 206 | 207 | # If false, no module index is generated. 208 | #latex_domain_indices = True 209 | 210 | 211 | # -- Options for manual page output -------------------------------------------- 212 | 213 | # One entry per manual page. List of tuples 214 | # (source start file, name, description, authors, manual section). 215 | man_pages = [ 216 | ('index', 'essay', u'essay Documentation', 217 | [u'SOHU MPC'], 1) 218 | ] 219 | 220 | # If true, show URL addresses after external links. 221 | #man_show_urls = False 222 | 223 | 224 | # -- Options for Texinfo output ------------------------------------------------ 225 | 226 | # Grouping the document tree into Texinfo files. List of tuples 227 | # (source start file, target name, title, author, 228 | # dir menu entry, description, category) 229 | texinfo_documents = [ 230 | ('index', 'essay', u'essay Documentation', 231 | u'SOHU MPC', 'essay', 'One line description of project.', 232 | 'Miscellaneous'), 233 | ] 234 | 235 | # Documents to append as an appendix to all manuals. 236 | #texinfo_appendices = [] 237 | 238 | # If false, no module index is generated. 239 | #texinfo_domain_indices = True 240 | 241 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 242 | #texinfo_show_urls = 'footnote' 243 | 244 | 245 | # -- Options for Epub output --------------------------------------------------- 246 | 247 | # Bibliographic Dublin Core info. 248 | epub_title = u'essay' 249 | epub_author = u'SOHU MPC' 250 | epub_publisher = u'SOHU MPC' 251 | epub_copyright = u'2012, SOHU MPC' 252 | 253 | # The language of the text. It defaults to the language option 254 | # or en if the language is not set. 255 | #epub_language = '' 256 | 257 | # The scheme of the identifier. Typical schemes are ISBN or URL. 258 | #epub_scheme = '' 259 | 260 | # The unique identifier of the text. This can be a ISBN number 261 | # or the project homepage. 262 | #epub_identifier = '' 263 | 264 | # A unique identification for the text. 265 | #epub_uid = '' 266 | 267 | # A tuple containing the cover image and cover page html template filenames. 268 | #epub_cover = () 269 | 270 | # HTML files that should be inserted before the pages created by sphinx. 271 | # The format is a list of tuples containing the path and title. 272 | #epub_pre_files = [] 273 | 274 | # HTML files shat should be inserted after the pages created by sphinx. 275 | # The format is a list of tuples containing the path and title. 276 | #epub_post_files = [] 277 | 278 | # A list of files that should not be packed into the epub file. 279 | #epub_exclude_files = [] 280 | 281 | # The depth of the table of contents in toc.ncx. 282 | #epub_tocdepth = 3 283 | 284 | # Allow duplicate toc entries. 285 | #epub_tocdup = True 286 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 介绍： 2 | 3 | ------------------------------------ 4 | 5 | essay是我们团队在过去两年Python项目持续部署上经验的总结，核心功能是打包和多Server部署。这个工具的主要功能是简化从开发到部署的流程，方便服务器的扩展。除了打包和部署功能之外，essay还提供了其他的工具，在后面的接口描述中有详细介绍。 6 | 7 | essay是基于Fabric开发，在使用上依赖于pypi Server和github/Gitlab。 8 | 9 | 10 | 11 | # 快速开始 12 | 13 | ------------------------------------ 14 | 15 | *这里使用自己搭建的pypiserver作为pypi server，github作为代码仓库。* 16 | 17 | 1. 安装essay： ``pip install essay`` 18 | 2. github上创建项目：essay_demo_webpy 19 | 3. **创建项目**。终端上执行： ``es create essay_demo_webpy`` ，根据提示输入你新创建的git仓库地址，回车完毕你会发现你的github上已经有了初始化好的一个项目结构 20 | 21 | 22 | . 23 | └── essay_demo_webpy 24 | ├── MANIFEST.in 25 | ├── README.md 26 | ├── conf 27 | │   └── supervisord.conf 28 | ├── essay_demo 29 | │   ├── __init__.py 30 | │   ├── log.py 31 | │   ├── main.py 32 | │   └── settings.py 33 | ├── fabfile 34 | │   └── __init__.py 35 | └── setup.py 36 | 37 | 4 directories, 9 files 38 | 39 | 4. **打包项目**。此时你可以在本地的essay_demo_webpy目录下执行： ``fab -l`` 命令查看哪些命令可用。然后执行 ``fab build.build`` 命令，会帮你打包成tar的源码包，并添加版本后缀，如：essay_demo_webpy-3.11.20.1.tar.gz，之后会被上传到上面说到得pypiserver上。 40 | 41 | 关于build.build有四个可选参数,fab build.build:name=fixbug,version=0.1,commit=451a9a2784f,branch=master,也可以使用位置参数: ``fab build.build:fixbug,0.1,451a9a2784f,master`` 。其中commit和branch不能同时设置,commit优先于branch起作用。 42 | 43 | 各参数详解： 44 | 45 | name, 版本的后缀，如上例的结果是: essay_demo_webpy-0.1-fixbug.tar.gz 46 | varsion, 版本号，如果为空则会产生：essay_demo_webpy-3.11.20.1-fixbug.tar.gz。规则是：年.月.日.次 47 | commit, 是要打包的commit节点 48 | branch, 要打包的分支，默认为fabfile下__init__.py中的配置 49 | 50 | 5. **部署项目**。这时在pypiserver上已经有了打好的包，假设为: essay_demo_webpy-3.11.20.1.tar.gz 。然后开始安装，执行： ``fab -R dev deploy:3.11.20.1,a,test`` 把项目安装到fabfile里设置的dev这个role的服务器上，可以是一个或者多个server。 51 | 52 | *deploy参数解释：* 53 | 54 | deploy后面有三个参数，分别表示：版本号，虚拟环境目录名，profile（会传递到supervisord.conf中） 55 | 版本号在上面已经提到，不赘述。 56 | 虚拟环境目录名：依赖于fabfile中设置的 env.VIRTUALENV_PREFIX='~/essay_demo_webpy' ，这里参数为a，表示将在服务器家目录下得essay_demo_webpy目录下创建一个目录为 a 的virtualenv环境。 57 | profile：用来控制程序处于什么状态，比如可以传递debug进去，需要程序自己处理。参数会被传递到supervisord.conf中。 58 | 59 | > 0.0.7 新增： 60 | > 可以在fabfile中增加: 61 | 62 | def pre_deploy(version, venv, profile): 63 | do_something() 64 | env.DEPLOY_PRE_DEPLOY = pre_deploy 65 | 66 | def post_deploy(version, venv, profile): 67 | do_something() 68 | env.DEPLOY_POST_DEPLOY = post_deploy 69 | 70 | > 来处理部署前后的操作，两个hook均会在虚拟环境激活状态下执行 71 | 72 | deploy的内部的过程是：登录-R指定的服务器，在配置的VIRTUALENV_PREFIX目录下创建a虚拟环境，之后在此环境中通过pip安装已打包好的项目，最后通过supervisord来启动程序进程。 73 | 74 | 6. **快速部署** 针对需要直接部署某个分支或者commit的需求，该功能是基于上面的两个功能： ``build`` 和 ``deploy``。 75 | 76 | *使用方法:* 77 | fab -R deploy.quickdeploy:venv_dir,profile,branch 78 | 79 | 参数解释:: 80 | venv_dir: 虚拟环境名称 81 | profile： 运行环境配置，同上面 82 | branch： 要部署的分支 83 | 84 | *可以参考Demo：https://github.com/SohuTech/essay_demo_webpy 中的一些配置* 85 | 86 | 87 | # 配置文件详解 88 | 89 | ------------------------------------ 90 | 91 | 介绍主要的两个配置文件fabfile和supervisord 92 | 93 | ### fabfile配置 94 | 95 | ** myproject/fabfile/__init__.py ** 96 | 97 | 所有env配置项，可根据自身业务删除/添加 98 | 99 | 100 | 1. git服务器 101 | 102 | env.GIT_SERVER = 'https://github.com/' # ssh地址只需要填：github.com 103 | 104 | 2. 项目名（与git上项目名保持一致） 105 | 106 | env.PROJECT = 'project_name' 107 | 108 | 3. 项目在打包服务器上的路径，会在此目录下执行打包操作 109 | 110 | env.BUILD_PATH = '/opt/deploy/' 111 | 112 | 4. 项目所有者（与git保持一致） 113 | 114 | env.PROJECT_OWNER = 'EssayTech' 115 | 116 | 5. git默认分支 117 | 118 | env.DEFAULT_BRANCH = 'master' 119 | 120 | 6. pypi-server地址 121 | 122 | env.PYPI_INDEX = 'http://pypi.python.org/simple/' 123 | 124 | 部署相关： 125 | 126 | 7. 部署启动进程数目 127 | 128 | env.PROCESS_COUNT = 2 #部署时启动的进程数目, 会传递到supervisord.conf中 129 | 130 | 8. 服务器角色, 打包或部署时通过 ``-R`` 参数指定服务器角色, 每个角色可以定义多组服务器 131 | 132 | env.roledefs = { 133 | 'build': ['username@buildserverip:port'], # 打包服务器配置 134 | 'dev': ['eassay@127.0.0.1:2202'], #部署服务器配置 135 | } 136 | 137 | 9. 虚拟环境目录, 部署时会在服务器上此目录下创建虚拟环境 138 | 139 | env.VIRTUALENV_PREFIX = '/home/SohuTech/project_name' 140 | 141 | 12. supervisor配置文件地址, PROJECT_ROOT为项目根目录 142 | 143 | env.SUPERVISOR_CONF_TEMPLATE = os.path.join(PROJECT_ROOT, 'conf', 'supervisord.conf') 144 | 145 | 13. 根据工程确定项目编号, 不同虚拟环境监听不同的端口，用来配置一个同一机器多个虚拟环境。通过port参数传到supervisord.conf中。 146 | 147 | PROJECT_NUM = 88 148 | env.VENV_PORT_PREFIX_MAP = { 149 | 'a': '%d0' % PROJECT_NUM, 150 | 'b': '%d1' % PROJECT_NUM, 151 | 'c': '%d2' % PROJECT_NUM, 152 | 'd': '%d3' % PROJECT_NUM, 153 | 'e': '%d4' % PROJECT_NUM, 154 | 'f': '%d5' % PROJECT_NUM, 155 | 'g': '%d6' % PROJECT_NUM, 156 | 'h': '%d7' % PROJECT_NUM, 157 | 'i': '%d8' % PROJECT_NUM, 158 | } 159 | 160 | 161 | nginx配置用于启动、停止、重启、修改配置文件: 162 | 163 | 14. nginx执行文件地址 164 | 165 | env.NGINX_BIN = '/usr/local/nginx/bin/nginx' 166 | 167 | 15. nginx配置文件地址 168 | 169 | env.NGINX_CONF = '/usr/local/nginx/conf/nginx.conf' 170 | 171 | 16. 切换nginx环境(关于此项，请看下方essay高级功能用法) 172 | 173 | env.NGINX_SWITCH_CONF = '/etc/nginx/nginx.conf' 174 | 175 | ### supervisord配置 176 | 177 | ** myproject/conf/supervisor.conf ** 178 | 179 | **几个被替换的字段** 180 | 181 | 以下参数在fab deploy的时候会替换为fabfile里配置的字段 182 | 183 | 1. {{ run_root }} 虚拟环境地址,对应于fabfile/__init__.py里的env.VIRTUALENV_PREFIX 184 | 185 | 2. {{ username }} essay.task.util自动生成用户名 186 | 187 | 3. {{ password }} essay.task.util自动生成密码 188 | 189 | 4. {{ project }} 项目名,对应于fabfile/__init__.py里的env.PROJECT 190 | 191 | 5. {{ port }} 取env.VENV_PORT_PREFIX_MAP对应虚拟环境的端口号 192 | 193 | 6. {{ process_count }} 对应于fabfile配置中的PROCESS_COUNT 194 | 195 | 196 | **具体配置** 197 | 198 | #项目名（被自动替换） 199 | [program:{{ project }}] 200 | 201 | #运行命令 202 | command={{ venv_dir}}/bin/{{ project }} --profile={{ profile }} 203 | 204 | #进程名，示例：test_2 205 | process_name=%(program_name)s_%(process_num)d 206 | 207 | #设置进程umask，即权限为755 208 | umask=022 209 | 210 | #启动后10秒内没有异常则认为启动成功 211 | startsecs=10 212 | 213 | #等待0秒退出 214 | stopwaitsecs=0 215 | 216 | #重定向日志输出地址 217 | redirect_stderr=true 218 | 219 | 220 | #日志输出地址 221 | stdout_logfile={{ run_root }}/logs/process_%(process_num)02d.log 222 | 223 | 224 | #启动进程数 225 | numprocs={{ process_count }} 226 | 227 | #如果开启进程数大于1，则指定从序号为0的进程开始 228 | numprocs_start=0 229 | 230 | 231 | 232 | # 可用命令清单及详解 233 | 234 | ------------------------------------ 235 | 236 | **1. 创建本地工程,然后同步到git服务器(需要git仓库地址)** 237 | 238 | 用默认模板创建 239 | 240 | 1. 先在gitlab/github上创建库：myprojecti，拿到git地址 241 | 242 | 2. es create myproject (后边不加-t参数会使用默认模板创建项目) 243 | 244 | 3. 输入项目git地址 245 | 246 | 用django模板创建 247 | 248 | es create myproject -t django 249 | 250 | **2. 创建本地工程,创建时不需要连接git服务器,后期需要自己git init** 251 | 252 | #####用法一： 253 | 254 | 生成全新项目 255 | 256 | 1. es init myproject 257 | 258 | 2. cd myproject 259 | 260 | 3. git init 261 | 262 | 4. git remote add origin [git库地址] 263 | 264 | 5. 然后就可以commit和push了 265 | 266 | #####用法二： 267 | 268 | 在已存在项目下 269 | 270 | 1. cd myproject 271 | 272 | 2. es init 273 | 274 | 会为项目生成fabfile, conf, setup.py, MANIFEST.in, README.md 275 | 276 | **3. 打包** 277 | 278 | 1. fab build.build 279 | 280 | **4. 获取项目最新版本号** 281 | 282 | 1. cd myproject 283 | 284 | 2. fab build.get_latest_version 285 | 286 | **5. 计算项目下一版本号** 287 | 288 | 1. cd myproject 289 | 290 | 2. fab build.get_next_version 291 | 292 | **6. 部署(参数: 项目版本号, 虚拟环境名, profile)** 293 | 294 | 1. cd myproject 295 | 296 | 2. fab deploy:3.11.18.2,virtualenv_name,test 297 | 298 | **7. 重启指定虚拟环境的supervisor(参数: 虚拟环境名)** 299 | 300 | 1.cd myproject 301 | 302 | 2.fab supervisor.reload:virtualenv_name 303 | 304 | 305 | **8. 重启虚拟环境的项目的特定进程(参数: 进程名, 虚拟环境名)** 306 | 307 | 1. cd myproject 308 | 309 | 2. fab supervisor.restart_process:process_name,virtualenv_name 310 | 311 | **9. 启动虚拟环境的项目的特定进程(参数: 进程名, 虚拟环境名)** 312 | 313 | 1. cd myproject 314 | 315 | 2. fab supervisor.start_process:process_name,virtualenv_name 316 | 317 | **10. 关闭虚拟环境的项目的特定进程(参数: 进程名, 虚拟环境名)** 318 | 319 | 1. cd myproject 320 | 321 | 2. fab supervisor.stop_process:process_name,virtualenv_name 322 | 323 | 324 | **11. 启动nginx(参数: nginx执行命令地址, nginx配置文件地址)** 325 | 326 | 327 | 注意: nginx_bin, nginx_conf 可在myproject/fabfile/__init__.py中配置 328 | 例如: env.NGINX_BIN = '/usr/local/nginx/sbin/nginx' 329 | env.NGINX_CONF = '/usr/local/nginx/conf/nginx.conf' 330 | 331 | 1. cd myproject 332 | 333 | 2. fab nginx.start 334 | 335 | 如果不配置__init__.py 336 | 337 | fab nginx.start:nginx_bin,nginx_conf 338 | 339 | **12. 重启nginx(参数: nginx执行命令地址, nginx配置文件地址)** 340 | 341 | 1. cd myproject 342 | 343 | 2. fab nginx.reload:nginx_bin,nginx_conf 344 | 345 | **13. 关闭nginx(参数: nginx执行命令地址, nginx配置文件地址)** 346 | 347 | 1. cd myproject 348 | 349 | 2. fab nginx.stop:nginx_bin,nginx_conf 350 | 351 | 352 | **14. 修改nginx配置文件并重启(参数: 源文本, 目标文本, nginx执行命令令, nginx配置文件地址)** 353 | 354 | 355 | 1. cd myproject 356 | 357 | 2. fab nginx.switch:src_pattern,dst_pattern,nginx_bin,nginx_conf 358 | 359 | 360 | **15. 获取帮助** 361 | 362 | es -h 363 | 364 | **16. 从内部pypi安装包,pypi服务器地址在project/__init__.py中可设置** 365 | 366 | es pinstall xxx 367 | 368 | **17. 从官方pypi安装包(支持pip其他命令)** 369 | 370 | es install xxx 371 | 372 | 373 | 374 | # 高级功能介绍 375 | 376 | ------------------------------------ 377 | 378 | 介绍nginx服务器配置以及a，b环境切换的处理 379 | 380 | **功能：** 381 | 382 | 1. 保证新代码上线时，重启某一虚拟环境中的web服务，不会对用户访问造成中断。 383 | 2. 新上线代码出现bug时，及时回滚到上一次的版本。 384 | 385 | **场景介绍** 386 | 387 | 假设目前有server1和server2两台服务器，ip分别为ip1、ip2。两台服务器均部署了两个虚拟环境a和b。a环境运行的项目监听在端口8801，b环境运行的项目监听在端口8811。 388 | 389 | 下面是简单的nginx示例： 390 | 391 | *nginx.conf* 392 | 393 | user nginx; 394 | worker_processes 1; 395 | 396 | #error_log /var/log/nginx/error.log warn; 397 | pid /var/run/nginx.pid; 398 | 399 | events { 400 | worker_connections 1024; 401 | } 402 | 403 | http { 404 | include /etc/nginx/mime.types; 405 | default_type application/octet-stream; 406 | 407 | log_format main '$remote_addr - $remote_user [$time_local] "$request" ' 408 | '$status $body_bytes_sent "$http_referer" ' 409 | '"$http_user_agent" "$http_x_forwarded_for"'; 410 | 411 | #access_log /var/log/nginx/access.log main; 412 | 413 | sendfile on; 414 | #tcp_nopush on; 415 | 416 | keepalive_timeout 65; 417 | 418 | #gzip on; 419 | 420 | 421 | include extra/essay_demo.conf; 422 | include extra/upstreamA.conf; 423 | } 424 | 425 | *extra/essay_demo.conf* 426 | 427 | server { 428 | listen 80; 429 | server_name localhost; 430 | access_log /var/log/nginx/essay.access.log main; 431 | location / { 432 | proxy_pass http://essay_demo; 433 | proxy_intercept_errors on; 434 | proxy_redirect off; 435 | proxy_connect_timeout 60; 436 | proxy_set_header Host $host; 437 | proxy_set_header X-Real-IP $remote_addr; 438 | set $cookiesize '0'; 439 | if ($http_cookie ~ "_"){ 440 | set $cookiesize 1; 441 | } 442 | } 443 | } 444 | 445 | *extra/upstreamA.conf* 446 | 447 | upstream essay_demo { 448 | server ip1:8801; 449 | server ip2:8801; 450 | } 451 | 452 | *extra/upstreamB.conf* 453 | 454 | upstream essay_demo { 455 | server ip1:8811; 456 | server ip2:8811; 457 | } 458 | 459 | 目前nginx中配置的是upstreamA.conf，此时用户的访问会被反向代理到ip1和ip2的8801端口上。现在发布新版本上线，使用命令： ``fab -R dev deploy:<版本号>,b,test`` ,发布到b环境。 460 | 461 | 之后通过命令： ``fab -R nginx nginx.switch:upstreamA,upstreamB`` 把nginx中的upstreamA替换为upstreamB，并reload nginx，这样用户的访问就会被反向代理到ip1和ip2的8811端口上，也就是刚才发布到b环境中的新部署的项目。 462 | 463 | **switch完整命令** 464 | fab -R nginx nginx.switch:src_pattern,dst_pattern,root,nginx_bin,nginx_conf 465 | 466 | *switch参数解释：* 467 | 468 | src_pattern: 原文本 (必填) 469 | 470 | dst_pattern: 目标文本 (必填) 471 | 472 | root: 是否使用root用户, 是：root 否：None (可选项，默认为None) 473 | 474 | nginx_bin: nginx执行命令路径，可以在fabfile/__init__.py下设定 (可选项) 475 | 476 | nginx_conf: nginx配置文件路径，可以在fabfile/__init__.py下设定 (可选项) 477 | 478 | 479 | **图示** 480 | 481 | 482 | ![essay](data/1.png) 483 | 484 | ### fab -R dev nginx.switch: upstreamA.conf, upstreamB.conf 485 | 486 | ![essay](data/2.png) 487 | --------------------------------------------------------------------------------