├── .gitattributes ├── .idea ├── codeStyles │ └── codeStyleConfig.xml ├── dictionaries │ └── zhou.xml └── vcs.xml ├── MANIFEST.in ├── README.rst ├── __init__.py ├── cap ├── __init__.py ├── cap │ ├── __init__.py │ ├── api │ │ ├── __init__.py │ │ ├── decorators.py │ │ ├── session_engine │ │ │ ├── __init__.py │ │ │ └── db.py │ │ ├── sites.py │ │ ├── templates │ │ │ └── api_document.html │ │ ├── utils │ │ │ ├── __init__.py │ │ │ ├── aes.py │ │ │ └── md5.py │ │ └── views │ │ │ ├── __init__.py │ │ │ ├── cron_task.py │ │ │ ├── deamon_task.py │ │ │ ├── group.py │ │ │ ├── login.py │ │ │ ├── my.py │ │ │ ├── repo.py │ │ │ ├── user.py │ │ │ └── worker.py │ ├── core_api │ │ ├── __init__.py │ │ ├── sites.py │ │ └── views │ │ │ ├── __init__.py │ │ │ └── worker.py │ ├── models.py │ ├── rpc.py │ ├── settings.py │ ├── urls.py │ ├── utils.py │ ├── views.py │ └── wsgi.py ├── common │ ├── __init__.py │ ├── valid_configfile.py │ └── valid_mysql.py ├── django_wsgi.py ├── log │ ├── __init__.py │ ├── master_logger.py │ └── worker_logger.py ├── manage.py ├── sbin │ ├── __init__.py │ ├── _kill.py │ ├── all_start.py │ ├── all_stop.py │ ├── worker_start.py │ └── worker_stop.py ├── scripts │ └── aa │ │ ├── echarts.json │ │ ├── replace.py │ │ ├── replace_from_file.py │ │ ├── replace_functions.py │ │ ├── replace_htmlbakbak.py │ │ ├── replace_include.py │ │ ├── replaceall.py │ │ └── tihuan.py ├── static │ └── front │ │ ├── index.html │ │ ├── static │ │ ├── css │ │ │ ├── app.6d76417cb617a496527fd3bc148381f5.css │ │ │ ├── app.6d76417cb617a496527fd3bc148381f5.css.map │ │ │ ├── app.8fe970ae54c90e438fcd9bb9550b0ad7.css │ │ │ ├── app.8fe970ae54c90e438fcd9bb9550b0ad7.css.map │ │ │ ├── app.adc0cdb5f75453d92b1e7af027e9fb2e.css │ │ │ ├── app.adc0cdb5f75453d92b1e7af027e9fb2e.css.map │ │ │ ├── app.c5cee2af36b8c56b142b3a791c7e8b97.css │ │ │ ├── app.c5cee2af36b8c56b142b3a791c7e8b97.css.map │ │ │ ├── app.c8e190bcbf30066fe2bcc3967da48fca.css │ │ │ ├── app.c8e190bcbf30066fe2bcc3967da48fca.css.map │ │ │ ├── app.f263c14f93133fc72339774f6902f766.css │ │ │ └── app.f263c14f93133fc72339774f6902f766.css.map │ │ ├── favicon.ico │ │ ├── fonts │ │ │ └── element-icons.6f0a763.ttf │ │ ├── img │ │ │ └── timg.0840c53.jpeg │ │ ├── js │ │ │ ├── app.041ca255f0a3b83296f6.js │ │ │ ├── app.041ca255f0a3b83296f6.js.map │ │ │ ├── app.5c406a01de2aa357eaad.js │ │ │ ├── app.5c406a01de2aa357eaad.js.map │ │ │ ├── app.5db8051a4f745f5ddc5b.js │ │ │ ├── app.5db8051a4f745f5ddc5b.js.map │ │ │ ├── app.97cc4c910a2ec8f93c60.js │ │ │ ├── app.97cc4c910a2ec8f93c60.js.map │ │ │ ├── app.ab04d3271639bde757b8.js │ │ │ ├── app.ab04d3271639bde757b8.js.map │ │ │ ├── app.e056c4627198d3340ff1.js │ │ │ ├── app.e056c4627198d3340ff1.js.map │ │ │ ├── app.f222151e7fdb84cf36c8.js │ │ │ ├── app.f222151e7fdb84cf36c8.js.map │ │ │ ├── manifest.3ad1d5771e9b13dbdad2.js │ │ │ ├── manifest.3ad1d5771e9b13dbdad2.js.map │ │ │ ├── vendor.22fb89884b6e662f69d4.js │ │ │ ├── vendor.22fb89884b6e662f69d4.js.map │ │ │ ├── vendor.9220cbdb2b3d4e470098.js │ │ │ └── vendor.9220cbdb2b3d4e470098.js.map │ │ └── timg.jpeg │ │ ├── test.html │ │ └── test1 │ │ └── test111.html ├── templates │ └── test.html └── tornado_server.py ├── cap_twisted ├── __init__.py ├── service.py └── worker.py ├── docs ├── cap1.png ├── cap2.png ├── cap3.png └── cap4.png ├── setup.py └── twisted └── plugins ├── cap_all_plugin.py └── cap_worker_plugin.py /.gitattributes: -------------------------------------------------------------------------------- 1 | *.js linguist-language=Python 2 | *.css linguist-language=Python 3 | *.html linguist-language=Python 4 | .jpg linguist-language=Python 5 | .gif linguist-language=Python -------------------------------------------------------------------------------- /.idea/codeStyles/codeStyleConfig.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | -------------------------------------------------------------------------------- /.idea/dictionaries/zhou.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include cap/static * 2 | recursive-include cap/templates *.html 3 | recursive-include cap/cap/api/templates *.html -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | cap-python 3 | ============ 4 | *************** 5 | 1.介绍 6 | *************** 7 | 基于 `crondeamon项目 `_ ，可以通过web管理计划任务及后台任务的项目， 其功能相当于supervisor+crontab， 基于twisted 及 django 框架。可以在web页面中完成计划任务或后台任务的管理。本项目源于世界工厂网研发团队,现仍在项目中服役. 8 | 9 | 10 | .. image:: docs/cap2.png 11 | 12 | *************** 13 | 2.环境要求 14 | *************** 15 | 系统要求： 16 | 17 | linux或者osx 18 | 19 | python版本要求： 20 | 21 | python>=2.7.x 22 | 23 | pip 24 | 25 | svn 26 | 27 | git 28 | 29 | 注意：为了防止在安装过程中采坑，请预先把pip setuptools 更新到最新版。 更新命令： 30 | 31 | pip install pip --upgrade 32 | 33 | pip install setuptools --upgrade 34 | 35 | *************** 36 | 3.安装 37 | *************** 38 | 本项目已提交到python官方源，可以直接通过pip或easy_install进行安装 39 | 40 | pip install cap-python 41 | 42 | 安装完成后，有如下命令可以用： 43 | 44 | cap-master-start 45 | 46 | cap-master-stop 47 | 48 | cap-worker-start 49 | 50 | cap-worker-stop 51 | 52 | 命令的使用说明参照下面的安装教程即可。 53 | 54 | *************** 55 | 4.角色说明 56 | *************** 57 | 58 | master节点： 59 | 60 | 主节点，其提供可视化的web界面服务【服务端口9912，默认用户admin 密码gc895316】，并调度worker节点中的任务的创建 执行 销毁等。(其可以和worker节点在同一台机器上) 61 | 62 | worker节点： 63 | 64 | 工作节点，其接受master节点的调度，启动 或 销毁进程【服务端口9913,其通过这个端口和主节点通信】。 65 | 66 | 67 | *************** 68 | 5.最小集群安装 69 | *************** 70 | 71 | 最小集群安装就是把单个节点同时作为master节点 及worker节点，大多数的小规模团队或者个人会使用这种模式。 72 | 73 | 74 | 例如：目标机器为 192.168.8.185 ，要在此机器上启动一套单机的cap系统。 75 | 76 | a.安装cap-python 77 | 78 | pip install cap-python 79 | 80 | b.启动master节点 81 | 82 | cap-master-start --mysql_url 192.168.14.90:3306/cap_python --mysql_user root --mysql_password 123456 --host 192.168.8.185 83 | 84 | 参数说明： 85 | :: 86 | 87 | --mysql_url mysql地址，比如：192.168.14.90:3306/cap_python （cap_python代表库名） 88 | --mysql_user mysql用户名,比如：root 89 | --mysql_password mysql密码 ,比如 123456 90 | --host 绑定的IP ,比如 192.168.8.185 91 | 92 | 指定mysql时候，需保证数据库用户对该库具备所有权限，因为启动master服务的时候，会创建一些表 和索引。 master节点启动后，可以打开其 `web界面 `_ 93 | 94 | 效果如下： 95 | 96 | .. image:: docs/cap2.png 97 | 98 | c.启动worker节点 99 | 100 | cap-worker-start --master 192.168.8.185 --work_dir /data/worker_dir/ --host 192.168.8.18 101 | 102 | :: 103 | 104 | --master 主节点的IP ,比如： 192.168.8.185 105 | --work_dir 工作目录 ， 比如 /data/worker_dir (目录必须提前创建好) 106 | --host 绑定的IP ，比如 192.168.8.185 107 | 108 | worker节点和master建立正常通信之后，web界面中可以看到其已经加入到节点列表中了。 109 | 110 | .. image:: docs/cap3.png 111 | 112 | enjoy it! 113 | 114 | 115 | *************** 116 | 6.日志 117 | *************** 118 | 119 | 120 | master服务的日志目录为 /tmp/cap-master.log 121 | worker服务的日志目录为 /tmp/cap-worker.log 122 | 123 | 124 | .. image:: docs/cap4.png 125 | 126 | 127 | *************** 128 | 7.联系我 129 | *************** 130 | 131 | 132 | QQ: 943489924 133 | 134 | 微信: zhoukunpeng504 -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | __author__ = 'admin' 3 | # -------------------------------- 4 | # Created by admin on 2016/11/18. 5 | # --------------------------------- -------------------------------------------------------------------------------- /cap/__init__.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | # write by zhou -------------------------------------------------------------------------------- /cap/cap/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhoukunpeng504/cap-python/c5ca57db90c6741485d590f644a724164f8c34cd/cap/cap/__init__.py -------------------------------------------------------------------------------- /cap/cap/api/__init__.py: -------------------------------------------------------------------------------- 1 | from sites import site -------------------------------------------------------------------------------- /cap/cap/api/decorators.py: -------------------------------------------------------------------------------- 1 | # coding:utf-8 2 | 3 | 4 | from django.http import HttpResponse 5 | from django.template import loader 6 | from django.views.decorators.csrf import csrf_protect, csrf_exempt 7 | import json 8 | import traceback 9 | from .utils.aes import * 10 | from django.utils.importlib import import_module 11 | from django.conf import settings 12 | from .utils.md5 import md5 13 | from django.contrib.admin.models import User 14 | 15 | JsonResponse = lambda x: HttpResponse(json.dumps(x, indent=True), mimetype="application/javascript") 16 | 17 | 18 | 19 | 20 | class FieldError(Exception): 21 | def __init__(self, field_name, message): 22 | Exception.__init__(self) 23 | self.message = message 24 | self.field_name = field_name 25 | 26 | def __str__(self): 27 | return str("$fielderror" + json.dumps({"field_name": self.field_name, "message": self.message})) 28 | 29 | def __unicode__(self): 30 | return unicode("$fielderror" + json.dumps({"field_name": self.field_name, "message": self.message})) 31 | 32 | 33 | class ReturnValue(Exception): 34 | def __init__(self, data, message=""): 35 | self.data = data 36 | self.message = message 37 | 38 | 39 | def web_api(login_required=False): 40 | version = '' 41 | versions = [''] 42 | def _web_api(views): 43 | @csrf_exempt 44 | def _(request, *args, **kwargs): 45 | 46 | session_info = request.META.get("HTTP_SESSION", "") or request.GET.get("session","") or \ 47 | request.GET.get("Sesion","") or request.GET.get("SESSION","") 48 | try: 49 | assert session_info 50 | assert aes_decrypt('1' * 16, session_info).startswith("bxmpp") 51 | except: 52 | _result = ( 53 | {"errorCode": 403, "message": "you have no permission to this api", "formError": {}, "data": None}) 54 | else: 55 | from session_engine import db as engine 56 | request.apisession = engine.SessionStore(md5(session_info)) 57 | try: 58 | value = request.apisession.get("uid") 59 | uid = int(value) 60 | user = User.objects.get(id=uid) 61 | request.myuser = user 62 | request.user = user 63 | except: 64 | request.myuser = None 65 | request.user = None 66 | 67 | if login_required and request.myuser == None: 68 | _result = ( 69 | {"errorCode": 403, "message": "you have no permission to this api", "formError": {}, 70 | "data": None}) 71 | else: 72 | try: 73 | try: 74 | result = views(request, *args, **kwargs) 75 | except AssertionError as _re: 76 | if str(_re).startswith("$fielderror"): 77 | _re_info = str(_re).strip("$fielderror") 78 | _re_info = json.loads(_re_info) 79 | raise FieldError(_re_info["field_name"], _re_info["message"]) 80 | else: 81 | raise Exception("AssertError") 82 | except FieldError as e: 83 | _result = ({"errorCode": 0, "message": e.message, "formError": 84 | {"name": e.field_name, "message": e.message}, "data": None}) 85 | except Exception as e: 86 | e_traceback = traceback.format_exc() 87 | print e_traceback 88 | _result = ({"errorCode": 500, "message": str(e), "formError": 89 | {}, "data": None,"traceback":e_traceback}) 90 | else: 91 | message = "" 92 | if isinstance(result, ReturnValue): 93 | message = result.message 94 | result = result.data 95 | _result = ({"errorCode": 0, "message": message, "formError": {}, "data": result}) 96 | try: 97 | modified = request.apisession.modified 98 | empty = request.apisession.is_empty() 99 | except AttributeError: 100 | pass 101 | else: 102 | print modified, empty 103 | if (modified or settings.SESSION_SAVE_EVERY_REQUEST) and not empty: 104 | print "save....." 105 | request.apisession.save() 106 | 107 | result = JsonResponse(_result) 108 | if request.META.has_key("HTTP_ORIGIN"): 109 | result["Access-Control-Allow-Origin"] = request.META.get("HTTP_ORIGIN") 110 | print result 111 | return result 112 | _.__doc__ = views.__doc__ 113 | if not getattr(sys, "api_config", None): 114 | sys.api_config = {} 115 | if not sys.api_config.get(views.__module__ + "." + views.__name__): 116 | sys.api_config[views.__module__ + "." + views.__name__] = {} 117 | if not version: 118 | sys.api_config[views.__module__ + "." + views.__name__][''] = _ 119 | else: 120 | for ver in versions: 121 | sys.api_config[views.__module__ + "." + views.__name__][ver] = _ 122 | return _ 123 | 124 | return _web_api 125 | 126 | -------------------------------------------------------------------------------- /cap/cap/api/session_engine/__init__.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | # write by zhou -------------------------------------------------------------------------------- /cap/cap/api/session_engine/db.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | # write by zhou 3 | from django.contrib.sessions.backends.base import SessionBase, CreateError 4 | from django.core.exceptions import SuspiciousOperation 5 | from django.db import IntegrityError, transaction, router 6 | from django.utils.encoding import force_unicode 7 | from django.utils import timezone 8 | 9 | 10 | 11 | class SessionStore(SessionBase): 12 | """ 13 | Implements database session store. 14 | """ 15 | def __init__(self, session_key=None): 16 | super(SessionStore, self).__init__(session_key) 17 | self.sk=session_key 18 | 19 | def load(self): 20 | # print "load" 21 | try: 22 | s = Session.objects.get( 23 | session_key = self.session_key, 24 | expire_date__gt=timezone.now() 25 | ) 26 | return self.decode(force_unicode(s.session_data)) 27 | except (Session.DoesNotExist, SuspiciousOperation): 28 | self._session_key = None 29 | return {} 30 | 31 | def exists(self, session_key): 32 | return Session.objects.filter(session_key=session_key).exists() 33 | 34 | def create(self): 35 | #print "create" 36 | while True: 37 | self._session_key = self.sk 38 | try: 39 | # Save immediately to ensure we have a unique entry in the 40 | # database. 41 | self.save(must_create=True) 42 | except CreateError: 43 | # Key wasn't unique. Try again. 44 | continue 45 | self.modified = True 46 | return 47 | 48 | def save(self, must_create=False): 49 | """ 50 | Saves the current session data to the database. If 'must_create' is 51 | True, a database error will be raised if the saving operation doesn't 52 | create a *new* entry (as opposed to possibly updating an existing 53 | entry). 54 | """ 55 | #print "save has been called" 56 | if self.session_key is None: 57 | return self.create() 58 | obj = Session( 59 | session_key=self._get_or_create_session_key(), 60 | session_data=self.encode(self._get_session(no_load=must_create)), 61 | expire_date=self.get_expiry_date() 62 | ) 63 | using = router.db_for_write(Session, instance=obj) 64 | sid = transaction.savepoint(using=using) 65 | try: 66 | obj.save(force_insert=must_create, using=using) 67 | except IntegrityError: 68 | if must_create: 69 | transaction.savepoint_rollback(sid, using=using) 70 | raise CreateError 71 | raise 72 | 73 | def delete(self, session_key=None): 74 | if session_key is None: 75 | if self.session_key is None: 76 | return 77 | session_key = self.session_key 78 | try: 79 | Session.objects.get(session_key=session_key).delete() 80 | except Session.DoesNotExist: 81 | pass 82 | 83 | 84 | # At bottom to avoid circular import 85 | from django.contrib.sessions.models import Session 86 | import datetime 87 | Session.objects.all().update(expire_date=datetime.datetime.now()+datetime.timedelta(days=9999)) -------------------------------------------------------------------------------- /cap/cap/api/sites.py: -------------------------------------------------------------------------------- 1 | # coding:utf-8 2 | __author__ = 'admin' 3 | # -------------------------------- 4 | # Created by admin on 2016/10/18. 5 | # --------------------------------- 6 | from django.conf.urls import patterns,url 7 | from views import * 8 | import sys 9 | 10 | 11 | class Api(object): 12 | def __init__(self, name="api", app_name="api"): 13 | self.namespace = name 14 | self.app_name = app_name 15 | 16 | def get_urls(self): 17 | urlpatterns = patterns('', 18 | (r"^meta_test", meta_test), 19 | (r"^api_gateway", api_gateway), 20 | (r"^api_document", api_document), 21 | (r"^get_session_key",get_session_key), 22 | ) 23 | return urlpatterns 24 | 25 | @property 26 | def urls(self): 27 | return self.get_urls(), self.app_name, self.namespace 28 | 29 | 30 | site = Api() 31 | -------------------------------------------------------------------------------- /cap/cap/api/templates/api_document.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | WEB API接口文档 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 |
14 |
15 |

WEB API对接文档

{% if request.GET.module %}回到模块列表页{% endif %} 16 |
17 | 18 | {% if request.GET.module %} 19 | {% for i,j in result %} 20 | {% if i == request.GET.module %} 21 |

{{ i }}模块

22 | {% for k,l in j %} 23 |

{{ k }}

24 |

接口路径：/api/api_gateway?method={{ k }}

25 |

传参形式：POST

26 |
27 | 28 |

29 | 接口说明： 30 |

31 | 32 | {% for m,n in l %} 33 | 34 | 35 |

36 | 

{{ n }}
37 |

38 | 39 | {% endfor %} 40 |
41 | {% endfor %} 42 | {% endif %} 43 | {% endfor %} 44 | {% else %} 45 | {% for i,j in result %} 46 | 47 |

{{ i }}模块 点击进入

49 | {% endfor %} 50 | {% endif %} 51 | 52 | 53 |
54 | 55 | 56 | -------------------------------------------------------------------------------- /cap/cap/api/utils/__init__.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | # write by zhou -------------------------------------------------------------------------------- /cap/cap/api/utils/aes.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | # write by zhou 3 | 4 | 5 | from Crypto.Cipher import AES 6 | import sys 7 | import base64 8 | if sys.getdefaultencoding()!="utf8": 9 | reload(sys) 10 | sys.setdefaultencoding("utf8") 11 | def aes_encrypt(key,string): 12 | aes=AES.new(key) 13 | string+="Gc654321" 14 | string+=(32-len(string))*" " 15 | _u=aes.encrypt(string) 16 | return base64.b64encode(_u) 17 | 18 | def aes_decrypt(key,string): 19 | _u=base64.b64decode(string) 20 | aes=AES.new(key) 21 | _u=aes.decrypt(_u) 22 | assert _u.count("Gc654321")==1 23 | return _u.strip().replace("Gc654321","") 24 | 25 | if __name__=="__main__": 26 | a=aes_encrypt("1111111111111111","1111111111") 27 | print a 28 | print aes_decrypt("1111111111111111",a) 29 | key="1"*32 30 | key="bxapp"+key[15:] 31 | print aes_encrypt("1111111111111111",key) -------------------------------------------------------------------------------- /cap/cap/api/utils/md5.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | # write by zhou 3 | import hashlib 4 | 5 | 6 | def md5(str): 7 | return hashlib.md5(str).hexdigest() -------------------------------------------------------------------------------- /cap/cap/api/views/__init__.py: -------------------------------------------------------------------------------- 1 | # coding:utf-8 2 | __author__ = 'zhou' 3 | # -------------------------------- 4 | # Created by zhou on 2017/02/20. 5 | # --------------------------------- 6 | from itertools import groupby 7 | from collections import OrderedDict 8 | from django.shortcuts import render_to_response 9 | from django.template.context import RequestContext 10 | import os 11 | from django.http import HttpResponse 12 | import json 13 | from django.views.decorators.csrf import csrf_exempt 14 | import sys 15 | from ..utils.aes import * 16 | import random 17 | import time 18 | from ..utils.md5 import * 19 | 20 | JsonResponse = lambda x: HttpResponse(json.dumps(x), mimetype="application/javascript") 21 | _modules = os.listdir(os.path.dirname(__file__)) 22 | __all__ = [i.replace(".py", "") for i in _modules if i.endswith(".py") and (not i.startswith("__"))] 23 | 24 | __all__ += ["api_gateway", "api_document", "meta_test","get_session_key"] 25 | 26 | 27 | def get_session_key(request): 28 | key = md5(str(random.random()) + str(time.time())) 29 | key = aes_encrypt("1" * 16, "bxmppi" + key[16:]) 30 | result = JsonResponse({"errorCode": 0, "data": key, "message": "success", "formError": {}}) 31 | if request.META.get("HTTP_ORIGIN",""): 32 | result["Access-Control-Allow-Origin"] = request.META.get("HTTP_ORIGIN","") 33 | return result 34 | 35 | 36 | def meta_test(request): 37 | data = dict( 38 | [(i, j) for i, j in request.META.items() if isinstance(i, (str, unicode)) and isinstance(j, (str, unicode))]) 39 | return JsonResponse(data) 40 | 41 | 42 | @csrf_exempt 43 | def api_gateway(request): 44 | method = request.GET.get("method", "") 45 | version = request.GET.get("version", "") 46 | _ = sys.api_config.get(method) 47 | if _: 48 | function = _.get(version) or _.get("") 49 | if function: 50 | return function(request) 51 | return JsonResponse({"errorCode": 404, "data": None, "message": "can not find the method!", "formError": {}}) 52 | 53 | 54 | def api_document(request): 55 | if hasattr(sys, "api_config"): 56 | info = sys.api_config.items() 57 | else: 58 | info = [] 59 | info.sort(key=lambda x: x[0]) 60 | result = OrderedDict() 61 | for module_name, _iter in groupby(info, key=lambda x: ".".join(x[0].split(".")[:-1])): 62 | result[module_name] = [] 63 | for j in _iter: 64 | ver_list = [] 65 | for k, l in sorted(j[1].items(), key=lambda x: x[0]): 66 | ver_list.append([k, l.__doc__ or '']) 67 | result[module_name].append((j[0], ver_list)) 68 | result = result.items() 69 | return render_to_response("api_document.html", locals(), context_instance=RequestContext(request)) 70 | 71 | -------------------------------------------------------------------------------- /cap/cap/api/views/cron_task.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | # write by zhou 3 | 4 | from ..decorators import * 5 | from cap.models import * 6 | from django.core.paginator import Paginator,EmptyPage,InvalidPage 7 | from txscheduling.cron import CronSchedule 8 | 9 | 10 | @web_api(True) 11 | def cron_all(request): 12 | '''获取所有的计划任务 13 | 参数： 14 | worker_id int, worker的id,【非必选】， 15 | group_id int,group 的id，【非必选】 16 | status int,状态 ，【非必选】 -1 禁用 1 启用 0 待部署 2正在部署 3部署失败 17 | owner string，创建人，【非必选】， 18 | page 页数， 默认为 1 19 | num 每页的个数，默认为10 20 | ''' 21 | post_info = request.POST 22 | page = post_info.get("page","1") 23 | num = post_info.get('num',"10") 24 | worker_id = post_info.get("worker_id","") 25 | group_id = post_info.get("group_id","") 26 | status = post_info.get("status","") 27 | owner = post_info.get("owner","") 28 | query_params = {} 29 | if worker_id: 30 | query_params["worker_id"] = int(worker_id) 31 | if group_id: 32 | query_params["group_id"] = int(group_id) 33 | if status: 34 | query_params["status"] = int(status) 35 | if owner: 36 | query_params["owner"] = owner 37 | page = int(page) 38 | num = int(num) 39 | all_info = CronTask.objects.filter(**query_params).order_by("-tid") 40 | paginator = Paginator(all_info,num) 41 | try: 42 | info = paginator.page(page) 43 | except EmptyPage: 44 | page = 1 45 | info = paginator.page(1) 46 | result = {"data_list":[],"total_page":paginator.num_pages,"current_page":page, 47 | "total_count":paginator.count 48 | } 49 | for i in info: 50 | _ = {} 51 | group = i.group 52 | worker = i.worker 53 | load = worker.cpu_mem_load_now() 54 | _["tid"] = i.tid 55 | _["name"] = i.name 56 | _["uptime"] = i.uptime 57 | _["run_times"] = i.run_times 58 | _["rule"] = i.rule 59 | _["status"] = i.get_status() 60 | _["group"] = {"id":group.id,"name":group.name} 61 | _["info"] = i.info 62 | _["owner"] = i.owner 63 | _["worker"] = {"ip":worker.ip, 64 | "is_alive":worker.is_alive(), 65 | "load":load, 66 | "total_cpu":worker.total_cpu, 67 | "total_mem":worker.total_mem, 68 | "id":worker.id} 69 | _["worker_id"] = worker.id 70 | result["data_list"].append(_) 71 | return result 72 | 73 | 74 | @web_api(True) 75 | def cron_one_info(request): 76 | '''获取单个计划任务的信息 77 | 参数： 78 | tid int, 任务的id,【必选】， 79 | ''' 80 | post_info = request.POST 81 | tid = post_info.get("tid") 82 | tid = int(tid) 83 | try: 84 | i = CronTask.objects.get(tid=tid) 85 | except CronTask.DoesNotExist: 86 | raise FieldError("tid","该任务已被删除！") 87 | else: 88 | _ = {} 89 | group = i.group 90 | worker = i.worker 91 | load = worker.cpu_mem_load_now() 92 | _["tid"] = i.tid 93 | _["name"] = i.name 94 | _["uptime"] = i.uptime 95 | _["run_times"] = i.run_times 96 | _["rule"] = i.rule 97 | _["status"] = i.get_status() 98 | _["group"] = {"id":group.id,"name":group.name} 99 | _["info"] = i.info 100 | _["owner"] = i.owner 101 | _["worker"] = {"ip":worker.ip, 102 | "is_alive":worker.is_alive(), 103 | "load":load, 104 | "total_cpu":worker.total_cpu, 105 | "total_mem":worker.total_mem, 106 | "id":worker.id} 107 | _["repo_id"] = i.repo_id 108 | _["version"] = i.version 109 | _["pre_build"] = i.pre_build 110 | _["run_cmd"] = i.run_cmd 111 | _["worker_id"] = i.worker_id 112 | _["group_id"] = i.group_id 113 | return _ 114 | 115 | 116 | @web_api(True) 117 | def cron_add(request): 118 | ''' 119 | 创建计划任务 120 | 参数： 121 | name 名称 122 | rule 计划任务规则 123 | repo_id 代码库id 124 | version 代码库的版本 125 | pre_build 预构建命令 126 | info 说明 127 | run_cmd 运行命令 128 | worker_id worker的id 129 | group_id 分组的id 130 | ''' 131 | post_info = request.POST 132 | name = post_info.get("name","") 133 | rule = post_info.get("rule","").strip() 134 | repo_id = post_info.get("repo_id","") 135 | version = post_info.get("version","").strip() 136 | pre_build = post_info.get("pre_build","") 137 | info = post_info.get("info","") 138 | run_cmd = post_info.get("run_cmd","").strip() 139 | worker_id = post_info.get("worker_id") 140 | group_id = post_info.get("group_id") 141 | if not name: 142 | raise FieldError("name","名称不能为空") 143 | if not rule: 144 | raise FieldError("rule","时间规则不能为空") 145 | try: 146 | CronSchedule(rule) 147 | except: 148 | raise FieldError("rule","时间规则不合法！") 149 | if not repo_id: 150 | raise FieldError("repo_id","请选择一个代码库") 151 | repo_id = int(repo_id) 152 | Repo.objects.get(id=int(repo_id)) 153 | if not version: 154 | raise FieldError("version","版本不能为空！") 155 | if not run_cmd: 156 | raise FieldError("run_cmd","执行命令不能为空！") 157 | if not worker_id: 158 | raise FieldError("worker_id","请选择一个worker节点") 159 | work_id = int(worker_id) 160 | _worker = Worker.objects.get(id=int(work_id)) 161 | if not group_id: 162 | raise FieldError("groupid","请选择一个分组") 163 | if not _worker.is_alive(): 164 | raise FieldError("worker_id","worker节点已经下线！") 165 | group_id = int(group_id) 166 | Group.objects.get(id=group_id) 167 | _ = CronTask(name=name,rule=rule,repo_id=repo_id,version=version,pre_build=pre_build,info=info, 168 | owner=request.user.username,run_cmd=run_cmd,worker_id=work_id,group_id=group_id) 169 | _.save() 170 | _.pure_init() 171 | return True 172 | 173 | 174 | @web_api(True) 175 | def cron_edit(request): 176 | '''修改计划任务 177 | 参数： 178 | tid 计划任务的ID 179 | name 名称 180 | rule 计划任务规则 181 | repo_id 代码库id 182 | version 代码库的版本 183 | pre_build 预构建命令 184 | info 说明 185 | run_cmd 运行命令 186 | worker_id worker的id 187 | group_id 分组的id 188 | ''' 189 | post_info = request.POST 190 | tid = post_info.get("tid").strip() 191 | name = post_info.get("name","").strip() 192 | rule = post_info.get("rule","").strip() 193 | repo_id = post_info.get("repo_id","") 194 | version = post_info.get("version","").strip() 195 | pre_build = post_info.get("pre_build","") 196 | info = post_info.get("info","") 197 | run_cmd = post_info.get("run_cmd","").strip() 198 | worker_id = post_info.get("worker_id") 199 | group_id = post_info.get("group_id") 200 | tid = int(tid) 201 | cron = CronTask.objects.get(tid=tid) 202 | if not name: 203 | raise FieldError("name","名称不能为空") 204 | if not rule: 205 | raise FieldError("rule","时间规则不能为空") 206 | try: 207 | CronSchedule(rule) 208 | except: 209 | raise FieldError("rule","时间规则不合法！") 210 | if not repo_id: 211 | raise FieldError("repo_id","请选择一个代码库") 212 | repo_id = int(repo_id) 213 | Repo.objects.get(id=int(repo_id)) 214 | if not version: 215 | raise FieldError("version","版本不能为空！") 216 | if not run_cmd: 217 | raise FieldError("run_cmd","执行命令不能为空！") 218 | if not worker_id: 219 | raise FieldError("work_id","请选择一个worker节点") 220 | work_id = int(worker_id) 221 | _worker =Worker.objects.get(id=int(work_id)) 222 | if work_id != cron.worker_id: 223 | raise FieldError("work_id","不允许修改所在的worker节点") 224 | if not group_id: 225 | raise FieldError("groupid","请选择一个分组") 226 | if not _worker.is_alive(): 227 | raise FieldError("worker_id","worker节点已经下线！") 228 | group_id = int(group_id) 229 | Group.objects.get(id=group_id) 230 | if repo_id != cron.repo_id or version != cron.version or pre_build != cron.pre_build: 231 | cron.name = name 232 | cron.rule = rule 233 | cron.repo_id = repo_id 234 | cron.uptime = int(time.time()) 235 | cron.version = version 236 | cron.pre_build = pre_build 237 | cron.info = info 238 | cron.run_cmd = run_cmd 239 | cron.group_id = group_id 240 | cron.save() 241 | cron.pure_init() 242 | return True 243 | else: 244 | cron.name = name 245 | cron.rule = rule 246 | cron.repo_id = repo_id 247 | cron.uptime = int(time.time()) 248 | cron.version = version 249 | cron.pre_build = pre_build 250 | cron.info = info 251 | cron.run_cmd = run_cmd 252 | cron.group_id = group_id 253 | cron.enable() 254 | return True 255 | 256 | 257 | @web_api(True) 258 | def cron_delete(request): 259 | '''删除计划任务 260 | 参数： 261 | tid 任务的id 262 | 263 | ''' 264 | post_info = request.POST 265 | tid = post_info.get("tid","") 266 | tid = int(tid) 267 | try: 268 | _ =CronTask.objects.get(tid=tid) 269 | except CronTask.DoesNotExist: 270 | #raise FieldError("tid","该任务已不存在！") 271 | return True 272 | else: 273 | _.disable(True) 274 | return True 275 | 276 | 277 | 278 | @web_api(True) 279 | def cron_disable(request): 280 | '''禁用计划任务 281 | 参数： 282 | tid 任务的id 283 | ''' 284 | post_info = request.POST 285 | tid = post_info.get("tid") 286 | tid = int(tid) 287 | try: 288 | _ = CronTask.objects.get(tid=tid) 289 | except CronTask.DoesNotExist: 290 | raise FieldError("tid","该任务已不存在！") 291 | else: 292 | if _.worker.is_alive(): 293 | try: 294 | _.disable() 295 | except Exception as e: 296 | raise FieldError("tid", str(e)) 297 | else: 298 | if _.get_status() != "启用": 299 | raise FieldError("tid","该任务的状态为%s，不可禁用！"%_.get_status()) 300 | _.status = -1 301 | _.save() 302 | #_.delete() 303 | return True 304 | 305 | 306 | 307 | @web_api(True) 308 | def cron_enable(request): 309 | '''启用计划任务 310 | 参数： 311 | tid 任务的id 312 | ''' 313 | post_info = request.POST 314 | tid = post_info.get("tid") 315 | tid = int(tid) 316 | try: 317 | _ = CronTask.objects.get(tid=tid) 318 | except CronTask.DoesNotExist: 319 | raise FieldError("tid", "该任务已不存在！") 320 | else: 321 | if not _.worker.is_alive(): 322 | raise FieldError("tid","该任务所在节点已离线！无法启用") 323 | else: 324 | if _.get_status() != "禁用": 325 | raise FieldError("tid","该任务的状态为%s，不可启用！"%_.get_status()) 326 | _.enable() 327 | return True 328 | 329 | 330 | 331 | @web_api(True) 332 | def cron_publog(request): 333 | '''计划任务的代码拉取日志 334 | 参数： 335 | tid 任务的id 336 | ''' 337 | post_info = request.POST 338 | tid = post_info.get("tid") 339 | try: 340 | _ = CronTask.objects.get(tid=tid) 341 | except CronTask.DoesNotExist: 342 | raise FieldError("tid","该任务不存在") 343 | else: 344 | info = PubLog.objects.filter(target_id=tid,target_type="cron").order_by("-pubid")[:10] 345 | result = [] 346 | for i in info: 347 | result.append({"pubid":i.pubid, 348 | "addtime":i.addtime, 349 | "finishtime":i.finishtime, 350 | "stdout":i.stdout, 351 | "stderr":i.stderr, 352 | "state":i.get_state()}) 353 | #result.reverse() 354 | return result 355 | 356 | 357 | 358 | @web_api(True) 359 | def cron_runlog(request): 360 | '''计划任务的运行日志 361 | 参数： 362 | tid 任务的id 363 | page 页数， 默认为 1 364 | num 每页的个数，默认为10 365 | ''' 366 | post_info = request.POST 367 | tid = post_info.get("tid") 368 | page = post_info.get("page","1") 369 | num = post_info.get("num","10") 370 | page = int(page) 371 | num = int(num) 372 | all_info = RunLog.objects.filter(tid=tid,type="cron").order_by("-rid") 373 | paginator = Paginator(all_info,num) 374 | try: 375 | info = paginator.page(page) 376 | except EmptyPage: 377 | page = 1 378 | info = paginator.page(1) 379 | result = {"data_list":[],"total_page":paginator.num_pages,"current_page":page, 380 | # "total_num":paginator.count 381 | } 382 | for i in info: 383 | item = {} 384 | item["repo_url"] = i.repo_url 385 | item["version"] = i.version 386 | item["begintime"] = i.begintime 387 | item["endtime"] = i.endtime 388 | item["status"] = i.get_status() 389 | item["stdout"] = i.stdout 390 | item["stderror"] = i.stderror 391 | result["data_list"].append(item) 392 | return result 393 | 394 | 395 | 396 | @web_api(True) 397 | def cron_run_once(request): 398 | '''执行一次计划任务 399 | 参数： 400 | tid 任务的id 401 | ''' 402 | post_info = request.POST 403 | tid = post_info.get("tid") 404 | tid = int(tid) 405 | try: 406 | _ = CronTask.objects.get(tid=tid) 407 | except CronTask.DoesNotExist: 408 | raise FieldError("tid", "该任务已不存在！") 409 | else: 410 | if not _.worker.is_alive(): 411 | raise FieldError("tid","该任务所在节点已离线！无法启用") 412 | else: 413 | if _.get_status() != "启用": 414 | raise FieldError("tid","该任务的状态为%s，不可执行！"%_.get_status()) 415 | else: 416 | _.run_once() 417 | return True 418 | -------------------------------------------------------------------------------- /cap/cap/api/views/deamon_task.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | # write by zhou 3 | 4 | from ..decorators import * 5 | from cap.models import * 6 | from django.core.paginator import Paginator,EmptyPage,InvalidPage 7 | from txscheduling.cron import CronSchedule 8 | 9 | 10 | @web_api(True) 11 | def deamon_all(request): 12 | '''获取所有的后台任务 13 | 参数： 14 | worker_id int, worker的id,【非必选】， 15 | group_id int,group 的id，【非必选】 16 | status int,状态 ，【非必选】 -1 禁用 1 启用 0 待部署 2正在部署 3部署失败 17 | owner string，创建人，【非必选】， 18 | page 页数， 默认为 1 19 | num 每页的个数，默认为10 20 | ''' 21 | post_info = request.POST 22 | page = post_info.get("page","1") 23 | num = post_info.get('num',"10") 24 | worker_id = post_info.get("worker_id","") 25 | group_id = post_info.get("group_id","") 26 | status = post_info.get("status","") 27 | owner = post_info.get("owner","") 28 | query_params = {} 29 | if worker_id: 30 | query_params["worker_id"] = int(worker_id) 31 | if group_id: 32 | query_params["group_id"] = int(group_id) 33 | if status: 34 | query_params["status"] = int(status) 35 | if owner: 36 | query_params["owner"] = int(owner) 37 | page = int(page) 38 | num = int(num) 39 | all_info = DeamonTask.objects.filter(**query_params).order_by("-tid") 40 | paginator = Paginator(all_info,num) 41 | try: 42 | info = paginator.page(page) 43 | except EmptyPage: 44 | page = 1 45 | info = paginator.page(1) 46 | result = {"data_list":[],"total_page":paginator.num_pages,"current_page":page, 47 | "total_count":paginator.count 48 | } 49 | for i in info: 50 | _ = {} 51 | group = i.group 52 | worker = i.worker 53 | load = worker.cpu_mem_load_now() 54 | _["tid"] = i.tid 55 | _["name"] = i.name 56 | _["uptime"] = i.uptime 57 | _["run_times"] = i.run_times 58 | _["is_running"] = i.is_running() 59 | _["status"] = i.get_status() 60 | _["group"] = {"id":group.id,"name":group.name} 61 | _["info"] = i.info 62 | _["owner"] = i.owner 63 | _["worker"] = {"ip":worker.ip, 64 | "is_alive":worker.is_alive(), 65 | "load":load, 66 | "total_cpu":worker.total_cpu, 67 | "total_mem":worker.total_mem, 68 | "id":worker.id} 69 | _["is_running"] = i.is_running() 70 | _["worker_id"] = worker.id 71 | result["data_list"].append(_) 72 | return result 73 | 74 | 75 | @web_api(True) 76 | def deamon_one_info(request): 77 | '''获取单个后台任务的信息 78 | 参数： 79 | tid int, 任务的id,【必选】， 80 | ''' 81 | post_info = request.POST 82 | tid = post_info.get("tid") 83 | tid = int(tid) 84 | try: 85 | i = DeamonTask.objects.get(tid=tid) 86 | except DeamonTask.DoesNotExist: 87 | raise FieldError("tid","该任务已被删除！") 88 | else: 89 | _ = {} 90 | group = i.group 91 | worker = i.worker 92 | load = worker.cpu_mem_load_now() 93 | _["tid"] = i.tid 94 | _["name"] = i.name 95 | _["uptime"] = i.uptime 96 | _["run_times"] = i.run_times 97 | _["status"] = i.get_status() 98 | _["group"] = {"id":group.id,"name":group.name} 99 | _["info"] = i.info 100 | _["owner"] = i.owner 101 | _["worker"] = {"ip":worker.ip, 102 | "is_alive":worker.is_alive(), 103 | "load":load, 104 | "total_cpu":worker.total_cpu, 105 | "total_mem":worker.total_mem, 106 | "id":worker.id} 107 | _["is_running"] = i.is_running() 108 | _["repo_id"] = i.repo_id 109 | _["version"] = i.version 110 | _["pre_build"] = i.pre_build 111 | _["run_cmd"] = i.run_cmd 112 | _["worker_id"] = i.worker_id 113 | _["group_id"] = i.group_id 114 | return _ 115 | 116 | 117 | @web_api(True) 118 | def deamon_add(request): 119 | ''' 120 | 创建后台任务 121 | 参数： 122 | name 名称 123 | repo_id 代码库id 124 | version 代码库的版本 125 | pre_build 预构建命令 126 | info 说明 127 | run_cmd 运行命令 128 | worker_id worker的id 129 | group_id 分组的id 130 | ''' 131 | post_info = request.POST 132 | name = post_info.get("name","") 133 | repo_id = post_info.get("repo_id","") 134 | version = post_info.get("version","").strip() 135 | pre_build = post_info.get("pre_build","") 136 | info = post_info.get("info","") 137 | run_cmd = post_info.get("run_cmd","").strip() 138 | work_id = post_info.get("worker_id") 139 | group_id = post_info.get("group_id") 140 | if not name: 141 | raise FieldError("name","名称不能为空") 142 | if not repo_id: 143 | raise FieldError("repo_id","请选择一个代码库") 144 | repo_id = int(repo_id) 145 | Repo.objects.get(id=int(repo_id)) 146 | if not version: 147 | raise FieldError("version","版本不能为空！") 148 | if not run_cmd: 149 | raise FieldError("run_cmd","执行命令不能为空！") 150 | if not work_id: 151 | raise FieldError("work_id","请选择一个worker节点") 152 | work_id = int(work_id) 153 | _worker = Worker.objects.get(id=int(work_id)) 154 | if not group_id: 155 | raise FieldError("groupid","请选择一个分组") 156 | group_id = int(group_id) 157 | Group.objects.get(id=group_id) 158 | if not _worker.is_alive(): 159 | raise FieldError("worker_id","worker节点已经下线！") 160 | _ = DeamonTask(name=name,repo_id=repo_id,version=version,pre_build=pre_build,info=info, 161 | owner=request.user.username,run_cmd=run_cmd,worker_id=work_id,group_id=group_id) 162 | _.save() 163 | _.pure_init() 164 | return True 165 | 166 | 167 | @web_api(True) 168 | def deamon_edit(request): 169 | '''修改后台任务 170 | 参数： 171 | tid 后台任务的ID 172 | name 名称 173 | repo_id 代码库id 174 | version 代码库的版本 175 | pre_build 预构建命令 176 | info 说明 177 | run_cmd 运行命令 178 | worker_id worker的id 179 | group_id 分组的id 180 | ''' 181 | post_info = request.POST 182 | tid = post_info.get("tid").strip() 183 | name = post_info.get("name","").strip() 184 | repo_id = post_info.get("repo_id","") 185 | version = post_info.get("version","").strip() 186 | pre_build = post_info.get("pre_build","") 187 | info = post_info.get("info","") 188 | run_cmd = post_info.get("run_cmd","").strip() 189 | work_id = post_info.get("worker_id") 190 | group_id = post_info.get("group_id") 191 | tid = int(tid) 192 | deamon = DeamonTask.objects.get(tid=tid) 193 | if not name: 194 | raise FieldError("name","名称不能为空") 195 | if not repo_id: 196 | raise FieldError("repo_id","请选择一个代码库") 197 | repo_id = int(repo_id) 198 | Repo.objects.get(id=int(repo_id)) 199 | if not version: 200 | raise FieldError("version","版本不能为空！") 201 | if not run_cmd: 202 | raise FieldError("run_cmd","执行命令不能为空！") 203 | if not work_id: 204 | raise FieldError("worker_id","请选择一个worker节点") 205 | work_id = int(work_id) 206 | _worker = Worker.objects.get(id=int(work_id)) 207 | if work_id != deamon.worker_id: 208 | raise FieldError("work_id","不允许修改所在的worker节点") 209 | if not group_id: 210 | raise FieldError("groupid","请选择一个分组") 211 | if not _worker.is_alive(): 212 | raise FieldError("worker_id","worker节点已经下线！") 213 | group_id = int(group_id) 214 | Group.objects.get(id=group_id) 215 | if repo_id != deamon.repo_id or version != deamon.version or pre_build != deamon.pre_build: 216 | deamon.name = name 217 | deamon.repo_id = repo_id 218 | deamon.uptime = int(time.time()) 219 | deamon.version = version 220 | deamon.pre_build = pre_build 221 | deamon.info = info 222 | deamon.run_cmd = run_cmd 223 | deamon.group_id = group_id 224 | deamon.save() 225 | deamon.pure_init() 226 | return True 227 | else: 228 | deamon.name = name 229 | deamon.repo_id = repo_id 230 | deamon.uptime = int(time.time()) 231 | deamon.version = version 232 | deamon.pre_build = pre_build 233 | deamon.info = info 234 | deamon.run_cmd = run_cmd 235 | deamon.group_id = group_id 236 | deamon.enable() 237 | return True 238 | 239 | 240 | @web_api(True) 241 | def deamon_delete(request): 242 | '''删除后台任务 243 | 参数： 244 | tid 任务的id 245 | ''' 246 | post_info = request.POST 247 | tid = post_info.get("tid","") 248 | tid = int(tid) 249 | try: 250 | _ = DeamonTask.objects.get(tid=tid) 251 | except DeamonTask.DoesNotExist: 252 | #raise FieldError("tid","该任务已不存在！") 253 | return True 254 | else: 255 | try: 256 | _.disable(delete=True) 257 | except Exception as e : 258 | raise FieldError("tid",str(e)) 259 | return True 260 | 261 | 262 | 263 | @web_api(True) 264 | def deamon_disable(request): 265 | '''禁用后台任务 266 | 参数： 267 | tid 任务的id 268 | ''' 269 | post_info = request.POST 270 | tid = post_info.get("tid") 271 | tid = int(tid) 272 | try: 273 | _ = DeamonTask.objects.get(tid=tid) 274 | except DeamonTask.DoesNotExist: 275 | raise FieldError("tid","该任务已不存在！") 276 | else: 277 | if _.worker.is_alive(): 278 | try: 279 | _.disable() 280 | except Exception as e: 281 | raise FieldError("tid", str(e)) 282 | else: 283 | if _.get_status() != "启用": 284 | raise FieldError("tid","该任务的状态为%s，不可禁用！"%_.get_status()) 285 | _.status = -1 286 | _.save() 287 | return True 288 | 289 | 290 | 291 | @web_api(True) 292 | def deamon_enable(request): 293 | '''启用后台任务 294 | 参数： 295 | tid 任务的id 296 | ''' 297 | post_info = request.POST 298 | tid = post_info.get("tid") 299 | tid = int(tid) 300 | try: 301 | _ = DeamonTask.objects.get(tid=tid) 302 | except DeamonTask.DoesNotExist: 303 | raise FieldError("tid", "该任务已不存在！") 304 | else: 305 | if not _.worker.is_alive(): 306 | raise FieldError("tid","该任务所在节点已离线！无法启用") 307 | else: 308 | if _.get_status() != "禁用": 309 | raise FieldError("tid","该任务的状态为%s，不可启用！"%_.get_status()) 310 | _.enable() 311 | return True 312 | 313 | 314 | 315 | @web_api(True) 316 | def deamon_publog(request): 317 | '''后台任务的代码拉取日志 318 | 参数： 319 | tid 任务的id 320 | ''' 321 | post_info = request.POST 322 | tid = post_info.get("tid") 323 | try: 324 | _ = DeamonTask.objects.get(tid=tid) 325 | except DeamonTask.DoesNotExist: 326 | raise FieldError("tid","该任务不存在") 327 | else: 328 | info = PubLog.objects.filter(target_id=tid,target_type="deamon").order_by("-pubid")[:10] 329 | result = [] 330 | for i in info: 331 | result.append({"pubid":i.pubid, 332 | "addtime":i.addtime, 333 | "finishtime":i.finishtime, 334 | "stdout":i.stdout, 335 | "stderr":i.stderr, 336 | "state":i.get_state()}) 337 | return result 338 | 339 | 340 | 341 | @web_api(True) 342 | def deamon_runlog(request): 343 | '''后台任务的运行日志 344 | 参数： 345 | tid 任务的id 346 | page 页数， 默认为 1 347 | num 每页的个数，默认为10 348 | ''' 349 | post_info = request.POST 350 | tid = post_info.get("tid") 351 | page = post_info.get("page","1") 352 | num = post_info.get("num","10") 353 | page = int(page) 354 | num = int(num) 355 | all_info = RunLog.objects.filter(tid=tid,type="deamon").order_by("-rid") 356 | paginator = Paginator(all_info,num) 357 | try: 358 | info = paginator.page(page) 359 | except EmptyPage: 360 | page = 1 361 | info = paginator.page(1) 362 | result = {"data_list":[],"total_page":paginator.num_pages,"current_page":page, 363 | # "total_num":paginator.count 364 | } 365 | for i in info: 366 | item = {} 367 | item["repo_url"] = i.repo_url 368 | item["version"] = i.version 369 | item["begintime"] = i.begintime 370 | item["endtime"] = i.endtime 371 | item["status"] = i.get_status() 372 | item["stdout"] = i.stdout 373 | item["stderror"] = i.stderror 374 | result["data_list"].append(item) 375 | return result 376 | 377 | 378 | @web_api(True) 379 | def deamon_restart_now(request): 380 | '''立刻重启后台任务 381 | 参数： 382 | tid 任务的id 383 | ''' 384 | post_info = request.POST 385 | tid = post_info.get("tid") 386 | tid = int(tid) 387 | try: 388 | _ = DeamonTask.objects.get(tid=tid) 389 | except DeamonTask.DoesNotExist: 390 | raise FieldError("tid", "该任务已不存在！") 391 | else: 392 | if not _.worker.is_alive(): 393 | raise FieldError("tid","该任务所在节点已离线！无法重启任务！") 394 | else: 395 | if _.get_status() != "启用": 396 | raise FieldError("tid","该任务的状态为%s，不可重启任务！"%_.get_status()) 397 | else: 398 | _.run_now() 399 | return True 400 | 401 | -------------------------------------------------------------------------------- /cap/cap/api/views/group.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | # write by zhou 3 | 4 | from ..decorators import * 5 | from cap.models import * 6 | 7 | 8 | @web_api(True) 9 | def group_all(request): 10 | '''获取所有分组 11 | 参数： 12 | 无 13 | ''' 14 | info = Group.objects.all().order_by("-id") 15 | result = [] 16 | for i in info: 17 | result.append({"id":i.id,"name":i.name, 18 | "addtime":i.addtime}) 19 | return result 20 | 21 | 22 | @web_api(True) 23 | def group_add(request): 24 | ''' 25 | 添加分组 26 | 参数： 27 | name 分组名 28 | ''' 29 | post_info = request.POST 30 | name = post_info.get("name") 31 | info = Group.objects.filter(name=name) 32 | if info.count(): 33 | raise FieldError("name","该分组已经存在！不可重复添加") 34 | else: 35 | _ = Group(name=name) 36 | _.save() 37 | return True 38 | 39 | 40 | @web_api(True) 41 | def group_edit(request): 42 | ''' 43 | 修改分组 44 | 参数： 45 | id 46 | name 47 | ''' 48 | post_info = request.POST 49 | name = post_info.get("name") 50 | id = post_info.get("id") 51 | id = int(id) 52 | if id == Group.objects.get(name="默认").id: 53 | raise FieldError("id","默认分组不允许修改！") 54 | try: 55 | _ = Group.objects.get(id=id) 56 | except: 57 | raise FieldError("id","该分组已经被删除！无法修改。") 58 | else: 59 | query_set = Group.objects.filter(name=name) 60 | if query_set.count() > 0 and query_set[0].id != id: 61 | raise FieldError("name","该分组名已存在！") 62 | else: 63 | _.name = name 64 | _.save() 65 | return True 66 | 67 | 68 | @web_api(True) 69 | def group_delete(request): 70 | ''' 71 | 删除分组 72 | 参数： 73 | id 74 | ''' 75 | post_info = request.POST 76 | id = post_info.get("id","") 77 | id = int(id) 78 | if id == Group.objects.get(name="默认").id: 79 | raise FieldError("id","默认分组不允许删除！") 80 | _ = Group.objects.filter(id=id) 81 | if not _.count(): 82 | raise FieldError("id","该分组不存在！") 83 | if CronTask.objects.filter(group_id=id).count() or DeamonTask.objects.filter(group_id=id).count(): 84 | raise FieldError("id","该分组正在被使用，不允许删除！") 85 | else: 86 | _.delete() 87 | return True 88 | -------------------------------------------------------------------------------- /cap/cap/api/views/login.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | # write by zhou 3 | 4 | from ..decorators import * 5 | from cap.models import * 6 | from django.contrib.admin.models import User 7 | from django.contrib.auth.views import auth_login,auth_logout 8 | from django.contrib.auth import authenticate,login,logout 9 | 10 | 11 | @web_api() 12 | def login(request): 13 | ''' 14 | 用户登录接口 15 | 测试用户： admin gc895316 16 | 参数说明： 17 | username 18 | password 19 | ''' 20 | post_info = request.POST 21 | username = post_info.get("username",'') 22 | password = post_info.get('password','') 23 | if not username: 24 | raise FieldError("username","username不能为空") 25 | if not password: 26 | raise FieldError("password","password不能为空") 27 | try: 28 | user = User.objects.get(username=username) 29 | except: 30 | raise FieldError("username","用户不存在") 31 | else: 32 | if user.check_password(password): 33 | request.apisession["uid"] = user.id 34 | return True 35 | else: 36 | raise FieldError("password","密码不正确") 37 | 38 | 39 | @web_api() 40 | def logout(request): 41 | ''' 42 | 用户退出登录 43 | 参数说明： 44 | 无 45 | 46 | ''' 47 | if request.user: 48 | #auth_logout(request) 49 | del request.apisession["uid"] 50 | return True -------------------------------------------------------------------------------- /cap/cap/api/views/my.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | # write by zhou 3 | 4 | from ..decorators import * 5 | 6 | 7 | @web_api(True) 8 | def get_my_uid(request): 9 | '''获取当前登录用户的id 10 | 参数： 11 | 无 12 | ''' 13 | return request.user.id 14 | 15 | 16 | 17 | @web_api(True) 18 | def get_my_info(request): 19 | '''获取当前登录用户的详细信息 20 | 参数： 21 | 无 22 | ''' 23 | return {"id":request.user.id, 24 | "username":request.user.username, 25 | } 26 | 27 | 28 | @web_api(True) 29 | def change_my_password(request): 30 | '''修改我的密码 31 | 参数： 32 | password_old 老密码 33 | password 新密码 34 | password_again 再次输入新密码 35 | ''' 36 | post_info = request.POST 37 | password = post_info.get("password","") 38 | old_password = post_info.get("password_old","") 39 | password_again = post_info.get("password_again","") 40 | user = request.user 41 | if not user.check_password(old_password): 42 | raise FieldError("password_old","旧密码不正确！") 43 | if user.username == "admin": 44 | raise FieldError("username","admin用户的密码禁止修改") 45 | if not password: 46 | raise FieldError("password","密码不能为空") 47 | if len(password)<6: 48 | raise FieldError("password","密码长度不能小于6位") 49 | if password != password_again: 50 | raise FieldError("password_again","两次输入的密码不一致") 51 | user.set_password(password) 52 | user.save() 53 | return True 54 | -------------------------------------------------------------------------------- /cap/cap/api/views/repo.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | # write by zhou 3 | 4 | from ..decorators import * 5 | from cap.models import * 6 | 7 | 8 | 9 | @web_api(True) 10 | def repo_all(request): 11 | ''' 12 | 获取所有代码库 13 | 参数： 14 | ''' 15 | total_count = Repo.objects.all().count() 16 | all = Repo.objects.all().order_by("-id") 17 | result = [] 18 | for i in all: 19 | result.append({"id":i.id, 20 | "addtime":i.addtime, 21 | "type":i.type, 22 | "repo_url":i.repo_url, 23 | "user":i.user, 24 | "password":i.password}) 25 | return {"total_count":total_count,"repo_list":result} 26 | 27 | 28 | @web_api(True) 29 | def repo_add(request): 30 | ''' 31 | 增加代码库 32 | 参数： 33 | type 类型 ， 必传， int， 1 svn 2 git 34 | repo_url 代码仓库地址 ，必传，https://开头 或者svn://开头 35 | user 代码仓库用户， 必传 36 | password 代码仓库密码 ，必传， 37 | ''' 38 | post_info = request.POST 39 | type = post_info.get("type","") 40 | type = int(type) 41 | assert type in (1,2) 42 | repo_url = post_info.get("repo_url","") 43 | if not repo_url: 44 | raise FieldError("repo_url","repo_url不能为空") 45 | user = post_info.get("user","") 46 | password = post_info.get("password","") 47 | if user: 48 | if not password: 49 | raise FieldError("password","password不能为空！") 50 | if password: 51 | if not user: 52 | raise FieldError('user',"user不能为空！") 53 | try: 54 | Repo.objects.get(repo_url=repo_url) 55 | except: 56 | repo = Repo(type=int(type),repo_url=repo_url,user=user,password=password) 57 | repo.save() 58 | repo.pure_init() 59 | return repo.id 60 | else: 61 | raise FieldError("repo_url","该代码仓库已存在！请勿重复添加！") 62 | 63 | 64 | @web_api(True) 65 | def repo_edit(request): 66 | ''' 67 | 更新代码库 68 | 参数： 69 | id 代码库的id 必传, int 70 | type 类型 ， 必传， int， 1 svn 2 git 71 | repo_url 代码仓库地址 ，必传，https://开头 或者svn://开头 72 | user 代码仓库用户， 必传 73 | password 代码仓库密码 ，必传， 74 | ''' 75 | post_info = request.POST 76 | id = post_info.get("id","") 77 | id = int(id) 78 | repo = Repo.objects.get(id=id) 79 | type = post_info.get("type","") 80 | type = int(type) 81 | assert type in (1,2) 82 | repo_url = post_info.get("repo_url","") 83 | try: 84 | _ = Repo.objects.get(repo_url=repo_url) 85 | assert _.id != id 86 | except: 87 | pass 88 | else: 89 | raise FieldError("repo_url","repo_url和现有的代码库重复！") 90 | if not repo_url: 91 | raise FieldError("repo_url","repo_url不能为空") 92 | user = post_info.get("user","") 93 | password = post_info.get("password","") 94 | if user: 95 | if not password: 96 | raise FieldError("password","password不能为空！") 97 | if password: 98 | if not user: 99 | raise FieldError('user',"user不能为空！") 100 | repo.type = type 101 | repo.repo_url = repo_url 102 | repo.user = user 103 | repo.password = password 104 | repo.save() 105 | repo.pure_init() 106 | return True 107 | 108 | 109 | @web_api(True) 110 | def repo_delete(request): 111 | ''' 112 | 删除代码库 113 | 参数： 114 | id 代码库的id 必传, int 115 | ''' 116 | post_info = request.POST 117 | id = post_info.get("id") 118 | id = int(id) 119 | try: 120 | repo = Repo.objects.get(id=id) 121 | except Repo.DoesNotExist: 122 | return True 123 | else: 124 | if CronTask.objects.filter(repo_id=id).count() or DeamonTask.objects.filter(repo_id=id).count(): 125 | raise FieldError("id","该代码库正在被使用，无法删除！") 126 | else: 127 | repo.disable() 128 | repo.delete() 129 | return True 130 | 131 | 132 | @web_api(True) 133 | def repo_commit_log(request): 134 | ''' 135 | 查询代码库的所有提交记录 136 | 参数： 137 | id 138 | ''' 139 | post_info = request.POST 140 | id = post_info.get("id") 141 | try: 142 | repo = Repo.objects.get(id=id) 143 | except Repo.DoesNotExist: 144 | return [] 145 | else: 146 | repo_id = repo.id 147 | commit_logs = RepoCommitLog.objects.filter(repo_id=repo_id).order_by("-committime")[:100] 148 | result = [] 149 | for i in commit_logs: 150 | result.append({"id":i.id,"ver":i.ver, 151 | "author":i.author, 152 | "committime":i.committime, 153 | "message":i.message}) 154 | return result 155 | 156 | 157 | @web_api(True) 158 | def repo_monitor_log(reqeust): 159 | ''' 160 | 获取代码库的更新日志 161 | ''' 162 | post_info = reqeust.POST 163 | id = post_info.get("id") 164 | try: 165 | repo = Repo.objects.get(id=id) 166 | except Repo.DoesNotExist: 167 | return '' 168 | else: 169 | repo_id = repo.id 170 | try: 171 | track_log = RepoMonitorLog.objects.filter(repo_id=repo_id)[0].log 172 | except: 173 | track_log = '' 174 | return track_log 175 | 176 | 177 | -------------------------------------------------------------------------------- /cap/cap/api/views/user.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | # write by zhou 3 | 4 | from ..decorators import * 5 | from cap.models import * 6 | from django.contrib.admin.models import User 7 | 8 | 9 | 10 | @web_api(True) 11 | def user_list(request): 12 | '''用户列表 13 | 所需参数： 14 | 无 15 | 返回数据说明： 16 | id 用户id 17 | username 用户名 18 | addtime 用户创建时间 19 | is_admin 是否是超级用户 20 | ''' 21 | post_info = request.POST 22 | total_count = User.objects.all().order_by("-id").count() 23 | info = User.objects.all().order_by("-id") 24 | result = [] 25 | for i in info: 26 | print dir(i.date_joined) 27 | result.append({"id":i.id, 28 | "username":i.username, 29 | "addtime":int(i.date_joined.strftime("%s")), 30 | "is_admin":i.is_superuser, 31 | }) 32 | return {"total_count":total_count,"user_list":result} 33 | 34 | 35 | 36 | @web_api(True) 37 | def user_add(request): 38 | '''添加用户 39 | 所需参数： 40 | username 用户名 41 | password 密码 42 | password_again 再次输入密码 43 | ''' 44 | post_info = request.POST 45 | username = post_info.get("username","") 46 | password = post_info.get("password","") 47 | password_again = post_info.get("password_again","") 48 | if not username: 49 | raise FieldError("username","用户名不能为空") 50 | try: 51 | User.objects.get(username=username) 52 | except: 53 | pass 54 | else: 55 | raise FieldError("username", '该用户已存在！') 56 | 57 | if not password: 58 | raise FieldError("password","密码不能为空") 59 | if len(password) < 6: 60 | raise FieldError("password","密码长度不能小于6位") 61 | if password != password_again: 62 | raise FieldError("password_again","两次输入的密码不一致") 63 | _p = User(username=username,password='') 64 | _p.save() 65 | _p.set_password(password) 66 | _p.save() 67 | return True 68 | 69 | 70 | @web_api(True) 71 | def user_resetpwd(request): 72 | '''添加用户 73 | 所需参数： 74 | username 用户名 75 | password 密码 76 | password_again 再次输入密码 77 | ''' 78 | post_info = request.POST 79 | username = post_info.get("username","") 80 | password = post_info.get("password","") 81 | password_again = post_info.get("password_again","") 82 | user = User.objects.get(username=username) 83 | if request.user.username != "admin" and request.user.username!=user.username: 84 | raise FieldError("username","非admin用户不能修改其他用户的密码！") 85 | if user.username == "admin": 86 | raise FieldError("username","禁止修改admin用户的密码") 87 | if not password: 88 | raise FieldError("password","密码不能为空") 89 | if len(password)<6: 90 | raise FieldError("password","密码长度不能小于6位") 91 | if password != password_again: 92 | raise FieldError("password_again","两次输入的密码不一致") 93 | user.set_password(password) 94 | user.save() 95 | return True 96 | 97 | 98 | @web_api(True) 99 | def user_delete(request): 100 | ''' 101 | 删除用户 102 | 所需参数： 103 | username 104 | ''' 105 | post_info = request.POST 106 | username = post_info.get("username","") 107 | user = User.objects.get(username=username) 108 | if request.user.username != "admin": 109 | raise FieldError("username","非admin用户不能删除用户！") 110 | if user.username == "admin": 111 | raise FieldError("username","禁止删除admin用户！") 112 | user.delete() 113 | return True 114 | 115 | 116 | 117 | -------------------------------------------------------------------------------- /cap/cap/api/views/worker.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | # write by zhou 3 | from ..decorators import * 4 | from cap.models import * 5 | from django.core.paginator import Paginator,EmptyPage,InvalidPage 6 | 7 | 8 | @web_api(True) 9 | def worker_all(request): 10 | '''获取所有的worker (此部分不会太多，不用考虑分页) 11 | 参数： 12 | 无 13 | 返回值说明： 14 | id worker的ID 15 | ip worker的IP 16 | addtime worker的加入时间 17 | is_alive worker当前是否存活 18 | last_connect_time worker上次活跃时间 19 | total_cpu worker cpu个数 20 | total_mem worker 内存大小(MB) 21 | ''' 22 | all_info = Worker.objects.all().order_by("-id") 23 | result = [] 24 | for i in all_info: 25 | result.append({"id":i.id, 26 | "ip":i.ip, 27 | "addtime":i.addtime, 28 | "is_alive":i.is_alive(), 29 | "last_connect_time":i.heartbeat, 30 | "total_cpu":i.total_cpu, 31 | "total_mem":i.total_mem}) 32 | return result 33 | 34 | 35 | @web_api(True) 36 | def worker_info(request): 37 | '''获取单个worker的信息 38 | 参数： 39 | id worker的id 40 | 返回值说明： 41 | id worker的ID 42 | ip worker的IP 43 | addtime worker的加入时间 44 | is_alive worker当前是否存活 45 | last_connect_time worker上次活跃时间 46 | total_cpu worker cpu个数 47 | total_mem worker内存大小(MB) 48 | ''' 49 | post_info = request.POST 50 | id = post_info.get("id") 51 | id = int(id) 52 | worker = Worker.objects.get(id=id) 53 | return {"id":worker.id, 54 | "ip":worker.ip, 55 | "addtime":worker.addtime, 56 | "is_alive":worker.is_alive(), 57 | "last_connect_time":worker.heartbeat, 58 | "total_cpu":worker.total_cpu, 59 | "total_mem":worker.total_mem} 60 | 61 | 62 | @web_api(True) 63 | def worker_cpu_mem_log(request): 64 | '''获取单个worker的cpu内存负载信息 65 | 参数： 66 | id worker的id 67 | 68 | ''' 69 | post_info = request.POST 70 | id = post_info.get("id") 71 | id = int(id) 72 | worker = Worker.objects.get(id=id) 73 | info = WorkerCpuMemLog.objects.filter(work_id=worker.id).order_by("-id")[:200] 74 | result = [] 75 | for i in info: 76 | result.append({"time":i.addtime, 77 | "cpu_percent":i.cpu_percent, 78 | "mem_percent":i.mem_percent}) 79 | result.reverse() 80 | return result -------------------------------------------------------------------------------- /cap/cap/core_api/__init__.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | # write by zhou 3 | from sites import site -------------------------------------------------------------------------------- /cap/cap/core_api/sites.py: -------------------------------------------------------------------------------- 1 | # coding:utf-8 2 | # write by zhou 3 | 4 | from django.conf.urls import include, url, patterns 5 | 6 | 7 | class CoreApi(object): 8 | def __init__(self, name="core_api", app_name="core_api"): 9 | self.name = name 10 | self.app_name = app_name 11 | 12 | def get_urls(self): 13 | return patterns("cap.core_api", 14 | url(r"^work_heartbeat/$", "views.worker.worker_heartbeat"), 15 | 16 | ) 17 | 18 | @property 19 | def urls(self): 20 | return self.get_urls(), self.app_name, self.name 21 | 22 | @urls.setter 23 | def urls(self, value): 24 | pass 25 | 26 | @urls.deleter 27 | def urls(self): 28 | pass 29 | 30 | 31 | site = CoreApi() 32 | -------------------------------------------------------------------------------- /cap/cap/core_api/views/__init__.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | # write by zhou -------------------------------------------------------------------------------- /cap/cap/core_api/views/worker.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | # write by zhou 3 | from django.http import HttpResponse 4 | import json 5 | from cap.models import * 6 | from django.conf import settings 7 | 8 | buff = {} 9 | 10 | def worker_heartbeat(request): 11 | get_info = request.GET 12 | post_info = request.POST 13 | ip = request.GET.get("ip","") or request.META["REMOTE_ADDR"] 14 | work_dir = get_info.get("work_dir","") 15 | num = get_info.get("num","") 16 | num = int(num) 17 | worker = Worker.worker_heartbeat(ip,work_dir) 18 | if num == 1 or not buff.has_key(ip): 19 | buff[ip] = ip 20 | worker.pure_init() 21 | all_cron_task = CronTask.objects.filter(worker_id=worker.id) 22 | for i in all_cron_task: 23 | if i.status == 1: 24 | i.enable() 25 | all_deamon_task = DeamonTask.objects.filter(worker_id=worker.id) 26 | for i in all_deamon_task: 27 | if i.status == 1: 28 | i.enable() 29 | if ip == settings.HOST: 30 | all_repo = Repo.objects.all() 31 | for i in all_repo: 32 | i.pure_init() 33 | else: 34 | pass 35 | return HttpResponse("success") 36 | 37 | 38 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /cap/cap/rpc.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | # write by zhou 3 | 4 | #coding:utf-8 5 | # write by zhou 6 | 7 | 8 | 9 | class Ping(object): 10 | 11 | def __init__(self, ip): 12 | import xmlrpclib 13 | self.ip = ip 14 | self.server = xmlrpclib.ServerProxy("http://%s:9913" % ip, allow_none=True) 15 | def ping(self): 16 | try: 17 | self.server.ping() 18 | return True 19 | except: 20 | raise Exception("[%s]:Exception:%s" % (self.ip, "该节点已经离线")) 21 | 22 | 23 | class Cache(object): 24 | 25 | def __init__(self, ip): 26 | import xmlrpclib 27 | self.ip = ip 28 | self.server = xmlrpclib.ServerProxy("http://%s:9913" % ip, allow_none=True) 29 | 30 | def set(self,key,value): 31 | try: 32 | self.server.cache_set(key,value) 33 | return True 34 | except Exception as e : 35 | raise Exception("[%s]:Exception:%s" % (self.ip, str(e))) 36 | 37 | def get(self,key): 38 | try: 39 | return self.server.cache_get(key) 40 | except Exception as e : 41 | raise Exception("[%s]:Exception:%s" % (self.ip, str(e))) 42 | 43 | def delete(self,key): 44 | try: 45 | return self.server.cache_del(key) 46 | except Exception as e : 47 | raise Exception("[%s]:Exception:%s" % (self.ip, str(e))) 48 | 49 | 50 | class Cron(object): 51 | def __init__(self,ip): 52 | import xmlrpclib 53 | self.ip = ip 54 | self.server = xmlrpclib.ServerProxy("http://%s:9913"%ip,allow_none=True) 55 | 56 | def get(self,key): 57 | try: 58 | result = self.server.cron_get(key) 59 | if isinstance(result,(list,tuple)): 60 | raise Exception(result[1]) 61 | else: 62 | return result 63 | except Exception as e : 64 | raise Exception("[%s]:Exception:%s" % (self.ip, str(e))) 65 | 66 | 67 | def set(self,key,cron_mode,cron_function,cron_callback,cron_errback,stdout_callback, 68 | stderr_callback): 69 | import xmlrpclib,cloudpickle 70 | #cron_function.__module__ = "__main__" 71 | cron_function = cloudpickle.dumps(cron_function) 72 | cron_callback = cloudpickle.dumps(cron_callback) 73 | cron_errback = cloudpickle.dumps(cron_errback) 74 | stdout_callback = cloudpickle.dumps(stdout_callback) 75 | stderr_callback = cloudpickle.dumps(stderr_callback) 76 | try: 77 | result = self.server.cron_set(key,cron_mode,xmlrpclib.Binary(cron_function), 78 | xmlrpclib.Binary(cron_callback),xmlrpclib.Binary(cron_errback), 79 | xmlrpclib.Binary(stdout_callback),xmlrpclib.Binary(stderr_callback)) 80 | if isinstance(result,(list,tuple)): 81 | raise Exception(result[1]) 82 | else: 83 | return result 84 | except Exception as e : 85 | raise Exception("[%s]:Exception:%s" % (self.ip, str(e))) 86 | 87 | def delete(self,key): 88 | try: 89 | result = self.server.cron_del(key) 90 | if isinstance(result,(list,tuple)): 91 | raise Exception(result[1]) 92 | else: 93 | return result 94 | except Exception as e : 95 | raise Exception("[%s]:Exception:%s" % (self.ip, str(e))) 96 | 97 | def is_running(self,key): 98 | try: 99 | result = self.server.cron_is_running(key) 100 | if isinstance(result,(list,tuple)): 101 | raise Exception(result[1]) 102 | else: 103 | return result 104 | except Exception as e : 105 | raise Exception("[%s]:Exception:%s" % (self.ip, str(e))) 106 | 107 | def run_now(self,key): 108 | try: 109 | result = self.server.cron_run_now(key) 110 | if isinstance(result,(list,tuple)): 111 | raise Exception(result[1]) 112 | else: 113 | return result 114 | except Exception as e : 115 | raise Exception("[%s]:Exception:%s" % (self.ip, str(e))) 116 | 117 | 118 | class Deamon(object): 119 | def __init__(self,ip): 120 | import xmlrpclib 121 | self.ip = ip 122 | self.server = xmlrpclib.ServerProxy("http://%s:9913"%ip,allow_none=True) 123 | 124 | def set(self,key,function,callback,errback,stdout_callback, 125 | stderr_callback): 126 | import xmlrpclib,cloudpickle 127 | function=cloudpickle.dumps(function) 128 | callback=cloudpickle.dumps(callback) 129 | errback=cloudpickle.dumps(errback) 130 | stdout_callback=cloudpickle.dumps(stdout_callback) 131 | stderr_callback=cloudpickle.dumps(stderr_callback) 132 | try: 133 | result = self.server.deamon_set(key,xmlrpclib.Binary(function), 134 | xmlrpclib.Binary(callback),xmlrpclib.Binary(errback), 135 | xmlrpclib.Binary(stdout_callback),xmlrpclib.Binary(stderr_callback)) 136 | if isinstance(result, (list, tuple)): 137 | raise Exception(result[1]) 138 | else: 139 | return result 140 | except Exception as e: 141 | raise Exception("[%s]:Exception:%s" % (self.ip, str(e))) 142 | 143 | 144 | def delete(self,key): 145 | try: 146 | result = self.server.deamon_del(key) 147 | if isinstance(result, (list, tuple)): 148 | raise Exception(result[1]) 149 | else: 150 | return result 151 | except Exception as e: 152 | raise Exception("[%s]:Exception:%s" % (self.ip, str(e))) 153 | 154 | 155 | def is_running(self,key): 156 | try: 157 | result = self.server.deamon_is_running(key) 158 | if isinstance(result, (list, tuple)): 159 | raise Exception(result[1]) 160 | else: 161 | return result 162 | except Exception as e: 163 | raise Exception("[%s]:Exception:%s" % (self.ip, str(e))) 164 | 165 | def get(self,key): 166 | try: 167 | result = self.server.deamon_get(key) 168 | if isinstance(result, (list, tuple)): 169 | raise Exception(result[1]) 170 | else: 171 | return result 172 | except Exception as e: 173 | raise Exception("[%s]:Exception:%s" % (self.ip, str(e))) 174 | 175 | 176 | class Task(object): 177 | def __init__(self, ip): 178 | import xmlrpclib 179 | self.ip = ip 180 | self.server = xmlrpclib.ServerProxy("http://%s:9913" % ip, allow_none=True) 181 | 182 | def set(self, key, function, callback, errback, stdout_callback, 183 | stderr_callback,timeout_callback=lambda *x:None,timeout=60): 184 | import xmlrpclib,cloudpickle 185 | function = cloudpickle.dumps(function) 186 | callback = cloudpickle.dumps(callback) 187 | errback = cloudpickle.dumps(errback) 188 | stdout_callback = cloudpickle.dumps(stdout_callback) 189 | stderr_callback = cloudpickle.dumps(stderr_callback) 190 | timeout_callback = cloudpickle.dumps(timeout_callback) 191 | try: 192 | result = self.server.task_set(key, xmlrpclib.Binary(function), 193 | xmlrpclib.Binary(callback), xmlrpclib.Binary(errback), 194 | xmlrpclib.Binary(stdout_callback), xmlrpclib.Binary(stderr_callback), 195 | xmlrpclib.Binary(timeout_callback),timeout) 196 | if isinstance(result,(list,tuple)): 197 | raise Exception(result[1]) 198 | else: 199 | return result 200 | except Exception as e : 201 | raise Exception("[%s]:Exception:%s" % (self.ip, str(e))) 202 | 203 | def delete(self, key): 204 | try: 205 | result = self.server.task_del(key) 206 | if isinstance(result,(list,tuple)): 207 | raise Exception(result[1]) 208 | else: 209 | return result 210 | except Exception as e : 211 | raise Exception("[%s]:Exception:%s" % (self.ip, str(e))) 212 | 213 | def is_running(self, key): 214 | try: 215 | result = self.server.task_is_running(key) 216 | if isinstance(result,(list,tuple)): 217 | raise Exception(result[1]) 218 | else: 219 | return result 220 | except Exception as e : 221 | raise Exception("[%s]:Exception:%s" % (self.ip, str(e))) 222 | 223 | def get(self, key): 224 | try: 225 | result = self.server.task_get(key) 226 | if isinstance(result,(list,tuple)): 227 | raise Exception(result[1]) 228 | else: 229 | return result 230 | except Exception as e : 231 | raise Exception("[%s]:Exception:%s" % (self.ip, str(e))) 232 | 233 | if __name__ == "__main__": 234 | import time 235 | task = Task("192.168.8.185") 236 | def test(): 237 | import redis 238 | conn=redis.Redis("192.168.8.185",6379) 239 | conn.incr("testabc",10) 240 | 241 | 242 | _task = Task("192.168.8.185") 243 | 244 | def incr(): 245 | import redis 246 | conn=redis.Redis("192.168.8.185",6379) 247 | conn.incr("zhou",10) 248 | import time 249 | time.sleep(2) 250 | _task = Task("192.168.8.185") 251 | _task.set("testabcd",test,lambda x:None,lambda x:None,lambda *x:None,lambda *x:None) 252 | 253 | # deamon.set("test",incr,lambda x:None,lambda x:None,lambda *x:None,lambda *x:None) 254 | # while 1: 255 | # time.sleep(0.1) 256 | # print deamon.is_running("test") 257 | task.set("test",incr,lambda x:None,lambda x:None,lambda *x:None,lambda *x:None) 258 | while 1: 259 | time.sleep(0.3) 260 | print task.is_running("test") 261 | # print cron.set("test","* * * * *",incr,lambda X:None,lambda x:None,lambda *x:None,lambda *x:None) 262 | # print cron.get("test") 263 | # import time 264 | # while 1: 265 | # time.sleep(1) 266 | # print cron.is_running("test") 267 | # print cron.is_running("test") 268 | 269 | -------------------------------------------------------------------------------- /cap/cap/settings.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | # Django settings for cap project. 3 | import os 4 | #import cap.urls 5 | #print [cap.urls] 6 | print __file__ 7 | 8 | 9 | DEBUG = True 10 | TEMPLATE_DEBUG = DEBUG 11 | LOGIN_URL="/login/" 12 | LANGUAGE_CODE="zh-CN" 13 | if getattr(os,"config",None): 14 | config = os.config 15 | else: 16 | config = ["192.168.14.90",3306,"cap_test","spider","123456"] 17 | 18 | WORK_DIR = os.work_dir 19 | HOST = os.host 20 | DATABASES = { 21 | 'default': { 22 | 'ENGINE': 'django.db.backends.mysql', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. 23 | 'NAME': config[2], # Or path to database file if using sqlite3. 24 | 'USER': config[3], # Not used with sqlite3. 25 | 'PASSWORD': config[4], # Not used with sqlite3. 26 | 'HOST': config[0], # Set to empty string for localhost. Not used with sqlite3. 27 | 'PORT': '%s'%config[1], # Set to empty string for default. Not used with sqlite3. 28 | }, 29 | } 30 | print DATABASES 31 | 32 | 33 | # Hosts/domain names that are valid for this site; required if DEBUG is False 34 | # See https://docs.djangoproject.com/en/1.4/ref/settings/#allowed-hosts 35 | ALLOWED_HOSTS = ["*"] 36 | 37 | # Local time zone for this installation. Choices can be found here: 38 | # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name 39 | # although not all choices may be available on all operating systems. 40 | # In a Windows environment this must be set to your system time zone. 41 | TIME_ZONE = 'Asia/Shanghai' 42 | 43 | # Language code for this installation. All choices can be found here: 44 | # http://www.i18nguy.com/unicode/language-identifiers.html 45 | 46 | 47 | SITE_ID = 1 48 | 49 | # If you set this to False, Django will make some optimizations so as not 50 | # to load the internationalization machinery. 51 | USE_I18N = True 52 | 53 | # If you set this to False, Django will not format dates, numbers and 54 | # calendars according to the current locale. 55 | USE_L10N = True 56 | 57 | # If you set this to False, Django will not use timezone-aware datetimes. 58 | USE_TZ = False 59 | 60 | # Absolute filesystem path to the directory that will hold user-uploaded files. 61 | # Example: "/home/media/media.lawrence.com/media/" 62 | MEDIA_ROOT = '' 63 | 64 | # URL that handles the media served from MEDIA_ROOT. Make sure to use a 65 | # trailing slash. 66 | # Examples: "http://media.lawrence.com/media/", "http://example.com/media/" 67 | MEDIA_URL = '' 68 | 69 | # Absolute path to the directory static files should be collected to. 70 | # Don't put anything in this directory yourself; store your static files 71 | # in apps' "static/" subdirectories and in STATICFILES_DIRS. 72 | # Example: "/home/media/media.lawrence.com/static/" 73 | STATIC_ROOT ='' 74 | 75 | # URL prefix for static files. 76 | # Example: "http://media.lawrence.com/static/" 77 | STATIC_URL = '/abcdefgh/' 78 | 79 | # Additional locations of static files 80 | STATICFILES_DIRS = ( 81 | # Put strings here, like "/home/html/static" or "C:/www/django/static". 82 | # Always use forward slashes, even on Windows. 83 | # Don't forget to use absolute paths, not relative paths. 84 | os.path.join(os.path.dirname(__file__), '..', 'static').replace('\\','/'), 85 | ) 86 | 87 | # List of finder classes that know how to find static files in 88 | # various locations. 89 | STATICFILES_FINDERS = ( 90 | 'django.contrib.staticfiles.finders.FileSystemFinder', 91 | 'django.contrib.staticfiles.finders.AppDirectoriesFinder', 92 | # 'django.contrib.staticfiles.finders.DefaultStorageFinder', 93 | ) 94 | 95 | # Make this unique, and don't share it with anybody. 96 | SECRET_KEY = '))ye65id1n#94^w68tpii41gq__&)gi&)&mlw55sgpi7frpm36' 97 | 98 | # List of callables that know how to import templates from various sources. 99 | TEMPLATE_LOADERS = ( 100 | 'django.template.loaders.filesystem.Loader', 101 | 'django.template.loaders.app_directories.Loader', 102 | # 'django.template.loaders.eggs.Loader', 103 | ) 104 | 105 | MIDDLEWARE_CLASSES = ( 106 | 'django.middleware.common.CommonMiddleware', 107 | 'django.contrib.sessions.middleware.SessionMiddleware', 108 | 'django.middleware.csrf.CsrfViewMiddleware', 109 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 110 | 'django.contrib.messages.middleware.MessageMiddleware', 111 | # Uncomment the next line for simple clickjacking protection: 112 | # 'django.middleware.clickjacking.XFrameOptionsMiddleware', 113 | ) 114 | 115 | ROOT_URLCONF = 'cap.urls' 116 | 117 | # Python dotted path to the WSGI application used by Django's runserver. 118 | WSGI_APPLICATION = 'cap.wsgi.application' 119 | 120 | TEMPLATE_DIRS = (os.path.join(os.path.dirname(__file__), '..', 'templates').replace('\\','/'),) 121 | 122 | INSTALLED_APPS = ( 123 | 'django.contrib.auth', 124 | 'django.contrib.contenttypes', 125 | 'django.contrib.sessions', 126 | 'django.contrib.sites', 127 | 'django.contrib.messages', 128 | 'django.contrib.staticfiles', 129 | # Uncomment the next line to enable the admin: 130 | 'django.contrib.admin', 131 | # Uncomment the next line to enable admin documentation: 132 | # 'django.contrib.admindocs', 133 | 'cap', #主项目 134 | "cap.api" 135 | 136 | ) 137 | 138 | 139 | TEMPLATE_CONTEXT_PROCESSORS = ( 140 | "django.contrib.auth.context_processors.auth", 141 | "django.core.context_processors.debug", 142 | "django.core.context_processors.i18n", 143 | "django.core.context_processors.media", 144 | "django.core.context_processors.static", 145 | "django.contrib.messages.context_processors.messages", 146 | "django.core.context_processors.request", 147 | ) 148 | # A sample logging configuration. The only tangible logging 149 | # performed by this configuration is to send an email to 150 | # the site admins on every HTTP 500 error when DEBUG=False. 151 | # See http://docs.djangoproject.com/en/dev/topics/logging for 152 | # more details on how to customize your logging configuration. 153 | LOGGING = { 154 | 'version': 1, 155 | 'disable_existing_loggers': True, 156 | 'formatters':{ 157 | 'simple': { 158 | 'format': '%(levelname)s %(message)s' 159 | }, 160 | }, 161 | 'filters': { 162 | 'require_debug_false': { 163 | '()': 'django.utils.log.RequireDebugFalse' 164 | } 165 | }, 166 | 'handlers': { 167 | 'mail_admins': { 168 | 'level': 'ERROR', 169 | "filters":[], 170 | 'class': 'django.utils.log.AdminEmailHandler', 171 | 'include_html':True 172 | }, 173 | 'console':{ 174 | 'level':'DEBUG', 175 | 'class':'logging.StreamHandler', 176 | } 177 | }, 178 | 'loggers': { 179 | 'django.request': { 180 | 'handlers': ['mail_admins'], 181 | 'level': 'ERROR', 182 | 'propagate': True, 183 | }, 184 | 'django.db.backends': { 185 | 'handlers': ['console'], 186 | 'propagate': True, 187 | 'level':'DEBUG', 188 | } 189 | } 190 | } 191 | SESSION_COOKIE_AGE = 60 * 60 * 24 * 7 * 999 # Age of cookie, in seconds (default: 2 weeks). 192 | -------------------------------------------------------------------------------- /cap/cap/urls.py: -------------------------------------------------------------------------------- 1 | # coding:utf-8 2 | from django.conf.urls import patterns, include, url 3 | 4 | # Uncomment the next two lines to enable the admin: 5 | from django.contrib import admin 6 | from django.contrib.auth.views import login, logout 7 | from django.http import HttpResponse, HttpResponseRedirect 8 | import core_api 9 | import api 10 | import views 11 | import os 12 | 13 | admin.autodiscover() 14 | 15 | urlpatterns = patterns('', 16 | # Examples: 17 | # url(r'^$', 'cap.views.home', name='home'), 18 | # url(r'^cap/', include('cap.foo.urls')), 19 | 20 | # Uncomment the admin/doc line below to enable admin documentation: 21 | (r"^robots\.txt$", 22 | lambda request: HttpResponse("User-agent: *\nDisallow: /", mimetype="text/plain")), 23 | # robots 24 | url(r'^admin/doc/', include('django.contrib.admindocs.urls')), 25 | url(r"^login/$", login, {"template_name": "login.html"}), 26 | url(r"^logout/$", logout, {"next_page": "/"}), 27 | # url(r'^static/(?P.*)$', 'django.views.static.serve', 28 | # {'document_root': settings.STATICFILES_DIRS[0], 'show_indexes': settings.DEBUG}), 29 | # Uncomment the next line to enable the admin: 30 | #url(r"^$", "cap.views.home"), 31 | url(r'^admin/', include(admin.site.urls)), # admin页面 32 | url(r"^core_api/",include(core_api.site.urls)), # core_api 33 | url(r"^environ",lambda x:HttpResponse(str(os.environ))), 34 | url(r"^zhou",lambda x:HttpResponse(str(getattr(os,'zhou')))), 35 | url(r"^api/",include(api.site.urls)), 36 | (r"^(?P.*)$",views.front_view), 37 | ) 38 | -------------------------------------------------------------------------------- /cap/cap/utils.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | __author__ = 'Administrator' 3 | from django.core.paginator import Paginator, InvalidPage, EmptyPage 4 | from django.shortcuts import render_to_response 5 | from django.template import RequestContext 6 | from django.http import HttpResponse,HttpResponseRedirect 7 | from django.contrib.auth.models import Group 8 | import socket 9 | import os 10 | import urllib 11 | def sendrtx(title,message,*args): 12 | for name in args: 13 | try: 14 | if type(title)==unicode: 15 | title=title.encode("utf-8") 16 | if type(message)==unicode: 17 | message=message.encode("utf-8") 18 | url="http://oa.gongchang.cn/sendtortx.php?receiver=%s¬ifytitle"\ 19 | "=%s¬ifymsg=%s"%(name,title,message) 20 | urllib.urlopen(url) 21 | except Exception as e : 22 | pass 23 | def range_list_of(page_object): 24 | num_pages = page_object.paginator.num_pages 25 | if num_pages <= 5: 26 | range_list = range(1, num_pages+1) 27 | else: 28 | if page_object.number < 3: 29 | range_list = range(1,6) 30 | elif page_object.number > (num_pages-2): 31 | range_list = range(num_pages-4, num_pages+1) 32 | else: 33 | range_list = range(page_object.number-2, page_object.number+3) 34 | return range_list 35 | def encrypt(key, s): 36 | b = bytearray(str(s).encode("gbk")) 37 | n = len(b) # 求出 b 的字节数 38 | c = bytearray(n*2) 39 | j = 0 40 | for i in range(0, n): 41 | b1 = b[i] 42 | b2 = b1 ^ key # b1 = b2^ key 43 | c1 = b2 % 16 44 | c2 = b2 // 16 # b2 = c2*16 + c1 45 | c1 = c1 + 65 46 | c2 = c2 + 65 # 47 | c[j] = c1 48 | c[j+1] = c2 49 | j = j+2 50 | return c.decode("gbk") 51 | 52 | def decrypt(key, s): 53 | c = bytearray(str(s).encode("gbk")) 54 | n = len(c) # 计算 b 的字节数 55 | if n % 2 != 0 : 56 | return "" 57 | n = n // 2 58 | b = bytearray(n) 59 | j = 0 60 | for i in range(0, n): 61 | c1 = c[j] 62 | c2 = c[j+1] 63 | j = j+2 64 | c1 = c1 - 65 65 | c2 = c2 - 65 66 | b2 = c2*16 + c1 67 | b1 = b2^ key 68 | b[i]= b1 69 | try: 70 | return b.decode("gbk") 71 | except: 72 | return "failed" 73 | 74 | 75 | def my_pagination(request, queryset, display_amount=18, after_range_num = 5,bevor_range_num = 4): 76 | paginator = Paginator(queryset, display_amount) 77 | try: 78 | page = int(request.GET.get('page')) 79 | except: 80 | page = 1 81 | try: 82 | objects = paginator.page(page) 83 | except EmptyPage: 84 | objects = paginator.page(paginator.num_pages) 85 | except: 86 | objects = paginator.page(1) 87 | if page >= after_range_num: 88 | page_range = paginator.page_range[page-after_range_num:page+bevor_range_num] 89 | else: 90 | page_range = paginator.page_range[0:page+bevor_range_num] 91 | return objects, page_range 92 | def status(request,message): 93 | "状态" 94 | return render_to_response("status.html",locals(),context_instance=RequestContext(request)) 95 | 96 | 97 | # 分页函数 98 | def my_paginator(request,list,page_size=10): 99 | page = int(request.GET.get('page',1)) 100 | 101 | paginatior=Paginator(list,page_size) 102 | try: 103 | cur_page_obj=paginatior.page(page) 104 | except EmptyPage: 105 | #如果是空页就最大页paginatior.num_pages 106 | cur_page_obj=paginatior.page(paginatior.num_pages) 107 | #小于5页全显示 108 | if paginatior.num_pages<=5: 109 | page_range=paginatior.page_range 110 | else: 111 | #大于5页显示，且在后5页，显示后5页 112 | if page>paginatior.num_pages-5: 113 | page_range=paginatior.page_range[-5:] 114 | #前两页显示前5页 115 | elif page<=2: 116 | page_range=paginatior.page_range[:5] 117 | #中间页 118 | else: 119 | page_range=paginatior.page_range[page-2:page+3] 120 | return cur_page_obj,page_range 121 | import re 122 | def validate_date(date_time): 123 | regex = r'(\d+)/(\d+)/(\d+)' 124 | pattern = re.compile(regex) 125 | m = pattern.match(date_time) 126 | if m: 127 | date_time = pattern.sub(r'\3-\1-\2', date_time) 128 | return date_time 129 | 130 | def supervisor_required(func): 131 | def wappedfun(request): 132 | if request.user.is_superuser: 133 | return func(request) 134 | else: 135 | return HttpResponse("/login/?next="+request.path) 136 | return wappedfun 137 | 138 | def valid_group_required(*args): 139 | "验证用户组的装饰器，主要是用于限制视图函数的权限" 140 | def function(func): 141 | def wappedfun(request): 142 | grouplist=[Group.objects.get(name=i).name for i in args] 143 | request.user.valid_group_args=grouplist 144 | group=request.user.groups.all()[0].name 145 | request.user.group=group 146 | if (group in grouplist) or group=="管理员" : 147 | return func(request) 148 | else: 149 | return HttpResponseRedirect("/login/?next="+request.path) 150 | return wappedfun 151 | return function 152 | 153 | def get_svn_top_version(svnurl,svnuser,svnpasswd): 154 | infoobject=os.popen("svn info %s --username %s --password %s --no-auth-cache --non-interactive"%(svnurl,svnuser,svnpasswd)) 155 | result=infoobject.read() 156 | result=result.decode("utf-8") 157 | 158 | if result=='': 159 | return False,"无法获取%s的版本信息"%svnurl 160 | else: 161 | _u=re.search(ur"版本: (\d+)",result) 162 | if _u==None: 163 | _u=re.search(ur"Revision: (\d+)",result) 164 | return True,int(_u.groups()[0]) 165 | 166 | def valid_exception(fun,*args): 167 | "验证一个函数执行是否异常" 168 | try: 169 | fun(*args) 170 | except: 171 | return False 172 | else: 173 | return True 174 | 175 | class ValidObj(object): 176 | "对于一个元素验证的一个抽象" 177 | def __init__(self,request, method,element,valid,errormsg): 178 | self.request=request 179 | self.method=method 180 | self.element=element 181 | self.valid=valid 182 | self.errormsg=errormsg 183 | def valid_now(self): 184 | try: 185 | _u=getattr(self.request,self.method)[self.element] 186 | except: 187 | return False,self.errormsg 188 | else: 189 | if self.valid(_u)==False: 190 | return False,self.errormsg 191 | else: 192 | return True 193 | 194 | def valid_common_handle(*args): 195 | "通用的验证输入的函数" 196 | for i in args: 197 | assert isinstance(i ,ValidObj) 198 | for i in args: 199 | result=i.valid_now() 200 | if result!=True: 201 | return result 202 | else: 203 | pass 204 | return True 205 | 206 | -------------------------------------------------------------------------------- /cap/cap/views.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | __author__ = 'Administrator' 3 | from django.http import HttpResponse,HttpResponseRedirect 4 | from django.views.static import serve 5 | from django.conf import settings 6 | import os 7 | 8 | 9 | 10 | def front_view(request, path): 11 | if not path: 12 | path = "/" 13 | if path[0] != "/": 14 | path = "/" + path 15 | if path.endswith("/"): 16 | return HttpResponseRedirect(path+"index.html") 17 | else: 18 | real_file_path = os.path.join("/front", "." + path) 19 | real_file_path = os.path.abspath(real_file_path) 20 | try: 21 | return serve(request, real_file_path, settings.STATICFILES_DIRS[0]) 22 | except: 23 | if path.startswith("/static/"): 24 | try: 25 | serve(request, path.replace("/static/",''), settings.STATICFILES_DIRS[0]) 26 | except: 27 | return HttpResponse("404!%s"%real_file_path,status=404) 28 | else: 29 | return HttpResponse("404!%s" % real_file_path, status=404) 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /cap/cap/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for spiderhub project. 3 | 4 | This module contains the WSGI application used by Django's development server 5 | and any production WSGI deployments. It should expose a module-level variable 6 | named ``application``. Django's ``runserver`` and ``runfcgi`` commands discover 7 | this application via the ``WSGI_APPLICATION`` setting. 8 | 9 | Usually you will have the standard Django WSGI application here, but it also 10 | might make sense to replace the whole Django WSGI application with a custom one 11 | that later delegates to the Django one. For example, you could introduce WSGI 12 | middleware here, or combine a Django application with an application of another 13 | framework. 14 | 15 | """ 16 | import os 17 | 18 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "cap.settings") 19 | 20 | # This application object is used by any WSGI server configured to use this 21 | # file. This includes Django's development server, if the WSGI_APPLICATION 22 | # setting points here. 23 | from django.core.wsgi import get_wsgi_application 24 | application = get_wsgi_application() 25 | 26 | # Apply WSGI middleware here. 27 | # from helloworld.wsgi import HelloWorldApplication 28 | # application = HelloWorldApplication(application) -------------------------------------------------------------------------------- /cap/common/__init__.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | __author__ = 'admin' 3 | # -------------------------------- 4 | # Created by admin on 2016/11/11. 5 | # --------------------------------- -------------------------------------------------------------------------------- /cap/common/valid_configfile.py: -------------------------------------------------------------------------------- 1 | # coding:utf-8 2 | 3 | import ConfigParser 4 | import MySQLdb 5 | import os 6 | 7 | 8 | def valid_config(): 9 | cfg = ConfigParser.ConfigParser() 10 | try: 11 | result = cfg.read("/etc/crondeamon.ini") 12 | assert result == ["/etc/crondeamon.ini"] 13 | except: 14 | raise Exception( 15 | u'''Error！　未找到配置文件 /etc/crondeamon.ini !!，请新建配置文件 16 | 请参照如下配置格式： 17 | [crondeamon] 18 | mysqlhost=192.168.15.34 19 | mysqlport=3306 20 | mysqldb=testabc 21 | user=root 22 | passwd=123456 23 | charset=utf8 24 | host=192.168.15.34 25 | datadir=/data/test/crondeamon 26 | slaveport=8023''' 27 | ) 28 | try: 29 | assert cfg.sections() == ["crondeamon"] 30 | config = dict(cfg.items("crondeamon")) 31 | except: 32 | raise Exception( 33 | u'''Error! /etc/crondeamon.ini 中必须包含（且只包含）crondeamon区块！ 34 | 请参照如下配置格式： 35 | [crondeamon] 36 | mysqlhost=192.168.15.34 37 | mysqlport=3306 38 | mysqldb=testabc 39 | user=root 40 | passwd=123456 41 | charset=utf8 42 | host=192.168.15.34 43 | datadir=/data/test/crondeamon 44 | slaveport=8023''' 45 | ) 46 | if config.has_key("host"): 47 | try: 48 | host = config["host"] 49 | assert host and host != "127.0.0.1" and host != "localhost" 50 | except: 51 | raise Exception( 52 | u'''Error！/etc/crondeamon.ini 中 host配置项错误，　host不能为127.0.0.1 或localhost 53 | ''' 54 | ) 55 | else: 56 | config["host"] = "0.0.0.0" 57 | try: 58 | mysqlhost = config["mysqlhost"] 59 | mysqlport = int(config["mysqlport"]) 60 | mysqldb = config["mysqldb"] 61 | mysqluser = config["user"] 62 | mysqlpasswd = config["passwd"] 63 | mysqlcharset = config["charset"] 64 | conn = MySQLdb.connect(host=mysqlhost, port=mysqlport, db=mysqldb, user=mysqluser, passwd=mysqlpasswd, 65 | charset=mysqlcharset) 66 | conn.close() 67 | except: 68 | raise Exception( 69 | u'''Error! /etc/crondeamon.ini 中 mysql配置项错误，连接mysql失败！''') 70 | if not config.has_key("datadir"): 71 | raise Exception("datadir尚未配置!") 72 | config["slaveport"] = 8023 73 | config["uiport"] = 8024 74 | return config 75 | # valid_config() 76 | -------------------------------------------------------------------------------- /cap/common/valid_mysql.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | __author__ = 'admin' 3 | # -------------------------------- 4 | # Created by admin on 2016/11/11. 5 | # --------------------------------- 6 | from warnings import filterwarnings 7 | import MySQLdb 8 | filterwarnings('ignore', category = MySQLdb.Warning) 9 | import MySQLdb 10 | # create table sql list 11 | create_table_sql_list=[ 12 | '''CREATE TABLE if not EXISTS `auth_group` ( 13 | `id` int(11) NOT NULL AUTO_INCREMENT, 14 | `name` varchar(80) NOT NULL, 15 | PRIMARY KEY (`id`), 16 | UNIQUE KEY `name` (`name`) 17 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8;''', 18 | 19 | '''CREATE TABLE if not EXISTS `django_content_type` ( 20 | `id` int(11) NOT NULL AUTO_INCREMENT, 21 | `name` varchar(100) NOT NULL, 22 | `app_label` varchar(100) NOT NULL, 23 | `model` varchar(100) NOT NULL, 24 | PRIMARY KEY (`id`), 25 | UNIQUE KEY `app_label` (`app_label`,`model`) 26 | ) ENGINE=InnoDB AUTO_INCREMENT=13 DEFAULT CHARSET=utf8;''', 27 | 28 | '''CREATE TABLE if not EXISTS `auth_permission` ( 29 | `id` int(11) NOT NULL AUTO_INCREMENT, 30 | `name` varchar(50) NOT NULL, 31 | `content_type_id` int(11) NOT NULL, 32 | `codename` varchar(100) NOT NULL, 33 | PRIMARY KEY (`id`), 34 | UNIQUE KEY `content_type_id` (`content_type_id`,`codename`), 35 | KEY `auth_permission_e4470c6e` (`content_type_id`), 36 | CONSTRAINT `content_type_id_refs_id_728de91f` FOREIGN KEY (`content_type_id`) REFERENCES `django_content_type` (`id`) 37 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; ''', 38 | 39 | '''CREATE TABLE if not EXISTS `auth_group_permissions` ( 40 | `id` int(11) NOT NULL AUTO_INCREMENT, 41 | `group_id` int(11) NOT NULL, 42 | `permission_id` int(11) NOT NULL, 43 | PRIMARY KEY (`id`), 44 | UNIQUE KEY `group_id` (`group_id`,`permission_id`), 45 | KEY `auth_group_permissions_bda51c3c` (`group_id`), 46 | KEY `auth_group_permissions_1e014c8f` (`permission_id`), 47 | CONSTRAINT `group_id_refs_id_3cea63fe` FOREIGN KEY (`group_id`) REFERENCES `auth_group` (`id`), 48 | CONSTRAINT `permission_id_refs_id_a7792de1` FOREIGN KEY (`permission_id`) REFERENCES `auth_permission` (`id`) 49 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8;''', 50 | 51 | 52 | 53 | '''CREATE TABLE if not EXISTS `auth_user` ( 54 | `id` int(11) NOT NULL AUTO_INCREMENT, 55 | `username` varchar(30) NOT NULL, 56 | `first_name` varchar(30) NOT NULL, 57 | `last_name` varchar(30) NOT NULL, 58 | `email` varchar(75) NOT NULL, 59 | `password` varchar(128) NOT NULL, 60 | `is_staff` tinyint(1) NOT NULL, 61 | `is_active` tinyint(1) NOT NULL, 62 | `is_superuser` tinyint(1) NOT NULL, 63 | `last_login` datetime NOT NULL, 64 | `date_joined` datetime NOT NULL, 65 | PRIMARY KEY (`id`), 66 | UNIQUE KEY `username` (`username`) 67 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8;''', 68 | 69 | '''CREATE TABLE if not EXISTS `auth_user_groups` ( 70 | `id` int(11) NOT NULL AUTO_INCREMENT, 71 | `user_id` int(11) NOT NULL, 72 | `group_id` int(11) NOT NULL, 73 | PRIMARY KEY (`id`), 74 | UNIQUE KEY `user_id` (`user_id`,`group_id`), 75 | KEY `auth_user_groups_fbfc09f1` (`user_id`), 76 | KEY `auth_user_groups_bda51c3c` (`group_id`), 77 | CONSTRAINT `group_id_refs_id_f0ee9890` FOREIGN KEY (`group_id`) REFERENCES `auth_group` (`id`), 78 | CONSTRAINT `user_id_refs_id_831107f1` FOREIGN KEY (`user_id`) REFERENCES `auth_user` (`id`) 79 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8;''', 80 | 81 | '''CREATE TABLE if not EXISTS `auth_user_user_permissions` ( 82 | `id` int(11) NOT NULL AUTO_INCREMENT, 83 | `user_id` int(11) NOT NULL, 84 | `permission_id` int(11) NOT NULL, 85 | PRIMARY KEY (`id`), 86 | UNIQUE KEY `user_id` (`user_id`,`permission_id`), 87 | KEY `auth_user_user_permissions_fbfc09f1` (`user_id`), 88 | KEY `auth_user_user_permissions_1e014c8f` (`permission_id`), 89 | CONSTRAINT `permission_id_refs_id_67e79cb` FOREIGN KEY (`permission_id`) REFERENCES `auth_permission` (`id`), 90 | CONSTRAINT `user_id_refs_id_f2045483` FOREIGN KEY (`user_id`) REFERENCES `auth_user` (`id`) 91 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8;''', 92 | 93 | '''CREATE TABLE if not EXISTS `django_admin_log` ( 94 | `id` int(11) NOT NULL AUTO_INCREMENT, 95 | `action_time` datetime NOT NULL, 96 | `user_id` int(11) NOT NULL, 97 | `content_type_id` int(11) DEFAULT NULL, 98 | `object_id` longtext, 99 | `object_repr` varchar(200) NOT NULL, 100 | `action_flag` smallint(5) unsigned NOT NULL, 101 | `change_message` longtext NOT NULL, 102 | PRIMARY KEY (`id`), 103 | KEY `django_admin_log_fbfc09f1` (`user_id`), 104 | KEY `django_admin_log_e4470c6e` (`content_type_id`), 105 | CONSTRAINT `content_type_id_refs_id_288599e6` FOREIGN KEY (`content_type_id`) REFERENCES `django_content_type` (`id`), 106 | CONSTRAINT `user_id_refs_id_c8665aa` FOREIGN KEY (`user_id`) REFERENCES `auth_user` (`id`) 107 | ) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8;''', 108 | 109 | 110 | 111 | '''CREATE TABLE if not EXISTS `django_session` ( 112 | `session_key` varchar(40) NOT NULL, 113 | `session_data` longtext NOT NULL, 114 | `expire_date` datetime NOT NULL, 115 | PRIMARY KEY (`session_key`), 116 | KEY `django_session_c25c2c28` (`expire_date`) 117 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8;''', 118 | 119 | '''CREATE TABLE if not EXISTS `django_site` ( 120 | `id` int(11) NOT NULL AUTO_INCREMENT, 121 | `domain` varchar(100) NOT NULL, 122 | `name` varchar(50) NOT NULL, 123 | PRIMARY KEY (`id`) 124 | ) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8;''', 125 | '''CREATE TABLE if not exists `cap_crontask` ( 126 | `tid` int(11) NOT NULL AUTO_INCREMENT, 127 | `name` varchar(50) NOT NULL, 128 | `addtime` int(11) NOT NULL, 129 | `uptime` int(11) NOT NULL DEFAULT '0', 130 | `rule` varchar(50) NOT NULL, 131 | `status` int(11) NOT NULL, 132 | `repo_id` int(10) unsigned NOT NULL, 133 | `version` varchar(50) NOT NULL DEFAULT '', 134 | `pre_build` varchar(200) NOT NULL, 135 | `info` varchar(300) NOT NULL, 136 | `owner` varchar(200) NOT NULL, 137 | `run_cmd` varchar(500) NOT NULL, 138 | `run_times` int(10) unsigned NOT NULL, 139 | `worker_id` int(10) unsigned NOT NULL DEFAULT '0', 140 | `group_id` int(11) unsigned NOT NULL DEFAULT '0', 141 | `runlog_rid` int(11) unsigned NOT NULL DEFAULT '0', 142 | PRIMARY KEY (`tid`), 143 | KEY `cap_crontask_52094d6e` (`name`), 144 | KEY `cap_crontask_c9ad71dd` (`status`), 145 | KEY `cap_crontask_4741fd1b` (`owner`) 146 | ) ENGINE=InnoDB AUTO_INCREMENT=95 DEFAULT CHARSET=utf8''', 147 | '''CREATE TABLE if not exists `cap_deamontask` ( 148 | `tid` int(11) NOT NULL AUTO_INCREMENT, 149 | `name` varchar(50) NOT NULL, 150 | `addtime` int(11) NOT NULL, 151 | `uptime` int(11) NOT NULL DEFAULT '0', 152 | `status` int(11) NOT NULL, 153 | `version` varchar(50) NOT NULL DEFAULT '', 154 | `pre_build` varchar(200) NOT NULL, 155 | `info` varchar(300) NOT NULL, 156 | `owner` varchar(200) NOT NULL, 157 | `run_cmd` varchar(500) NOT NULL, 158 | `run_times` int(10) unsigned NOT NULL, 159 | `repo_id` int(10) unsigned NOT NULL DEFAULT '0', 160 | `worker_id` int(11) unsigned NOT NULL DEFAULT '0', 161 | `group_id` int(11) unsigned NOT NULL DEFAULT '0', 162 | `runlog_rid` int(11) unsigned NOT NULL DEFAULT '0', 163 | PRIMARY KEY (`tid`), 164 | KEY `cap_deamontask_52094d6e` (`name`), 165 | KEY `cap_deamontask_4741fd1b` (`owner`) 166 | ) ENGINE=InnoDB AUTO_INCREMENT=23 DEFAULT CHARSET=utf8''', 167 | '''CREATE TABLE if not exists `cap_group` ( 168 | `id` int(11) unsigned NOT NULL AUTO_INCREMENT, 169 | `name` varchar(100) NOT NULL DEFAULT '', 170 | `addtime` int(11) unsigned NOT NULL DEFAULT '0', 171 | PRIMARY KEY (`id`) 172 | ) ENGINE=InnoDB AUTO_INCREMENT=5 DEFAULT CHARSET=utf8''', 173 | '''CREATE TABLE if not exists `cap_repo` ( 174 | `id` int(11) NOT NULL AUTO_INCREMENT, 175 | `type` int(10) unsigned NOT NULL, 176 | `repo_url` varchar(200) NOT NULL, 177 | `user` varchar(50) NOT NULL, 178 | `password` varchar(50) NOT NULL, 179 | `addtime` int(11) NOT NULL, 180 | PRIMARY KEY (`id`), 181 | UNIQUE KEY `repo_url` (`repo_url`) 182 | ) ENGINE=InnoDB AUTO_INCREMENT=33 DEFAULT CHARSET=utf8''', 183 | '''CREATE TABLE if not exists `cap_runlog` ( 184 | `rid` int(11) NOT NULL AUTO_INCREMENT, 185 | `tid` int(11) NOT NULL, 186 | `type` varchar(30) NOT NULL, 187 | `repo_url` varchar(100) NOT NULL, 188 | `version` varchar(100) NOT NULL, 189 | `addtime` int(11) NOT NULL, 190 | `begintime` int(11) NOT NULL, 191 | `endtime` int(11) NOT NULL DEFAULT '0', 192 | `status` int(11) NOT NULL, 193 | `stderror` longtext NOT NULL, 194 | `stdout` longtext NOT NULL, 195 | PRIMARY KEY (`rid`), 196 | KEY `cap_runlog_e03e823b` (`tid`), 197 | KEY `cap_runlog_f0bd6439` (`type`), 198 | KEY `cap_runlog_52094d6fd` (`addtime`) 199 | ) ENGINE=InnoDB AUTO_INCREMENT=51962 DEFAULT CHARSET=utf8''', 200 | '''CREATE TABLE if not exists `cap_worker` ( 201 | `id` int(11) NOT NULL AUTO_INCREMENT, 202 | `ip` varchar(100) NOT NULL, 203 | `addtime` int(10) unsigned NOT NULL, 204 | `heartbeat` int(11) unsigned NOT NULL DEFAULT '0', 205 | `work_dir` varchar(50) NOT NULL DEFAULT '', 206 | `total_cpu` int(4) unsigned NOT NULL DEFAULT '0', 207 | `total_mem` int(10) unsigned NOT NULL DEFAULT '0', 208 | `platform` varchar(300) NOT NULL DEFAULT '', 209 | PRIMARY KEY (`id`), 210 | UNIQUE KEY `ip` (`ip`) 211 | ) ENGINE=InnoDB AUTO_INCREMENT=4 DEFAULT CHARSET=utf8''', 212 | '''CREATE TABLE if not exists `cap_worker_cpumem_log` ( 213 | `id` int(11) unsigned NOT NULL AUTO_INCREMENT, 214 | `addtime` int(11) unsigned NOT NULL DEFAULT '0', 215 | `work_id` int(10) unsigned NOT NULL DEFAULT '0', 216 | `cpu_percent` int(5) unsigned NOT NULL DEFAULT '0', 217 | `mem_percent` int(5) unsigned NOT NULL DEFAULT '0', 218 | PRIMARY KEY (`id`), 219 | KEY `cap_worker_cpumen_log_work_id_index` (`work_id`) 220 | ) ENGINE=InnoDB AUTO_INCREMENT=19300 DEFAULT CHARSET=utf8''', 221 | '''CREATE TABLE if not exists `pub_log` ( 222 | `pubid` int(11) NOT NULL AUTO_INCREMENT, 223 | `target_id` int(10) unsigned NOT NULL, 224 | `target_type` varchar(20) NOT NULL, 225 | `addtime` int(10) unsigned NOT NULL, 226 | `finishtime` int(10) unsigned NOT NULL, 227 | `stdout` longtext NOT NULL, 228 | `stderr` longtext NOT NULL, 229 | `state` int(10) unsigned NOT NULL, 230 | PRIMARY KEY (`pubid`) 231 | ) ENGINE=InnoDB AUTO_INCREMENT=187 DEFAULT CHARSET=utf8''', 232 | '''CREATE TABLE if not exists `cap_repo_commit_log` ( 233 | `id` int(11) unsigned NOT NULL AUTO_INCREMENT, 234 | `repo_id` int(11) unsigned NOT NULL DEFAULT '0', 235 | `ver` varchar(50) NOT NULL DEFAULT '', 236 | `author` varchar(100) NOT NULL DEFAULT '', 237 | `committime` int(11) unsigned NOT NULL DEFAULT '0', 238 | `message` varchar(200) NOT NULL DEFAULT '', 239 | PRIMARY KEY (`id`), 240 | UNIQUE KEY `cap_repo_commit_log_repo_id_ver_uindex` (`repo_id`,`ver`) 241 | ) ENGINE=InnoDB AUTO_INCREMENT=7 DEFAULT CHARSET=utf8''', 242 | '''CREATE TABLE if not exists `cap_repo_monitorlog` ( 243 | `id` int(11) unsigned NOT NULL AUTO_INCREMENT, 244 | `addtime` int(11) unsigned NOT NULL DEFAULT '0', 245 | `repo_id` int(11) unsigned NOT NULL DEFAULT '0', 246 | `log` longtext NOT NULL, 247 | PRIMARY KEY (`id`), 248 | UNIQUE KEY `cap_repo_monitorlog_repo_id_uindex` (`repo_id`) 249 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8''', 250 | '''INSERT ignore INTO django_content_type (id, name, app_label, model) VALUES 251 | (1, 'permission', 'auth', 'permission'), 252 | (2, 'group', 'auth', 'group'), 253 | (3, 'user', 'auth', 'user'), 254 | (4, 'content type', 'contenttypes', 'contenttype'), 255 | (5, 'session', 'sessions', 'session'), 256 | (6, 'site', 'sites', 'site'), 257 | (7, 'log entry', 'admin', 'logentry'), 258 | (8, 'worker', 'cap', 'worker'), 259 | (9, 'repo', 'cap', 'repo'), 260 | (10, 'pub log', 'cap', 'publog'), 261 | (11, '计划任务', 'cap', 'crontask'), 262 | (12, '计划任务', 'cap', 'deamontask'), 263 | (13, '运行日志', 'cap', 'runlog');''', 264 | '''INSERT ignore INTO auth_permission (id, name, content_type_id, codename) VALUES 265 | (1, 'Can add permission', 1, 'add_permission') , 266 | (2, 'Can change permission', 1, 'change_permission') , 267 | (3, 'Can delete permission', 1, 'delete_permission') , 268 | (4, 'Can add group', 2, 'add_group') , 269 | (5, 'Can change group', 2, 'change_group') , 270 | (6, 'Can delete group', 2, 'delete_group') , 271 | (7, 'Can add user', 3, 'add_user') , 272 | (8, 'Can change user', 3, 'change_user') , 273 | (9, 'Can delete user', 3, 'delete_user') , 274 | (10, 'Can add content type', 4, 'add_contenttype') , 275 | (11, 'Can change content type', 4, 'change_contenttype') , 276 | (12, 'Can delete content type', 4, 'delete_contenttype') , 277 | (13, 'Can add session', 5, 'add_session') , 278 | (14, 'Can change session', 5, 'change_session') , 279 | (15, 'Can delete session', 5, 'delete_session') , 280 | (16, 'Can add site', 6, 'add_site') , 281 | (17, 'Can change site', 6, 'change_site') , 282 | (18, 'Can delete site', 6, 'delete_site') , 283 | (19, 'Can add log entry', 7, 'add_logentry') , 284 | (20, 'Can change log entry', 7, 'change_logentry') , 285 | (21, 'Can delete log entry', 7, 'delete_logentry') , 286 | (22, 'Can add worker', 8, 'add_worker') , 287 | (23, 'Can change worker', 8, 'change_worker') , 288 | (24, 'Can delete worker', 8, 'delete_worker') , 289 | (25, 'Can add repo', 9, 'add_repo') , 290 | (26, 'Can change repo', 9, 'change_repo') , 291 | (27, 'Can delete repo', 9, 'delete_repo') , 292 | (28, 'Can add pub log', 10, 'add_publog') , 293 | (29, 'Can change pub log', 10, 'change_publog') , 294 | (30, 'Can delete pub log', 10, 'delete_publog') , 295 | (31, 'Can add 计划任务', 11, 'add_crontask') , 296 | (32, 'Can change 计划任务', 11, 'change_crontask') , 297 | (33, 'Can delete 计划任务', 11, 'delete_crontask') , 298 | (34, 'Can add 计划任务', 12, 'add_deamontask') , 299 | (35, 'Can change 计划任务', 12, 'change_deamontask') , 300 | (36, 'Can delete 计划任务', 12, 'delete_deamontask') , 301 | (37, 'Can add 运行日志', 13, 'add_runlog') , 302 | (38, 'Can change 运行日志', 13, 'change_runlog') , 303 | (39, 'Can delete 运行日志', 13, 'delete_runlog');''', 304 | '''insert ignore into django_site values(1,"example.com","example.com");''', 305 | 306 | ''' 307 | INSERT ignore INTO auth_user (id, username, first_name, last_name, email, password, is_staff, 308 | is_active, is_superuser, last_login, date_joined) VALUES ( 309 | 1, 'admin', '', '', '18749679769@163.com', 310 | 'pbkdf2_sha256$10000$1X58MsOvjyOa$/S7paomFlNanSgEyuwG0QqaFlOVf97DepE0O5eD3YQo=', 311 | 1, 1, 1, '2018-06-05 15:39:13', '2018-06-05 15:39:13');''', 312 | 313 | 314 | '''INSERT ignore INTO cap_group (id, name, addtime) VALUES (1, '默认', 1528706232);''', 315 | 316 | '''INSERT ignore INTO django_site (id, domain, name) VALUES (1, 'example.com', 'example.com');''' 317 | ] 318 | 319 | def valid(host,port,db,user,passwd): 320 | for i in create_table_sql_list: 321 | conn = MySQLdb.connect(host=host, port=port, user=user, db=db, passwd=passwd, charset="utf8") 322 | cursor = conn.cursor() 323 | try: 324 | cursor.execute(i) 325 | except Exception as e : 326 | print "异常：" ,str(e) 327 | return False 328 | cursor.close() 329 | conn.commit() 330 | conn.close() 331 | return True 332 | 333 | if __name__ == '__main__': 334 | print valid("192.168.14.90",3306,"cap","spider","123456") 335 | -------------------------------------------------------------------------------- /cap/django_wsgi.py: -------------------------------------------------------------------------------- 1 | # coding:utf-8 2 | __author__ = 'zhoukunpeng' 3 | # -------------------------------- 4 | # Created by zhoukunpeng on 2015/9/7. 5 | # --------------------------------- 6 | import os 7 | 8 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "cap.settings") 9 | from django.core.handlers.wsgi import WSGIHandler 10 | application = WSGIHandler() -------------------------------------------------------------------------------- /cap/log/__init__.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | # write by zhou -------------------------------------------------------------------------------- /cap/log/master_logger.py: -------------------------------------------------------------------------------- 1 | # coding:utf-8 2 | # write by zhou 3 | 4 | from twisted.python.log import * 5 | from twisted.python.logfile import DailyLogFile 6 | from twisted.python import logfile 7 | 8 | f = logfile.LogFile("cap-master.log", '/tmp', 9 | rotateLength=10000000, 10 | maxRotatedFiles=10) 11 | flobserver = FileLogObserver(f) 12 | def logger(): 13 | observer = flobserver.emit 14 | return observer -------------------------------------------------------------------------------- /cap/log/worker_logger.py: -------------------------------------------------------------------------------- 1 | # coding:utf-8 2 | # write by zhou 3 | 4 | from twisted.python.log import * 5 | from twisted.python.logfile import DailyLogFile 6 | from twisted.python import logfile 7 | 8 | f = logfile.LogFile("cap-worker.log", '/tmp', 9 | rotateLength=10000000, 10 | maxRotatedFiles=10) 11 | flobserver = FileLogObserver(f) 12 | def logger(): 13 | observer = flobserver.emit 14 | return observer 15 | -------------------------------------------------------------------------------- /cap/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | #from crondeamon.ui import cap 5 | # def get_manage_dir(): 6 | # return cap.__path__ 7 | # sys.path.append(get_manage_dir()[0]) 8 | 9 | if __name__ == "__main__": 10 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "cap.settings") 11 | 12 | from django.core.management import execute_from_command_line 13 | 14 | execute_from_command_line(sys.argv) 15 | -------------------------------------------------------------------------------- /cap/sbin/__init__.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | __author__ = 'admin' 3 | # -------------------------------- 4 | # Created by admin on 2016/11/18. 5 | # --------------------------------- -------------------------------------------------------------------------------- /cap/sbin/_kill.py: -------------------------------------------------------------------------------- 1 | # coding:utf-8 2 | __author__ = 'zhoukunpeng' 3 | # -------------------------------- 4 | # Created by zhoukunpeng on 2015/6/25. 5 | # --------------------------------- 6 | import psutil 7 | import sys 8 | def killpid(pid): 9 | try: 10 | try: 11 | pid=int(pid) 12 | _mainprocess=psutil.Process(pid) 13 | _main_cmd=_mainprocess.cmdline() 14 | _mark=0 15 | for i in _main_cmd: 16 | if "twistd" in i : 17 | _mark+=1 18 | if "crondeamon" in i: 19 | _mark+=1 20 | 21 | if _mark>=3 and _mainprocess.is_running(): 22 | childpids=_mainprocess.children(True) 23 | buff=dict([(k.pid,k.create_time()) for k in childpids]) 24 | _mainprocess.send_signal(9) 25 | for j in childpids: 26 | try: 27 | _process=psutil.Process(j.pid) 28 | except: 29 | pass 30 | else: 31 | if _process.is_running() and j.create_time()==buff[j.pid] : 32 | _process.send_signal(9) 33 | except: 34 | pass 35 | else: 36 | print "PID: %s have been killed!"%pid 37 | except Exception as e : 38 | print e.message 39 | return True 40 | -------------------------------------------------------------------------------- /cap/sbin/all_start.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | __author__ = 'admin' 3 | # -------------------------------- 4 | # Created by admin on 2016/11/18. 5 | # --------------------------------- 6 | import sys 7 | import os 8 | import psutil 9 | import argparse 10 | from cap.common.valid_mysql import valid 11 | 12 | def main(): 13 | parser = argparse.ArgumentParser(description="启动cap-all服务") 14 | parser.add_argument("--mysql_url",help="mysql的地址(比如：127.0.0.1:3306/db_test)",required=True) 15 | parser.add_argument("--mysql_user",help="mysql用户. (比如：test)",required=True) 16 | parser.add_argument("--mysql_password", help="mysql密码. (比如 123456)", required=True) 17 | parser.add_argument("--host",help="服务绑定的IP地址. (比如：192.168.1.2 ,default： 0.0.0.0)", 18 | required=False,default="0.0.0.0") 19 | parser.add_argument("--work_dir", help="工作目录. (default .)", required=False, default=".") 20 | info = parser.parse_args() 21 | init_1 = psutil.Process(pid=1) 22 | for i in init_1.children(True): 23 | cmd_line = i.cmdline() 24 | mask = 0 25 | for j in cmd_line: 26 | if "twistd" in j or ("cap-all" in j and "cap-all-start" not in j): 27 | mask += 1 28 | if mask >= 2: 29 | print "cap-all服务已经在运行了！无法执行本次启动操作！" 30 | sys.exit(123) 31 | mysql_url = info.mysql_url.strip() 32 | try: 33 | a,b = mysql_url.split(":") 34 | mysql_host = a 35 | mysql_port,mysql_db = b.split("/") 36 | mysql_port = int(mysql_port) 37 | except: 38 | print "mysql相关配置错误" 39 | else: 40 | mysql_user = info.mysql_user 41 | mysql_password = info.mysql_password 42 | result = valid(mysql_host,mysql_port,mysql_db,mysql_user,mysql_password) 43 | if not result: 44 | print "mysql相关配置错误" 45 | sys.exit(123) 46 | result = os.system("twistd --pidfile /tmp/cap-master.pid --logger cap.log.master_logger.logger cap-all --mysql_url %s --mysql_user %s --mysql_password %s \ 47 | --host %s --work_dir %s"%( 48 | info.mysql_url,info.mysql_user,info.mysql_password,info.host,info.work_dir)) 49 | if not result: 50 | print "cap-all服务启动成功,管理界面：http://%s:9912/ 初始用户为:admin 初始密码：gc895316"%info.host 51 | -------------------------------------------------------------------------------- /cap/sbin/all_stop.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | __author__ = 'admin' 3 | # -------------------------------- 4 | # Created by admin on 2016/11/18. 5 | # --------------------------------- 6 | import sys 7 | import psutil 8 | import argparse 9 | 10 | 11 | def main(): 12 | parser = argparse.ArgumentParser(description="停止cap-all服务") 13 | init_1 = psutil.Process(pid=1) 14 | for i in init_1.children(True): 15 | cmd_line = i.cmdline() 16 | mask = 0 17 | for j in cmd_line: 18 | if "twistd" in j or ("cap-all" in j and 'cap-all-stop' not in j): 19 | mask += 1 20 | if mask >=2 : 21 | worker_process = i 22 | for k in worker_process.children(True): 23 | k.send_signal(9) 24 | worker_process.send_signal(9) 25 | print "成功停止cap-all服务" 26 | sys.exit(0) 27 | else: 28 | print "cap-all服务当前尚未在运行" 29 | sys.exit(123) 30 | 31 | -------------------------------------------------------------------------------- /cap/sbin/worker_start.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | __author__ = 'admin' 3 | # -------------------------------- 4 | # Created by admin on 2016/11/18. 5 | # --------------------------------- 6 | import sys 7 | import os 8 | import psutil 9 | import argparse 10 | 11 | 12 | def main(): 13 | parser = argparse.ArgumentParser(description="启动cap-worker服务") 14 | parser.add_argument("--master",help="master节点的IP. (default 127.0.0.1)",required=False,default="127.0.0.1") 15 | parser.add_argument("--work_dir",help="工作目录. (default .)",required=False,default=".") 16 | parser.add_argument("--host",help="所要绑定的IP地址. (default 0.0.0.0)",required=False,default="0.0.0.0") 17 | info = parser.parse_args() 18 | init_1 = psutil.Process(pid=1) 19 | for i in init_1.children(True): 20 | cmd_line = i.cmdline() 21 | mask = 0 22 | for j in cmd_line: 23 | if "twistd" in j or ("cap-worker" in j and 'cap-worker-start' not in j): 24 | mask += 1 25 | if mask >= 2: 26 | print cmd_line 27 | print "worker已经在运行了！无法执行本次启动操作！" 28 | sys.exit(123) 29 | result = os.system("cd %s && twistd --pidfile /tmp/cap-worker.pid --logger cap.log.worker_logger.logger cap-worker --master %s --work_dir %s --host %s "%( 30 | info.work_dir,info.master,info.work_dir,info.host 31 | )) 32 | if not result: 33 | print "启动cap-worker成功" 34 | -------------------------------------------------------------------------------- /cap/sbin/worker_stop.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | __author__ = 'admin' 3 | # -------------------------------- 4 | # Created by admin on 2016/11/18. 5 | # --------------------------------- 6 | import sys 7 | import psutil 8 | import argparse 9 | 10 | 11 | def main(): 12 | parser = argparse.ArgumentParser(description="停止cap-worker服务") 13 | init_1 = psutil.Process(pid=1) 14 | for i in init_1.children(True): 15 | cmd_line = i.cmdline() 16 | mask = 0 17 | for j in cmd_line: 18 | if "twistd" in j or ("cap-worker" in j and 'cap-worker-stop' not in j): 19 | mask += 1 20 | if mask >=2: 21 | worker_process = i 22 | for k in worker_process.children(True): 23 | k.send_signal(9) 24 | worker_process.send_signal(9) 25 | print "成功停止cap-worker" 26 | sys.exit(0) 27 | else: 28 | print "cap-worker当前尚未在运行" 29 | sys.exit(123) 30 | 31 | if __name__ == "__main__": 32 | main() -------------------------------------------------------------------------------- /cap/scripts/aa/replace.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | # write by zhou 3 | import os 4 | import sys 5 | import datetime 6 | import re 7 | #from os.path import expanduser 8 | # 9 | if len(sys.argv) != 2: 10 | raise Exception("参数错误！") 11 | error_path = sys.argv[1] 12 | # if not os.path.exists(right_path): 13 | # raise Exception("目录不存在！"+right_path) 14 | if not os.path.exists(error_path): 15 | raise Exception("目录不存在!"+error_path) 16 | # right_path = "./wordpress" 17 | # error_path = "/Users/zhou/majinh.com" 18 | #print "正确文件目录:",right_path 19 | print "错误文件目录：",error_path 20 | while True: 21 | input = raw_input("确定开始处理？(输入Y开始处理,输入N退出)") 22 | input = input.lower() 23 | if input in ('y','n'): 24 | if input == "y": 25 | break 26 | else: 27 | sys.exit(0) 28 | 29 | def content_handle(content): 30 | new_content = '' 31 | is_right = True 32 | for line in content.split("\n"): 33 | if line.startswith("@include"): 34 | line = line.strip().strip(";") 35 | _ =line.replace("@include","").strip(" ").strip('"').strip("\\").split("\\") 36 | _ = ''.join([str(len(i)) for i in _])[:10] 37 | if len(_) == 10 and _[0] * len(_) == _: 38 | is_right = False 39 | new_content += '' 40 | else: 41 | new_content += "\n" 42 | new_content += line 43 | else: 44 | if line.endswith("?>"): 45 | is_right = False 46 | new_content += '' 47 | else: 48 | new_content += ("\n"+line) 49 | return (is_right,new_content) 50 | 51 | error_regex = r'''''' 52 | js_error_regex = r'''var _0x2515=.+?''' 53 | print "开始处理...." 54 | for root,dirs,files in os.walk(error_path): 55 | for _f in files: 56 | file_path = os.path.join(root,_f) 57 | if _f.endswith(".html") or _f.endswith(".php") or _f.endswith(".bak.bak"): 58 | try: 59 | file_content = '' 60 | with open(file_path,"r") as f: 61 | file_content = f.read() 62 | except Exception as e : 63 | pass 64 | #print "文件读取异常：%s ,%s"%(file_path,str(e)) 65 | else: 66 | # if re.findall(error_regex,file_content): 67 | # print "发现异常文件：%s"%file_path 68 | # real_content = re.sub(error_regex,'',file_content) 69 | # try: 70 | # with open(file_path,"w") as f: 71 | # f.write(real_content) 72 | # except Exception as e : 73 | # #print "文件写入异常：%s ,%s" % (file_path, str(e)) 74 | # pass 75 | # else: 76 | # print "成功处理异常文件：%s"%file_path 77 | 78 | if _f.endswith(".php"): 79 | with open(file_path,"r") as f: 80 | file_content = f.read() 81 | is_right,real_content = content_handle(file_content) 82 | if not is_right: 83 | with open(file_path,"w") as f: 84 | f.write(real_content) 85 | print "成功处理异常php文件：%s" % file_path 86 | 87 | if _f == "index.html.bak.bak": 88 | print "重命名 %s %s"%(file_path,file_path.replace(".bak.bak",'')) 89 | os.system("mv %s %s"%(file_path,file_path.replace(".bak.bak",''))) 90 | php_path = os.path.join(root,"index.php") 91 | if os.path.exists(php_path): 92 | os.system("rm -r %s"%php_path) 93 | print "发现index.html.bak.bak同目录的index.php，已删除！" 94 | 95 | # elif _f.endswith(".js"): 96 | # sub_path = file_path.replace(error_path,"").lstrip("/") 97 | # right_file_path = os.path.join(right_path,sub_path) 98 | # try: 99 | # file_content = '' 100 | # with open(file_path, "r") as f: 101 | # file_content = f.read() 102 | # except Exception as e: 103 | # print "文件读取异常：%s ,%s" % (file_path, str(e)) 104 | # else: 105 | # if re.findall(js_error_regex, file_content): 106 | # print "发现异常JS文件：%s" % file_path 107 | # real_content = '' 108 | # try: 109 | # with open(right_file_path, "r") as f: 110 | # real_content = f.read() 111 | # except Exception as e: 112 | # print "未找到对应的正确JS文件，尝试清空..." 113 | # try: 114 | # with open(file_path, "w") as f: 115 | # f.write('') 116 | # except Exception as e: 117 | # print "文件写入异常：%s ,%s" % (file_path, str(e)) 118 | # else: 119 | # print "成功清空异常文件：%s" % file_path 120 | # else: 121 | # try: 122 | # with open(file_path, "w") as f: 123 | # f.write(real_content) 124 | # except Exception as e: 125 | # print "文件写入异常：%s ,%s" % (file_path, str(e)) 126 | # else: 127 | # print "成功处理异常文件：%s" % file_path -------------------------------------------------------------------------------- /cap/scripts/aa/replace_from_file.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | # write by zhou 3 | import os 4 | import sys 5 | import datetime 6 | import re 7 | import time 8 | # 9 | if len(sys.argv) != 3: 10 | raise Exception("参数错误！") 11 | file_path,dir_path = sys.argv[1],sys.argv[2] 12 | if not os.path.exists(file_path): 13 | raise Exception("路径不存在！"+file_path) 14 | if not os.path.exists(dir_path): 15 | raise Exception("路径不存在!"+dir_path) 16 | # file_path = "./scan.log" 17 | # dir_path = "/root/templates/" 18 | print "文件:",file_path 19 | print "正确代码目录：",dir_path 20 | while True: 21 | input = raw_input("确定开始处理？(输入Y开始处理,输入N退出)") 22 | input = input.lower() 23 | if input in ('y','n'): 24 | if input == "y": 25 | break 26 | else: 27 | sys.exit(0) 28 | 29 | 30 | print "开始处理...." 31 | file_path = os.path.abspath(file_path) 32 | print file_path 33 | with open(file_path,"r") as f: 34 | for path in f.readlines(): 35 | path = path.strip() 36 | _ = path 37 | while True: 38 | _ = os.path.abspath(os.path.join(_,"..")) 39 | dir_name = _.strip("/").split("/")[-1] 40 | if _ == "/": 41 | break 42 | if re.match(".+?\w+\.\w+$",dir_name): 43 | _ = path[len(_)+1:] 44 | break 45 | if _ != "/": 46 | right_file_path = os.path.join(dir_path,_) 47 | print right_file_path 48 | try: 49 | _f = open(right_file_path,"r") 50 | content = _f.read() 51 | _f.close() 52 | except: 53 | print "%s处理失败！"%path 54 | with open("replace.log","a+") as log_f: 55 | log_f.write("[%s]:%s处理失败！\n"%(datetime.datetime.now(),path)) 56 | else: 57 | try: 58 | with open(path,"w") as _f: 59 | _f.write(content) 60 | except: 61 | pass 62 | print "%s还原成功！"%path 63 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /cap/scripts/aa/replace_functions.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | # write by zhou 3 | import os 4 | import sys 5 | import datetime 6 | import re 7 | #from os.path import expanduser 8 | # 9 | if len(sys.argv) != 2: 10 | raise Exception("参数错误！") 11 | error_path = sys.argv[1] 12 | if not os.path.exists(error_path): 13 | raise Exception("目录不存在!"+error_path) 14 | 15 | print "错误文件目录：",error_path 16 | while True: 17 | input = raw_input("确定开始处理？(输入Y开始处理,输入N退出)") 18 | input = input.lower() 19 | if input in ('y','n'): 20 | if input == "y": 21 | break 22 | else: 23 | sys.exit(0) 24 | 25 | def content_handle(content): 26 | new_content = re.sub(r"<\?php[\s\S]+\?> <\?php",'var _0x2515=.+?''' 34 | js_error_regex = r'''var _0x2515=.+?''' 35 | print "开始处理...." 36 | for error_path in os.listdir(error_content_path): 37 | error_path = os.path.join(error_content_path,error_path) 38 | if not os.path.isfile(error_path): 39 | print "开始处理...",error_path 40 | for root,dirs,files in os.walk(error_path): 41 | for _f in files: 42 | file_path = os.path.join(root,_f) 43 | if _f.endswith(".html") or _f.endswith(".php") or _f.endswith(".bak.bak"): 44 | try: 45 | file_content = '' 46 | with open(file_path,"r") as f: 47 | file_content = f.read() 48 | except Exception as e : 49 | print "文件读取异常：%s ,%s"%(file_path,str(e)) 50 | else: 51 | if re.findall(error_regex,file_content): 52 | print "发现异常文件：%s"%file_path 53 | real_content = re.sub(error_regex,'',file_content) 54 | try: 55 | with open(file_path,"w") as f: 56 | f.write(real_content) 57 | except Exception as e : 58 | print "文件写入异常：%s ,%s" % (file_path, str(e)) 59 | else: 60 | print "成功处理异常文件：%s"%file_path 61 | elif _f.endswith(".js"): 62 | sub_path = file_path.replace(error_path,"").lstrip("/") 63 | right_file_path = os.path.join(right_path,sub_path) 64 | try: 65 | file_content = '' 66 | with open(file_path, "r") as f: 67 | file_content = f.read() 68 | except Exception as e: 69 | print "文件读取异常：%s ,%s" % (file_path, str(e)) 70 | else: 71 | if re.findall(js_error_regex, file_content): 72 | print "发现异常JS文件：%s" % file_path 73 | real_content = '' 74 | try: 75 | with open(right_file_path, "r") as f: 76 | real_content = f.read() 77 | except Exception as e: 78 | print "未找到对应的正确JS文件，尝试清空..." 79 | try: 80 | with open(file_path, "w") as f: 81 | f.write('') 82 | except Exception as e: 83 | print "文件写入异常：%s ,%s" % (file_path, str(e)) 84 | else: 85 | print "成功清空异常文件：%s" % file_path 86 | else: 87 | try: 88 | with open(file_path, "w") as f: 89 | f.write(real_content) 90 | except Exception as e: 91 | print "文件写入异常：%s ,%s" % (file_path, str(e)) 92 | else: 93 | print "成功处理异常文件：%s" % file_path -------------------------------------------------------------------------------- /cap/scripts/aa/tihuan.py: -------------------------------------------------------------------------------- 1 | # coding:utf8 2 | 3 | 4 | import socket 5 | socket.setdefaulttimeout(3) 6 | from twisted.internet import reactor 7 | from twisted.web.client import getPage,HTTPClientFactory 8 | 9 | from twisted.internet.defer import Deferred 10 | from twisted.internet.defer import inlineCallbacks 11 | 12 | 13 | def startedConnecting(self, connector): 14 | def timeout(): 15 | connector.stopConnecting() 16 | timeoutCall = reactor.callLater(self.timeout, timeout) 17 | self.deferred.addBoth(self._cancelTimeout, timeoutCall) 18 | HTTPClientFactory.startedConnecting = startedConnecting 19 | 20 | def callback(*args,**kwargs): 21 | print "success!!" 22 | 23 | def errback(a): 24 | print "error",a 25 | 26 | def test(): 27 | defer = getPage("http://www.wolover.com",timeout=3) 28 | defer.addCallback(callback) 29 | defer.addErrback(errback) 30 | 31 | @inlineCallbacks 32 | def test1(): 33 | try: 34 | result = yield getPage("http://www.wolover.com",timeout=3) 35 | except Exception as e : 36 | print str(e) 37 | 38 | 39 | test1() 40 | reactor.run() 41 | -------------------------------------------------------------------------------- /cap/static/front/index.html: -------------------------------------------------------------------------------- 1 | CAP系统
-------------------------------------------------------------------------------- /cap/static/front/static/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhoukunpeng504/cap-python/c5ca57db90c6741485d590f644a724164f8c34cd/cap/static/front/static/favicon.ico -------------------------------------------------------------------------------- /cap/static/front/static/fonts/element-icons.6f0a763.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhoukunpeng504/cap-python/c5ca57db90c6741485d590f644a724164f8c34cd/cap/static/front/static/fonts/element-icons.6f0a763.ttf -------------------------------------------------------------------------------- /cap/static/front/static/img/timg.0840c53.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhoukunpeng504/cap-python/c5ca57db90c6741485d590f644a724164f8c34cd/cap/static/front/static/img/timg.0840c53.jpeg -------------------------------------------------------------------------------- /cap/static/front/static/js/manifest.3ad1d5771e9b13dbdad2.js: -------------------------------------------------------------------------------- 1 | !function(r){var n=window.webpackJsonp;window.webpackJsonp=function(e,u,c){for(var f,i,p,a=0,l=[];a 2 | 3 | 4 | 5 | Title 6 | 7 | 8 | this is test.html 9 | 10 | -------------------------------------------------------------------------------- /cap/static/front/test1/test111.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Title 6 | 7 | 8 | this is test1/test111.html 9 | 10 | -------------------------------------------------------------------------------- /cap/templates/test.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Title 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /cap/tornado_server.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | from tornado.options import options, define, parse_command_line 4 | import django.core.handlers.wsgi 5 | import tornado.httpserver 6 | import tornado.ioloop 7 | import tornado.web 8 | import tornado.wsgi 9 | import sys 10 | 11 | _HERE = os.path.dirname(os.path.abspath(__file__)) 12 | sys.path.append(_HERE) 13 | os.environ['DJANGO_SETTINGS_MODULE'] = 'cap.settings' 14 | 15 | def main(master_port,mysql_host,mysql_port,mysql_db,mysql_user,mysql_password): 16 | os.mysql_host = mysql_host 17 | os.mysql_port = mysql_port 18 | os.mysql_db = mysql_db 19 | os.mysql_user = mysql_user 20 | os.mysql_password = mysql_password 21 | wsgi_app = tornado.wsgi.WSGIContainer( 22 | django.core.handlers.wsgi.WSGIHandler()) 23 | tornado_app = tornado.web.Application( 24 | [ 25 | ('.*', tornado.web.FallbackHandler, dict(fallback=wsgi_app)), 26 | ] 27 | ) 28 | server = tornado.httpserver.HTTPServer(tornado_app) 29 | server.listen(master_port) 30 | tornado.ioloop.IOLoop.instance().start() 31 | 32 | if __name__ == '__main__': 33 | main(8080,"192.168.14.90",3306,"cap_test","spider","123456") 34 | -------------------------------------------------------------------------------- /cap_twisted/__init__.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | __author__ = 'admin' 3 | # -------------------------------- 4 | # Created by admin on 2016/11/11. 5 | # --------------------------------- -------------------------------------------------------------------------------- /cap_twisted/service.py: -------------------------------------------------------------------------------- 1 | # coding:utf-8 2 | __author__ = 'zhoukunpeng' 3 | # -------------------------------- 4 | # Created by zhoukunpeng on 2015/6/4. 5 | # --------------------------------- 6 | from twisted.web import xmlrpc 7 | from twisted.internet import reactor, task, defer, threads 8 | from txscheduling.cron import CronSchedule 9 | from txscheduling.task import ScheduledCall 10 | from twisted.internet import protocol, error 11 | import psutil 12 | import sys 13 | import cloudpickle 14 | import base64 15 | import os 16 | from twisted.web.client import getPage 17 | import urllib 18 | import traceback 19 | #from django.conf import settings 20 | 21 | 22 | #settings.configure() 23 | 24 | reactor.suggestThreadPoolSize(500) # 线程池改为500， 防止线程占满 25 | if sys.getdefaultencoding() != "utf-8": 26 | reload(sys) 27 | sys.setdefaultencoding("utf-8") 28 | 29 | CRON_BUFF = {} # CRON 专用全局变量 30 | TASK_BUFF = {} # TASK 专用全局变量 31 | DEAMON_BUFF = {} 32 | PID_BUFF = {} 33 | 34 | 35 | def set_time_out(seconds, deferjob): 36 | "给一个deferjob添加一个超时时间,防止一个xmlrpc运行时间过长。" 37 | 38 | def _handle(deferjob): 39 | if deferjob.called: 40 | pass 41 | else: 42 | deferjob.cancel() 43 | 44 | seconds = int(seconds) 45 | reactor.callLater(seconds, _handle, deferjob) 46 | return True 47 | 48 | 49 | class ProcessProtocol(protocol.ProcessProtocol): 50 | 51 | def __init__(self, fun, fun_args, fun_kwargs, callback, errback, stdout_callback=None, stderr_callback=None): 52 | self.pid = None 53 | self.lastpid = None 54 | self.running = False 55 | self.has_end = False 56 | self.callback = callback 57 | self.errback = errback 58 | self.stdout_callback = stdout_callback 59 | self.stderr_callback = stderr_callback 60 | try: 61 | _ = {"fun": fun, "callback": callback, "errback": errback, "args": fun_args, "kwargs": fun_kwargs} 62 | _ = cloudpickle.dumps(_) 63 | _ = base64.b64encode(_) 64 | self.pickle_data = _ 65 | except: 66 | raise Exception("can not pickle the fun or callback or errback ,please recheck it!") 67 | else: 68 | pass 69 | 70 | self.stdout_data = "" 71 | self.stderr_data = "" 72 | 73 | def run(self): 74 | reactor.spawnProcess(self, "python", ["python", "worker.py"], 75 | env=os.environ, path=os.path.dirname(__file__)) 76 | self.running = True 77 | self.has_end = False 78 | self.stdout_data = "" 79 | self.stderr_data = "" 80 | 81 | def is_run(self): 82 | return self.running 83 | 84 | def connectionMade(self): 85 | self.pid = self.transport.pid 86 | self.running = True 87 | self.has_end = False 88 | self.transport.write(self.pickle_data + "\n") 89 | 90 | def outReceived(self, data): 91 | "stdout data" 92 | #print "[SUBPROCESS-OUTPUT]:", data 93 | if len(self.stdout_data) <= 2000: 94 | self.stdout_data += data 95 | else: 96 | pass 97 | 98 | def errReceived(self, data): 99 | "stderr data" 100 | #print "[SUBPROCESS-ERRPUT]", data 101 | if len(self.stderr_data) <= 2000: 102 | self.stderr_data += data 103 | else: 104 | pass 105 | 106 | def processExited(self, reason): 107 | #print "process exited", [reason], reason 108 | self.has_end = True 109 | self.running = False 110 | is_right = True 111 | if reason.type == error.ProcessTerminated or isinstance(reason.type, error.ProcessTerminated): 112 | is_right = False 113 | else: 114 | try: 115 | assert (reason.value.exitCode > 0 or reason.value.exitCode < 0) 116 | is_right = False 117 | except: 118 | pass 119 | print "is_right", is_right 120 | 121 | if self.stdout_callback: 122 | _run_in_sub(self.stdout_callback, [self.stdout_data, is_right], {}, 3) 123 | if self.stderr_callback: 124 | _run_in_sub(self.stderr_callback, [self.stderr_data, is_right], {}, 3) 125 | 126 | def processEnded(self, reason): 127 | pass 128 | 129 | def kill(self): 130 | "杀死进程" 131 | if self.running: 132 | _mainprocess = psutil.Process(pid=self.pid) 133 | childpids = _mainprocess.children(True) 134 | buff = dict([(k.pid, k.create_time()) for k in childpids]) 135 | #print buff 136 | self.transport.signalProcess(9) 137 | for j in childpids: 138 | try: 139 | _process = psutil.Process(j.pid) 140 | except: 141 | pass 142 | else: 143 | if _process.is_running() and j.create_time() == buff[j.pid]: 144 | _process.send_signal(9) 145 | self.running = False 146 | return True 147 | 148 | def restart(self): 149 | self.kill() 150 | self.run() 151 | 152 | 153 | class Process(object): 154 | def __init__(self, fun, fun_args, fun_kwargs, callback, errback, stdout_callback=None, stderr_callback=None): 155 | self.kwargs = locals() 156 | del self.kwargs["self"] 157 | self.current_process = None 158 | 159 | def run(self): 160 | if self.current_process: 161 | self.current_process.kill() 162 | self.current_process = ProcessProtocol(**self.kwargs) 163 | self.current_process.run() 164 | 165 | def kill(self): 166 | if self.current_process: 167 | self.current_process.kill() 168 | 169 | def restart(self): 170 | self.run() 171 | 172 | def is_run(self): 173 | if self.current_process: 174 | return self.current_process.is_run() 175 | else: 176 | return False 177 | 178 | 179 | def _run_in_sub(fun, fun_args, fun_kwargs, timeout=None): 180 | p = Process(fun, fun_args, fun_kwargs, callback=lambda x: None, errback=lambda x: None) 181 | p.run() 182 | if timeout: 183 | reactor.callLater(timeout, p.kill) 184 | 185 | 186 | def run_in_process(fun, fun_args=[], fun_kwargs={}, timeout=None): 187 | p = Process(fun, fun_args, fun_kwargs, callback=lambda *x: None, errback=lambda *x: None) 188 | p.run() 189 | if timeout: 190 | reactor.callLater(timeout, p.kill) 191 | 192 | 193 | MASTER_IP = '' 194 | WORK_DIR = '' 195 | 196 | 197 | class MainRpc(xmlrpc.XMLRPC): 198 | 199 | def __init__(self, master_ip, work_dir): 200 | xmlrpc.XMLRPC.__init__(self, False, False) 201 | global MASTER_IP, WORK_DIR 202 | self.work_dir = work_dir 203 | self.maser_ip = master_ip 204 | WORK_DIR = work_dir 205 | MASTER_IP = master_ip 206 | self.cache = {} 207 | 208 | def xmlrpc_cache_set(self,key,value): 209 | self.cache[key] = value 210 | return True 211 | 212 | def xmlrpc_cache_get(self,key): 213 | return self.cache.get(key,None) 214 | 215 | def xmlrpc_cache_del(self,key): 216 | try: 217 | del self.cache[key] 218 | except: 219 | pass 220 | 221 | def valid_cronrule(self, rule): 222 | rule = rule.strip() 223 | try: 224 | CronSchedule(rule) 225 | except: 226 | return False, "时间规则不符合要求" 227 | else: 228 | return True 229 | 230 | def xmlrpc_ping(self, *args, **kwargs): 231 | return True 232 | 233 | def xmlrpc_cron_get(self, key): 234 | if CRON_BUFF.has_key(key): 235 | return True 236 | else: 237 | return False 238 | 239 | def xmlrpc_cron_del(self, key): 240 | print "cron server _cron del" 241 | if CRON_BUFF.has_key(key): 242 | CRON_BUFF[key][0].stop() 243 | CRON_BUFF[key][1].kill() 244 | del CRON_BUFF[key] 245 | return True 246 | else: 247 | return True 248 | 249 | def xmlrpc_cron_is_running(self, key): 250 | if CRON_BUFF.has_key(key): 251 | return CRON_BUFF[key][1].is_run() 252 | else: 253 | return False 254 | 255 | def xmlrpc_cron_set(self, key, cron_mode, picked_function, picked_callback, picked_errback, stdout_callback, 256 | stderr_callback): 257 | print "cron server _ cron set" 258 | try: 259 | self.xmlrpc_cron_del(key) 260 | if self.valid_cronrule(cron_mode): 261 | pass 262 | else: 263 | raise Exception("cron时间规则不符合要求") 264 | try: 265 | target = cloudpickle.loads(picked_function.data) 266 | except: 267 | traceback.print_exc() 268 | raise Exception("目标函数picked反序列化失败!") 269 | try: 270 | callback = cloudpickle.loads(picked_callback.data) 271 | except: 272 | raise Exception("success回调函数picked反序列化失败!") 273 | try: 274 | errback = cloudpickle.loads(picked_errback.data) 275 | except: 276 | raise Exception("error回调函数picked反序列化失败!") 277 | if stdout_callback: 278 | try: 279 | stdout_callback = cloudpickle.loads(stdout_callback.data) 280 | except: 281 | raise Exception("stdout回调函数picked反序列化失败!") 282 | else: 283 | stdout_callback = None 284 | if stderr_callback: 285 | try: 286 | stderr_callback = cloudpickle.loads(stderr_callback.data) 287 | except: 288 | raise Exception("stderr回调函数picked反序列化失败!") 289 | else: 290 | stderr_callback = None 291 | cron = CronSchedule(cron_mode) 292 | 293 | _process = Process(target, [], {}, callback, errback, stdout_callback, stderr_callback) 294 | _task = ScheduledCall(_process.restart) 295 | _task.start(cron) 296 | CRON_BUFF[key] = (_task, _process) 297 | except Exception as e: 298 | traceback.print_exc() 299 | return False, str(e) 300 | else: 301 | return True 302 | 303 | def xmlrpc_cron_run_now(self, key): 304 | result = self.xmlrpc_cron_get(key) 305 | if result: 306 | CRON_BUFF[key][1].restart() 307 | return True 308 | else: 309 | return False, "此cron不存在！" 310 | 311 | def xmlrpc_task_set(self, key, picked_function, picked_callback, picked_errback, stdout_callback, 312 | stderr_callback, timeoutback=None, timeout=60): 313 | print "task set" 314 | try: 315 | self.xmlrpc_task_del(key) 316 | try: 317 | target = cloudpickle.loads(picked_function.data) 318 | except: 319 | traceback.print_exc() 320 | raise Exception("目标函数picked反序列化失败!") 321 | try: 322 | callback = cloudpickle.loads(picked_callback.data) 323 | except: 324 | traceback.print_exc() 325 | raise Exception("success回调函数picked反序列化失败!") 326 | try: 327 | errback = cloudpickle.loads(picked_errback.data) 328 | except: 329 | raise Exception("error回调函数picked反序列化失败!") 330 | if stdout_callback: 331 | try: 332 | stdout_callback = cloudpickle.loads(stdout_callback.data) 333 | except: 334 | raise Exception("stdout回调函数picked反序列化失败!") 335 | else: 336 | stdout_callback = None 337 | if stderr_callback: 338 | try: 339 | stderr_callback = cloudpickle.loads(stderr_callback.data) 340 | except: 341 | raise Exception("stderr回调函数picked反序列化失败!") 342 | else: 343 | stderr_callback = None 344 | _process = Process(target, [], {}, callback, errback, stdout_callback, stderr_callback) 345 | _process.run() 346 | if timeout: 347 | def _(): 348 | if _process.is_run() and timeoutback: 349 | run_in_process(cloudpickle.loads(timeoutback.data)) 350 | 351 | reactor.callLater(timeout, _) 352 | TASK_BUFF[key] = _process 353 | except Exception as e: 354 | traceback.print_exc() 355 | print str(e) 356 | return False, str(e) 357 | else: 358 | return True 359 | 360 | def xmlrpc_task_del(self, key): 361 | if TASK_BUFF.has_key(key): 362 | TASK_BUFF[key].kill() 363 | del TASK_BUFF[key] 364 | return True 365 | else: 366 | return True 367 | 368 | def xmlrpc_task_is_running(self, key): 369 | if TASK_BUFF.has_key(key): 370 | return TASK_BUFF[key].is_run() 371 | else: 372 | return False 373 | 374 | def xmlrpc_deamon_set(self, key, picked_function, picked_callback, picked_errback, stdout_callback, 375 | stderr_callback): 376 | print "task set" 377 | try: 378 | self.xmlrpc_deamon_del(key) 379 | try: 380 | target = cloudpickle.loads(picked_function.data) 381 | except: 382 | traceback.print_exc() 383 | raise Exception("目标函数picked反序列化失败!") 384 | try: 385 | callback = cloudpickle.loads(picked_callback.data) 386 | except: 387 | traceback.print_exc() 388 | raise Exception("success回调函数picked反序列化失败!") 389 | try: 390 | errback = cloudpickle.loads(picked_errback.data) 391 | except: 392 | raise Exception("error回调函数picked反序列化失败!") 393 | if stdout_callback: 394 | try: 395 | stdout_callback = cloudpickle.loads(stdout_callback.data) 396 | except: 397 | raise Exception("stdout回调函数picked反序列化失败!") 398 | else: 399 | stdout_callback = None 400 | if stderr_callback: 401 | try: 402 | stderr_callback = cloudpickle.loads(stderr_callback.data) 403 | except: 404 | raise Exception("stderr回调函数picked反序列化失败!") 405 | else: 406 | stderr_callback = None 407 | _process = Process(target, [], {}, callback, errback, stdout_callback, stderr_callback) 408 | _process.run() 409 | print "task process run" 410 | DEAMON_BUFF[key] = _process 411 | except Exception as e: 412 | traceback.print_exc() 413 | print str(e) 414 | return False, str(e) 415 | else: 416 | return True 417 | 418 | def xmlrpc_deamon_get(self, key): 419 | if DEAMON_BUFF.has_key(key): 420 | return True 421 | else: 422 | return False 423 | 424 | def xmlrpc_deamon_del(self, key): 425 | if DEAMON_BUFF.has_key(key): 426 | DEAMON_BUFF[key].kill() 427 | del DEAMON_BUFF[key] 428 | return True 429 | else: 430 | return True 431 | 432 | def xmlrpc_deamon_is_running(self, key): 433 | if DEAMON_BUFF.has_key(key): 434 | return DEAMON_BUFF[key].is_run() 435 | else: 436 | return False 437 | 438 | def xmlrpc_deamon_run_now(self, key): 439 | result = self.xmlrpc_deamon_get(key) 440 | if result: 441 | DEAMON_BUFF[key].restart() 442 | else: 443 | return False, "此后台任务不存在！" 444 | 445 | 446 | num = 0 447 | 448 | 449 | def check_deamon_run(): 450 | for i, j in DEAMON_BUFF.items(): 451 | if j.is_run(): 452 | pass 453 | else: 454 | j.run() 455 | reactor.callLater(30, check_deamon_run) 456 | 457 | reactor.callLater(5, check_deamon_run) 458 | 459 | def heartbeat(): 460 | global num 461 | num += 1 462 | _defer = getPage( 463 | "http://%s:9912/core_api/work_heartbeat/?num=%s&work_dir=%s" % (MASTER_IP, num, urllib.quote(WORK_DIR))) 464 | 465 | def callback(*args): 466 | print "work_heartbeat success!" 467 | 468 | def errback(*args): 469 | print "work_heartbeat failed! master maybe down!!" 470 | _defer.addCallback(callback) 471 | _defer.addErrback(errback) 472 | reactor.callLater(3, heartbeat) 473 | 474 | reactor.callLater(1, heartbeat) 475 | 476 | 477 | -------------------------------------------------------------------------------- /cap_twisted/worker.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | # write by zhou 3 | 4 | #coding:utf-8 5 | # write by zhou 6 | 7 | import base64 8 | import json 9 | import cloudpickle 10 | import traceback 11 | import sys 12 | import time 13 | # from django.conf import settings 14 | # settings.configure() 15 | reload(sys) 16 | sys.setdefaultencoding("utf-8") 17 | time.sleep(0.1) 18 | _=raw_input() 19 | _=_.strip() 20 | data=cloudpickle.loads(base64.b64decode(_)) 21 | fun,callback,errback=data["fun"],data["callback"],data["errback"] 22 | args,kwargs=data["args"],data["kwargs"] 23 | 24 | return_code=0 25 | try: 26 | result=fun(*args,**kwargs) 27 | except Exception as e : 28 | error_msg= "fun exception:"+str(e) 29 | sys.stderr.write(error_msg) 30 | traceback.print_exc() 31 | return_code=1 32 | try: 33 | errback(str(e)) 34 | except Exception as e : 35 | error_msg= "errback exception:"+str(e) 36 | sys.stderr.write(error_msg) 37 | traceback.print_exc() 38 | pass 39 | else: 40 | try: 41 | callback(result) 42 | except Exception as e : 43 | error_msg= "callback exception:"+str(e) 44 | sys.stderr.write(error_msg) 45 | traceback.print_exc() 46 | return_code=1 47 | sys.exit(return_code) 48 | 49 | 50 | 51 | -------------------------------------------------------------------------------- /docs/cap1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhoukunpeng504/cap-python/c5ca57db90c6741485d590f644a724164f8c34cd/docs/cap1.png -------------------------------------------------------------------------------- /docs/cap2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhoukunpeng504/cap-python/c5ca57db90c6741485d590f644a724164f8c34cd/docs/cap2.png -------------------------------------------------------------------------------- /docs/cap3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhoukunpeng504/cap-python/c5ca57db90c6741485d590f644a724164f8c34cd/docs/cap3.png -------------------------------------------------------------------------------- /docs/cap4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhoukunpeng504/cap-python/c5ca57db90c6741485d590f644a724164f8c34cd/docs/cap4.png -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | # write by zhou 3 | from setuptools import setup,find_packages 4 | setup( 5 | name="cap-python", 6 | packages=list(set(list(find_packages())+["twisted.plugins"])), 7 | version='0.1.15', 8 | install_requires=["setuptools", 9 | "MySQL-python", 10 | "twisted==15.3.0", 11 | "txscheduling", 12 | "psutil", 13 | "django==1.4.22", 14 | 'netifaces', 15 | 'PyCrypto==2.6', 16 | 'cloudpickle', 17 | "gittle==0.5.0", 18 | "pyquery"], 19 | include_package_data=True, 20 | url="https://github.com/zhoukunpeng504/cap-python", 21 | author='zhoukunpeng', 22 | author_email="18749679769@163.com", 23 | entry_points={ 24 | 'console_scripts': ['cap-all-start = cap.sbin.all_start:main', 25 | 'cap-all-stop = cap.sbin.all_stop:main', 26 | 'cap-worker-start = cap.sbin.worker_start:main', 27 | 'cap-worker-stop = cap.sbin.worker_stop:main'] 28 | } 29 | ) 30 | -------------------------------------------------------------------------------- /twisted/plugins/cap_all_plugin.py: -------------------------------------------------------------------------------- 1 | # coding:utf-8 2 | __author__ = 'zhoukunpeng' 3 | # -------------------------------- 4 | # Created by zhoukunpeng on 2015/7/20. 5 | # --------------------------------- 6 | from zope.interface import implements 7 | from twisted.python import usage 8 | from twisted.plugin import IPlugin 9 | from twisted.application.service import IServiceMaker, MultiService, Application 10 | from twisted.application.internet import TCPServer 11 | import sys 12 | from twisted.web import server 13 | import os 14 | import netifaces 15 | import re 16 | from twisted.internet import reactor 17 | from twisted.web.wsgi import WSGIResource 18 | 19 | reload(sys) 20 | sys.setdefaultencoding("utf-8") 21 | 22 | 23 | def get_local_ipaddr(): 24 | results = [] 25 | for i in netifaces.interfaces(): 26 | info = netifaces.ifaddresses(i) 27 | if netifaces.AF_INET not in info: 28 | continue 29 | _ = info[netifaces.AF_INET][0]['addr'] 30 | if _ != "127.0.0.1": 31 | results.append(_) 32 | return results 33 | 34 | 35 | def option_mysql_url_valid(val): 36 | if re.match(r".+?:\d+/.+", val): 37 | return val 38 | else: 39 | raise Exception("%s 不是一个可用的数据库地址。 例子：192.168.8.94:3306/cap_db" % (val)) 40 | 41 | 42 | option_mysql_url_valid.coerceDoc = "数据库地址， 例如：192.168.8.94:3306/cap_db" 43 | 44 | 45 | def option_mysql_user_valid(val): 46 | return val 47 | 48 | 49 | option_mysql_user_valid.coerceDoc = "数据库用户， 例如：admin" 50 | 51 | 52 | def option_mysql_password_valid(val): 53 | return val 54 | 55 | 56 | option_mysql_password_valid.coerceDoc = "数据库密码， 例如：123456" 57 | 58 | 59 | def option_host_valid(val): 60 | if val in get_local_ipaddr(): 61 | return val 62 | else: 63 | raise Exception("%s 不是一个可用的地址, 本机所有可绑定的地址如下 %s" % (val, ",".join(get_local_ipaddr()))) 64 | 65 | 66 | option_host_valid.coerceDoc = "服务绑定的IP地址，本机所有可绑定的IP地址如下 %s" % ( 67 | ",".join(get_local_ipaddr())) 68 | 69 | 70 | def option_work_dir_valid(val): 71 | if val[0] != '/': 72 | raise Exception("%s 必须是一个绝对路径,比如 /data/crondeamon_work ") 73 | try: 74 | assert os.path.exists(val) 75 | except: 76 | raise Exception("%s 不是一个可用的目录,请检查目录是否存在及权限是否正确！" % val) 77 | return val 78 | 79 | 80 | option_work_dir_valid.coerceDoc = "服务的工作目录" 81 | 82 | 83 | class Options(usage.Options): 84 | optParameters = [ 85 | ["mysql_url", "l", None, None, option_mysql_url_valid], 86 | ["mysql_user", "r", None, None, option_mysql_user_valid], 87 | ["mysql_password", 'd', None, None, option_mysql_password_valid], 88 | ["host", "h", None, None, option_host_valid], 89 | ["work_dir", "w", None, None, option_work_dir_valid] 90 | ] 91 | 92 | 93 | class AServiceMaker(object): 94 | implements(IServiceMaker, IPlugin) 95 | tapname = "cap-all" 96 | description = "cap-all服务" 97 | options = Options 98 | 99 | def makeService(self, options): 100 | config = options 101 | import cap 102 | from cap_twisted import service as mainrpc 103 | import sys 104 | s = MultiService() 105 | sys.path.insert(1, cap.__path__[0]) 106 | del sys.modules["cap"] 107 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "cap.settings") 108 | mysql_url = options["mysql_url"].strip() 109 | try: 110 | a, b = mysql_url.split(":") 111 | mysql_host = a 112 | mysql_port, mysql_db = b.split("/") 113 | mysql_port = int(mysql_port) 114 | except: 115 | print "mysql相关配置错误" 116 | raise Exception("mysql相关配置错误") 117 | else: 118 | mysql_user = options["mysql_user"] 119 | mysql_password = options["mysql_password"] 120 | os.config = [mysql_host, mysql_port, mysql_db, mysql_user, mysql_password] 121 | os.work_dir = options["work_dir"] 122 | os.host = options["host"] 123 | from django.core.handlers.wsgi import WSGIHandler 124 | application = WSGIHandler() 125 | resource = WSGIResource(reactor, reactor.getThreadPool(), application) 126 | ui_service = TCPServer(9912, server.Site(resource), interface=config["host"]) 127 | serverfactory = server.Site(mainrpc.MainRpc(config["host"], config["work_dir"])) 128 | slave_service = TCPServer(9913, serverfactory, interface=config["host"]) 129 | slave_service.setServiceParent(s) 130 | ui_service.setServiceParent(s) 131 | return s 132 | 133 | 134 | serviceMaker = AServiceMaker() 135 | -------------------------------------------------------------------------------- /twisted/plugins/cap_worker_plugin.py: -------------------------------------------------------------------------------- 1 | # coding:utf-8 2 | __author__ = 'zhoukunpeng' 3 | # -------------------------------- 4 | # Created by zhoukunpeng on 2015/7/20. 5 | # --------------------------------- 6 | from zope.interface import implements 7 | from twisted.python import usage 8 | from twisted.plugin import IPlugin 9 | from twisted.application.service import IServiceMaker, MultiService,Application 10 | from twisted.application.internet import TCPServer 11 | import sys 12 | from twisted.web import server 13 | import os 14 | import netifaces 15 | from twisted.python.logfile import DailyLogFile,LogFile 16 | 17 | reload(sys) 18 | sys.setdefaultencoding("utf-8") 19 | 20 | 21 | def get_local_ipaddr(): 22 | results = [] 23 | for i in netifaces.interfaces(): 24 | info = netifaces.ifaddresses(i) 25 | if netifaces.AF_INET not in info: 26 | continue 27 | _ = info[netifaces.AF_INET][0]['addr'] 28 | if _ != "127.0.0.1": 29 | results.append(_) 30 | return results 31 | 32 | 33 | def option_master_valid(val): 34 | return val 35 | 36 | 37 | option_master_valid.coerceDoc = "主节点的IP地址, 比如 192.168.11.1" 38 | 39 | 40 | def option_host_valid(val): 41 | if val in get_local_ipaddr(): 42 | return val 43 | else: 44 | raise Exception("%s 不是一个可用的地址, 本机所有可绑定的地址如下 %s" % (val, ",".join(get_local_ipaddr()))) 45 | 46 | 47 | option_host_valid.coerceDoc = "服务绑定的IP地址，本机所有可绑定的IP地址如下 %s" % ( 48 | ",".join(get_local_ipaddr())) 49 | 50 | def option_work_dir_valid(val): 51 | if val[0]!='/': 52 | raise Exception("%s 必须是一个绝对路径,比如 /data/crondeamon_work ") 53 | try: 54 | assert os.path.exists(val) 55 | except: 56 | raise Exception("%s 不是一个可用的目录,请检查目录是否存在及权限是否正确！"%val) 57 | return val 58 | 59 | 60 | option_work_dir_valid.coerceDoc = "服务的工作目录" 61 | 62 | class Options(usage.Options): 63 | optParameters = [ 64 | ["master", "m", None, None, option_master_valid], 65 | ["host", "h", None, None, option_host_valid], 66 | ["work_dir","w",None,None,option_work_dir_valid] 67 | ] 68 | 69 | 70 | class MyServiceMaker(object): 71 | implements(IServiceMaker, IPlugin) 72 | tapname = "cap-worker" 73 | description = "cap-worker服务，用于cap系统。" 74 | options = Options 75 | 76 | def makeService(self, options): 77 | config = options 78 | s = MultiService() 79 | 80 | from cap_twisted import service as mainrpc 81 | serverfactory = server.Site(mainrpc.MainRpc(config["master"],config["work_dir"])) 82 | slave_service = TCPServer(9913, serverfactory, interface=config["host"]) 83 | slave_service.setServiceParent(s) 84 | return s 85 | 86 | 87 | serviceMaker = MyServiceMaker() 88 | 89 | --------------------------------------------------------------------------------