├── README.md ├── doc └── README.md └── trunk ├── README.md ├── fs_agent ├── README.md ├── agent │ └── fs_main.py ├── base │ ├── __init__.py │ ├── fs_base_cfg.py │ ├── fs_log.py │ └── fs_syshead.py ├── bean │ ├── det_dan_func.py │ ├── det_file_att.py │ ├── det_fuzz_hash.py │ ├── det_statis_tics.py │ ├── det_web_log.py │ ├── fsa_task.py │ └── fsa_task_type.py ├── conf │ └── fshell_agent.conf └── net │ ├── fsa_net.py │ └── fsa_nethead.py ├── fs_datahandle └── README.md ├── fs_kernel └── README.md ├── fs_manager ├── README.md └── src │ └── base │ ├── fs_base_cfg.py │ ├── fs_database_pid.py │ ├── fs_email.py │ ├── fs_log.py │ ├── fs_mongodb_pid.py │ ├── fs_process.py │ ├── fs_syshead.py │ ├── fs_time.py │ └── fs_util.py └── fs_server ├── conf └── README.md └── src ├── base ├── fs_base_cfg.py ├── fs_database_pid.py ├── fs_log.py ├── fs_syshead.py ├── fs_time.py └── fs_util.py ├── bean └── README.md ├── dao └── README.md ├── fss_cfg.py ├── net ├── fss_net.py └── fss_nethead.py └── route ├── fss_proto_type.py ├── fss_srv.py ├── fss_srv_base.py └── fss_srv_manager.py /README.md: -------------------------------------------------------------------------------- 1 | ### fshell 2 | -------------------------------------------------------------------------------- /doc/README.md: -------------------------------------------------------------------------------- 1 | ### Document Of Fshell v0.1 2 | 3 | #### 基于机器学习的分布式Webshell检测系统-绪论篇 4 | http://www.s0nnet.com/archives/fshell 5 | 6 | 7 | #### 基于机器学习的分布式webshell检测系统-特征工程（1） 8 | http://www.s0nnet.com/archives/fshell-feature-1 9 | -------------------------------------------------------------------------------- /trunk/README.md: -------------------------------------------------------------------------------- 1 | ### Module Introduce v0.1 2 | 3 | #### fs_agent 客户端agent模块 4 | 5 | 6 | #### fs_datahandle 特征向量化（预处理）模块 7 | 8 | 9 | #### fs_manager 服务端管理模块 10 | 11 | #### fs_server 服务端数据接入模块 12 | -------------------------------------------------------------------------------- /trunk/fs_agent/README.md: -------------------------------------------------------------------------------- 1 | fshell agent 2 | -------------------------------------------------------------------------------- /trunk/fs_agent/agent/fs_main.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # project: fshell 4 | # author: s0nnet 5 | # time: 2016-12-12 6 | # desc: fshell agent main 7 | 8 | 9 | import os 10 | import sys 11 | sys.path.append("../base") 12 | sys.path.append("../bean") 13 | sys.path.append("../net") 14 | 15 | from fsa_task import * 16 | from fsa_task_type import * 17 | from det_web_log import * 18 | from det_statis_tics import * 19 | from det_file_att import * 20 | from det_dan_func import * 21 | from det_fuzz_hash import * 22 | 23 | 24 | class FsaMain: 25 | 26 | def __init__(self): 27 | 28 | self.taskPid = 0 29 | self.task = None 30 | 31 | 32 | def _task_run(self, taskType): 33 | 34 | if taskType == FsaTaskType.F_WEBLOG: 35 | taskWeblog = FsaTaskWeblog() 36 | taskWeblogTid = threading.Thread(target = taskWeblog.start_task) 37 | taskWeblogTid.start() 38 | 39 | elif taskType == FsaTaskType.F_STATICS: 40 | taskStatics = FsaTaskStatics() 41 | taskStaticsTid = threading.Thread(target = taskStatics.start_task) 42 | taskStaticsTid.start() 43 | 44 | elif taskType == FsaTaskType.F_FILEATT: 45 | taskFileatt = FsaTaskFileatt() 46 | taskFileattTid = threading.Thread(target = taskFileatt.start_task) 47 | taskFileattTid.start() 48 | 49 | elif taskType == FsaTaskType.F_DANFUNC: 50 | taskDanfunc = FsaTaskDanfunc() 51 | taskDanfuncTid = threading.Thread(target = taskDanfunc.start_task) 52 | taskDanfuncTid.start() 53 | 54 | elif taskType == FsaTaskType.F_FUZZHASH: 55 | taskFuzzhash = FsaTaskFuzzhash() 56 | taskFuzzhashTid = threading.Thread(target = taskFuzzhash.start_task) 57 | taskFuzzhashTid.start() 58 | 59 | else: 60 | return False 61 | 62 | return True 63 | 64 | 65 | def _task_start(self, taskType): 66 | self.taskType = taskType 67 | 68 | pid = os.fork() 69 | if pid == -1: 70 | Log.err("os.fork err!") 71 | return False 72 | 73 | if pid == 0: 74 | retCode = 0 75 | try: 76 | bRun = self._task_run(taskType) 77 | if not bRun: retCode = -1 78 | except Exception, e: 79 | Log.err("taskType: %s, err: %s" %(str(taskType), str(traceback.format_exc()))) 80 | retCode = -1 81 | 82 | if retCode == -1: 83 | FsaTaskClient.report_task(self.taskType, FsaTaskStatus.T_FAIL, "task_exec ERR!") 84 | else: 85 | FsaTaskClient.report_task(self.taskType, FsaTaskStatus.T_FINISH, "sdsd") 86 | 87 | os._exit(retCode) 88 | 89 | self.taskPid = pid 90 | return True 91 | 92 | 93 | def _task_check(self): 94 | if not self.taskPid: return False 95 | 96 | try: 97 | os.waitpid(self.taskPid, os.WNOHANG) 98 | except Exception, e: 99 | self.taskPid = 0 100 | return False 101 | 102 | return True 103 | 104 | 105 | def run(self): 106 | count = 0 107 | while True: 108 | 109 | if self.taskPid > 0: 110 | if self._task_check() == False: 111 | count += 1 112 | 113 | else: 114 | bRet, taskType = FsaTaskClient.get_task() 115 | if bRet: 116 | if taskType != None: 117 | try: 118 | bRet = self._task_start(taskType) 119 | if not bRet: 120 | Log.err("task_start(%s) ERR" %(str(taskType))) 121 | FsaTaskClient.report_task(taskType, FsaTaskStatus.T_FAIL, "task_start Fail!") 122 | 123 | except Exception, e: 124 | Log.err("taskType: %s, %s" %(taskType, traceback.format_exc())) 125 | FsaTaskClient.report_task(taskType, FsaTaskStatus.T_FAIL, "task_start Fail!") 126 | 127 | time.sleep(3) 128 | 129 | 130 | 131 | if __name__ == "__main__": 132 | 133 | def main(): 134 | fsaMain = FsaMain() 135 | fsaMain.run() 136 | 137 | main() 138 | 139 | -------------------------------------------------------------------------------- /trunk/fs_agent/base/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /trunk/fs_agent/base/fs_base_cfg.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # project: fshell 4 | # author: s0nnet 5 | # time: 2016-12-10 6 | # desc: agent base conf 7 | 8 | ''' 9 | LOG_LEVEL 10 | DEBUG = 1 11 | INFO = 2 12 | WARINING = 3 13 | ERR = 4 14 | ALERT = 5 15 | CLOSE = 10 16 | ''' 17 | 18 | 19 | import os 20 | path = os.path.dirname(os.path.realpath(__file__)) 21 | os.chdir(path) 22 | 23 | import ConfigParser 24 | 25 | 26 | class AgentConf: 27 | 28 | def __init__(self): 29 | cfgFile = path + '/../conf/fshell_agent.conf' 30 | self.conf = ConfigParser.ConfigParser() 31 | self.conf.read(cfgFile) 32 | 33 | 34 | def get_base_conf(self, option): 35 | try: 36 | return self.conf.get("BASE", option) 37 | except: 38 | return None 39 | 40 | 41 | def get_weblog_conf(self, option): 42 | try: 43 | return self.conf.get("WEBLOG", option) 44 | except: 45 | return None 46 | 47 | 48 | def get_statics_conf(option): 49 | try: 50 | return self.conf.get("STATICS", option) 51 | except: 52 | return None 53 | 54 | 55 | def get_fileatt_conf(option): 56 | try: 57 | return self.conf.get("FILEATT", option) 58 | except: 59 | return None 60 | 61 | 62 | def get_danfunc_conf(option): 63 | try: 64 | return self.conf.get("DANFUNC", option) 65 | except: 66 | return None 67 | 68 | 69 | def get_fuzzhash_conf(option): 70 | try: 71 | return self.conf.get("FUZZHASH", option) 72 | except: 73 | return None 74 | 75 | 76 | 77 | class BaseConf: 78 | 79 | conf = AgentConf() 80 | 81 | LOG_LEVEL = int(conf.get_base_conf("log_level")) 82 | LOG_DIR = conf.get_base_conf("log_dir") 83 | LOG_PREFIX = conf.get_base_conf("prefix") 84 | IS_CTR_LOG = bool(conf.get_base_conf("is_ctr_log")) 85 | 86 | SERVER_IP = conf.get_base_conf("server_ip") 87 | SERVER_PORT = int(conf.get_base_conf("server_port")) 88 | 89 | 90 | 91 | if __name__ == "__main__": 92 | 93 | print BaseConf.SERVER_IP 94 | 95 | 96 | 97 | 98 | -------------------------------------------------------------------------------- /trunk/fs_agent/base/fs_log.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # project: fshell 4 | # author: s0nnet 5 | # time: 2016-12-11 6 | # desc: 标准log模块 7 | 8 | 9 | from fs_syshead import * 10 | 11 | class LogLevel: 12 | DEBUG = 1 13 | INFO = 2 14 | WARNING = 3 15 | ERR = 4 16 | ALERT = 5 17 | CLOSE = 10 18 | 19 | def _get_log_level(logLevel): 20 | if logLevel == LogLevel.DEBUG: return "DEBUG" 21 | if logLevel == LogLevel.INFO: return "INFO" 22 | if logLevel == LogLevel.WARNING: return "WARNING" 23 | if logLevel == LogLevel.ERR: return "ERROR" 24 | if logLevel == LogLevel.ALERT: return "ALERT" 25 | return "CLOSE" 26 | 27 | def _get_log_tm(): 28 | "[01/Jan/2013 04:51:46]" 29 | return (datetime.datetime.now()).strftime("[%d/%m/%Y %H:%M:%S]") 30 | 31 | 32 | def _get_file(f): return f[1] 33 | def _get_function(f): return f[3] 34 | def _get_line(f): return f[2] 35 | def _get_location(stackIdx): 36 | f = inspect.stack()[stackIdx] 37 | return "%s::%s(%d)" %(_get_file(f), _get_function(f), _get_line(f)) 38 | 39 | 40 | class LogFile: 41 | 42 | @staticmethod 43 | def rm_file(path, delflag = False): 44 | 45 | if not path: return 46 | if not os.path.exists(path): return 47 | 48 | if os.path.isfile(path): 49 | os.unlink(path) 50 | return 51 | 52 | fileList = os.listdir(path) 53 | if len(fileList) != 0: 54 | for fname in fileList: 55 | filePath = "%s/%s" %(path, fname) 56 | LogFile.rm_file(filePath, True) 57 | 58 | if delflag: 59 | os.rmdir(path) 60 | 61 | def __init__(self): 62 | #LogFile.rm_file(BaseConf.LOG_DIR) 63 | 64 | self.fp = None; 65 | #self.maxLine = 1024 66 | #self.fIdx = 0 67 | self.maxSzie = 1024 * 1024 68 | self.fName = -1 69 | 70 | def _open_logfile(self): 71 | self.fName += 1 72 | if not os.path.exists(BaseConf.LOG_DIR): 73 | os.makedirs(BaseConf.LOG_DIR) 74 | 75 | curTm = (datetime.datetime.now()).strftime("%Y_%m_%d_%H_%M_%S") 76 | preFix = BaseConf.LOG_PREFIX + "_%s" %(curTm) 77 | #tmpName = "%s/%d.txt" %(BaseConf.LOG_DIR, self.fName) 78 | tmpName = "%s/%s.txt" %(BaseConf.LOG_DIR, preFix) 79 | 80 | try: 81 | if self.fp: self.fp.close() 82 | self.fp = open(tmpName, "a") 83 | #self.fIdx = 0 84 | 85 | os.dup2(self.fp.fileno(), sys.stdout.fileno()) 86 | os.dup2(self.fp.fileno(), sys.stderr.fileno()) 87 | except Exception, e: 88 | sys.stderr.write("%s: %s\n" %(_get_location(1), str(e))) 89 | 90 | def _print_log(self, buf): 91 | try: 92 | #if self.fp == None or self.fIdx >= self.maxLine: 93 | if self.fp == None or self.fp.tell() >= self.maxSzie: 94 | self._open_logfile() 95 | 96 | self.fp.write(buf) 97 | self.fp.flush() 98 | #self.fIdx += 1 99 | 100 | except Exception, e: 101 | sys.stderr.write("%s: %s" %(_get_location(1), str(e))) 102 | sys.stderr.flush() 103 | 104 | logFile = LogFile() 105 | 106 | 107 | class Log: 108 | 109 | @staticmethod 110 | def _loglog(buf, stackIdx, logLevel): 111 | if logLevel >= BaseConf.LOG_LEVEL: 112 | msg = "%s[%s]%s: %s\n" %(_get_log_tm(), _get_log_level(logLevel), _get_location(stackIdx + 1), buf) 113 | if BaseConf.IS_CTR_LOG: sys.stderr.write(msg); sys.stderr.flush() 114 | else: logFile._print_log(msg) 115 | 116 | 117 | @staticmethod 118 | def debug(buf, stackIdx = 1): 119 | Log._loglog(buf, stackIdx + 1, LogLevel.DEBUG) 120 | 121 | @staticmethod 122 | def info(buf, stackIdx = 1): 123 | Log._loglog(buf, stackIdx + 1, LogLevel.INFO) 124 | 125 | @staticmethod 126 | def warning(buf, stackIdx = 1): 127 | Log._loglog(buf, stackIdx + 1, LogLevel.WARNING) 128 | 129 | @staticmethod 130 | def err(buf, stackIdx = 1): 131 | Log._loglog(buf, stackIdx + 1, LogLevel.ERR) 132 | 133 | @staticmethod 134 | def alert(buf, stackIdx = 1): 135 | Log._loglog(buf, stackIdx + 1, LogLevel.ALERT) 136 | 137 | 138 | if __name__ == "__main__": 139 | Log.debug("Test......") 140 | Log.info("Test.......") 141 | Log.err("Test.......") 142 | 143 | 144 | -------------------------------------------------------------------------------- /trunk/fs_agent/base/fs_syshead.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # project: fshell 4 | # author: s0nnet 5 | # time: 2016-12-11 6 | # desc: 相关模块导入 7 | 8 | 9 | 10 | import sys 11 | import os 12 | import time 13 | import datetime 14 | import signal 15 | import inspect 16 | import threading 17 | import thread 18 | import signal 19 | import uuid 20 | import re 21 | import traceback 22 | from multiprocessing import Process 23 | 24 | try: 25 | import json 26 | except Exception, e: 27 | import simplejson as json 28 | 29 | 30 | from fs_base_cfg import * 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /trunk/fs_agent/bean/det_dan_func.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # project: fshell 4 | # author: s0nnet 5 | # time: 2016-12-21 6 | # desc: detection danger func 7 | 8 | 9 | 10 | 11 | 12 | class FsaTaskDanfunc: 13 | 14 | 15 | 16 | 17 | 18 | 19 | def start_task(self): 20 | pass 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /trunk/fs_agent/bean/det_file_att.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # project: fshell 4 | # author: s0nnet 5 | # time: 2016-12-21 6 | # desc: detection file_attribute 7 | 8 | 9 | class FsaTaskFileatt: 10 | 11 | 12 | 13 | 14 | def start_task(self): 15 | 16 | pass 17 | -------------------------------------------------------------------------------- /trunk/fs_agent/bean/det_fuzz_hash.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # project: fshell 4 | # author: s0nnet 5 | # time: 2016-12-21 6 | # desc: detection fuzzy_hashing 7 | 8 | 9 | 10 | 11 | class FsaTaskFuzzhash: 12 | 13 | 14 | 15 | 16 | 17 | def start_task(self): 18 | 19 | pass 20 | -------------------------------------------------------------------------------- /trunk/fs_agent/bean/det_statis_tics.py: -------------------------------------------------------------------------------- 1 | # -*- coding: UTF-8 -*- 2 | 3 | # project: fshell 4 | # author: s0nnet 5 | # time: 2016-12-21 6 | # desc: detecion statistics 7 | 8 | class FsaTaskStatics: 9 | 10 | def start_task(self): 11 | 12 | pass 13 | -------------------------------------------------------------------------------- /trunk/fs_agent/bean/det_web_log.py: -------------------------------------------------------------------------------- 1 | # -*- coding: UTF-8 -*- 2 | 3 | # project: fshell 4 | # author: s0nnet 5 | # time: 2016-12-21 6 | # desc: detection web_log 7 | 8 | 9 | 10 | 11 | import time 12 | 13 | class FsaTaskWeblog: 14 | 15 | def __init__(self): 16 | pass 17 | 18 | 19 | 20 | 21 | 22 | 23 | def _read_content(self, filename=""): 24 | file_seek = self.read_seek(filename) 25 | f = open(filename, "r") 26 | f.seek(file_seek) 27 | while True: 28 | _row = f.readline() 29 | file_seek += len(_row) 30 | self.write_seek(filename, file_seek) 31 | 32 | try: 33 | # here get web_log content ... 34 | req_time = "" 35 | req_data = "" 36 | req_method = "" 37 | req_uri = "" 38 | req_referer = "" 39 | 40 | except: 41 | pass 42 | if not _row: break 43 | f.close() 44 | 45 | 46 | 47 | @staticmethod 48 | def start_task(self): 49 | while True: 50 | self.read_log_file() 51 | if self.read_time_log() == 0: 52 | pass 53 | break 54 | 55 | 56 | -------------------------------------------------------------------------------- /trunk/fs_agent/bean/fsa_task.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # project: fshll 4 | # author: s0nnet 5 | # time: 2016-12-12 6 | # desc: 任务接收/上报模块 7 | 8 | import sys 9 | import uuid 10 | 11 | sys.path.append("../net") 12 | from fsa_net import * 13 | 14 | 15 | 16 | class FsaTaskStatus: 17 | 18 | T_RUN = "run" 19 | T_FAIL = "fail" 20 | T_FINISH = "finish" 21 | 22 | statusList = [T_RUN, T_FAIL, T_FINISH] 23 | 24 | @staticmethod 25 | def valid_check(status): 26 | for st in FsaTaskStatus.statusList: 27 | if st == status: 28 | return True 29 | 30 | return False 31 | 32 | 33 | 34 | class FsaTaskClient: 35 | 36 | @staticmethod 37 | def _send_pkt(taskData): 38 | reqJson = {} 39 | reqJson['task_id'] = uuid.uuid1().get_hex() 40 | reqJson['dev_name'] = 'test' 41 | reqJson['agent_id'] = 1234 42 | reqJson['msg_protocol'] = 3 43 | reqJson['msg_type'] = 22 44 | reqJson['data'] = taskData 45 | 46 | return FsaNet.send_req(reqJson) 47 | 48 | 49 | @staticmethod 50 | def get_task(): 51 | 52 | # read from local conf file 53 | 54 | 55 | bRet = True 56 | taskType = "web_log" 57 | 58 | return bRet, taskType 59 | 60 | 61 | @staticmethod 62 | def report_task(taskType, taskStatus, taskData): 63 | if FsaTaskStatus.valid_check(taskStatus) == False: 64 | Log.err("status(%d) is not valid" %(taskStatus)) 65 | return False, "status(%d) is not valid" %(taskStatus) 66 | 67 | if taskData == None: taskData = taskStatus 68 | 69 | # 预留　结果本地缓存模块 70 | # 71 | # 72 | # 73 | # 74 | # 75 | 76 | return FsaTaskClient._send_pkt(taskData) 77 | 78 | 79 | if __name__ == "__main__": 80 | 81 | bRet, taskType = FsaTaskClient.get_task() 82 | 83 | print bRet, taskType 84 | 85 | print FsaTaskClient.report_task("web_log", FsaTaskStatus.T_RUN, "askef xx") 86 | 87 | -------------------------------------------------------------------------------- /trunk/fs_agent/bean/fsa_task_type.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # project: fshell 4 | # author: s0nnet 5 | # time: 2016-12-21 6 | # desc: task type 7 | 8 | 9 | class FsaTaskType: 10 | 11 | F_WEBLOG = "web_log" 12 | F_STATICS = "statis_tics" 13 | F_FILEATT = "file_att" 14 | F_DANFUNC = "dan_func" 15 | F_FUZZHASH = "fuzz_hash" 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /trunk/fs_agent/conf/fshell_agent.conf: -------------------------------------------------------------------------------- 1 | # Fshell v1.0 release. 2 | # This is the conf file of fshell agent. 3 | # For more information, see 4 | # if have any question, mailto: s0nnet@qq.com. 5 | 6 | 7 | [BASE] 8 | server_ip = 127.0.0.1 9 | server_port = 8000 10 | agent_id = 192.168.3.12 11 | dev_name = xupt_web_01 12 | web_enum = php 13 | cache_dir = /home/fshell/fs_agent/cache 14 | log_dir = /home/fshell/fs_agent/log 15 | log_level = 1 16 | log_prefix = 17 | is_ctr_log = 1 18 | 19 | 20 | [WEBLOG] 21 | web_log = /var/log/httpd/xypt_web_01.access.log 22 | 23 | 24 | 25 | [STATICS] 26 | 27 | 28 | [FILEATT] 29 | 30 | 31 | [DANFUNC] 32 | 33 | 34 | [FUZZHASH] 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /trunk/fs_agent/net/fsa_net.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # project: fshell 4 | # author: s0nnet 5 | # time: 2016-12-09 6 | # desc: Agent 网络数据包发送 7 | 8 | import sys 9 | 10 | sys.path.append('../net') 11 | sys.path.append("../base") 12 | 13 | from fs_log import * 14 | from fs_base_cfg import * 15 | from fsa_nethead import * 16 | 17 | 18 | ''' 19 | 跟进参数封装成相应，发给route_srv 20 | ''' 21 | class FsaNet: 22 | 23 | @staticmethod 24 | def _recv_packet(sock): 25 | sock.settimeout(60) 26 | 27 | try: 28 | buf = sock.recv(16 * 1024) 29 | if buf == None or len(buf) == 0: 30 | Log.err("sock.recv ERR!") 31 | return False, "client close connection" 32 | 33 | if len(buf) < NetHead.size(): return False, "recv info error!" 34 | 35 | netHead = NetHead.decode(buf) 36 | if netHead == None: 37 | return False, "NetHead.decode ERR" 38 | 39 | pkgLen = netHead.pkgLen 40 | while len(buf) < pkgLen: 41 | sock.settimeout(3) 42 | val = sock.recv(16 * 1024) 43 | sock.settimeout(None) 44 | if val == None or len(val) == 0: 45 | Log.err("sock.recv ERR!") 46 | return False, "client close connection" 47 | 48 | buf = buf + val 49 | 50 | return True, buf[NetHead.size() : ] 51 | 52 | except Exception, e: 53 | Log.err(traceback.format_exc()) 54 | return False, str(e) 55 | 56 | 57 | @staticmethod 58 | def _send_str_conn(sendStr): 59 | sock = None 60 | try: 61 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM); 62 | code = sock.connect_ex((BaseConf.SERVER_IP, BaseConf.SERVER_PORT)) 63 | if code != 0: 64 | Log.err("sock.connect(%s, %d) ERR(%d)" % (BaseConf.SERVER_IP, BaseConf.SERVER_PORT, code)) 65 | sock.close() 66 | return False, "connect to route_srv Fail!" 67 | 68 | print "send_to_route", len(sendStr) 69 | sock.sendall(sendStr) 70 | bRet, sRet = FsaNet._recv_packet(sock) 71 | Log.info("send_cmd: %s, recv_pack: %s" %(sendStr, sRet)) 72 | sock.close() 73 | return bRet, sRet 74 | 75 | 76 | except Exception, e: 77 | Log.err("sock.connect(%s, %d) ERR(%s)" %(BaseConf.SERVER_IP, BaseConf.SERVER_PORT, str(e))) 78 | if sock != None: sock.close() 79 | return False, str(e) 80 | 81 | 82 | @staticmethod 83 | def _send_str(reqStr): 84 | 85 | netHead = NetHead() 86 | netHead.start = NetHead.START_CODE 87 | netHead.type = 0 88 | netHead.pkgLen = len(reqStr) + NetHead.size() 89 | headStr = NetHead.encode(netHead) 90 | sendStr = headStr + reqStr 91 | 92 | return FsaNet._send_str_conn(sendStr) 93 | 94 | 95 | @staticmethod 96 | def _check_valid(reqJson): 97 | if not reqJson.has_key("task_id"): 98 | return False, "task_id is not exist!" 99 | 100 | if not reqJson.has_key("dev_name"): 101 | return False, "dev_name is not exist!" 102 | 103 | if not reqJson.has_key("agent_id"): 104 | return False, "agent_id is not exist" 105 | 106 | if not reqJson.has_key("msg_protocol"): 107 | return False, "msg_protocol is not exist!" 108 | 109 | if not reqJson.has_key("msg_type"): 110 | return False, "msg_type is not exist!" 111 | 112 | if not reqJson.has_key("data"): 113 | return False, "data is not exist!" 114 | 115 | return True, "" 116 | 117 | 118 | # 发给srv route 119 | @staticmethod 120 | def send_req(reqJson): 121 | 122 | def _get_rsp_info(rspCnt): 123 | try: 124 | rspJson = json.loads(sRet) 125 | if rspJson['data'].find("code") == -1: 126 | Log.err("cmd_rsp: %s is not valid!" %(rspJson['data'])) 127 | return False, rspJson['data'] 128 | rspDataJson = json.loads(rspJson['data']) 129 | 130 | if not rspDataJson.has_key('code'): 131 | return False, rspJson['data'] 132 | if not rspDataJson.has_key('val'): 133 | return False, rspJson['data'] 134 | 135 | code = rspDataJson['code'] 136 | val = rspDataJson['val'] 137 | 138 | if code == 0: return True, val 139 | return False, val 140 | 141 | except Exception, e: 142 | Log.err(traceback.format_exc()) 143 | return False, rspCnt 144 | 145 | 146 | bRet, sRet = FsaNet._check_valid(reqJson) 147 | if not bRet: 148 | Log.err("%s %s" %(str(reqJson), sRet)) 149 | return bRet, sRet 150 | 151 | reqStr = json.dumps(reqJson) 152 | Log.debug(reqStr) 153 | bRet, sRet = FsaNet._send_str(reqStr) 154 | if bRet: 155 | return _get_rsp_info(sRet) 156 | 157 | return bRet, sRet 158 | 159 | 160 | 161 | 162 | if __name__ == "__main__": 163 | 164 | def test_send_data(data): 165 | reqJson = {} 166 | reqJson['task_id'] = '123ver234cv234' 167 | reqJson['dev_name'] = 'xy_01' 168 | reqJson['agent_id'] = 12345 169 | reqJson['msg_protocol'] = 0x03 170 | reqJson['msg_type'] = 0x64 171 | reqJson['data'] = data 172 | 173 | return FsaNet.send_req(reqJson) 174 | 175 | test_send_data("heheda") 176 | -------------------------------------------------------------------------------- /trunk/fs_agent/net/fsa_nethead.py: -------------------------------------------------------------------------------- 1 | # -*- coding: UTF-8 -*- 2 | # 完成网络接收，并反向分发 3 | 4 | import socket 5 | import sys 6 | import os 7 | import time 8 | import threading 9 | import json 10 | 11 | from fs_log import * 12 | 13 | 14 | class NetHead: 15 | 16 | START_CODE = 0x11111111 17 | 18 | def __init__(self): 19 | self.start = 0 20 | self.type = 0 21 | self.pkgLen = 0 22 | 23 | @staticmethod 24 | def size(): 25 | return 9 26 | 27 | @staticmethod 28 | def str2int(s): 29 | if len(s) < 4: return None 30 | 31 | a = s[0 : 1] 32 | b = s[1 : 2] 33 | c = s[2 : 3] 34 | d = s[3 : 4] 35 | 36 | ts = (ord(a) << 24) + (ord(b) << 16) + (ord(c) << 8) + ord(d) 37 | return ts 38 | 39 | @staticmethod 40 | def int2str(n): 41 | 42 | d = n & 0xff; n = n >> 8; 43 | c = n & 0xff; n = n >> 8; 44 | b = n & 0xff; n = n >> 8; 45 | a = n & 0xff; n = n >> 8; 46 | 47 | ts = chr(a) + chr(b) + chr(c) + chr(d) 48 | return ts 49 | 50 | 51 | @staticmethod 52 | def decode(data): 53 | 54 | if data == None or len(data) < 9: 55 | Log.err("data: %s is not valid!" %(str(data))) 56 | return None 57 | 58 | netHead = NetHead() 59 | netHead.start = NetHead.str2int(data[0 : 4]) 60 | netHead.type = ord(data[4 : 5]) 61 | netHead.pkgLen = NetHead.str2int(data[5 : 9]) 62 | 63 | if netHead.start != NetHead.START_CODE: 64 | Log.err("netHead not valid(%d, %d, %d)" %(netHead.start, netHead.type, netHead.pkgLen)); 65 | return None 66 | 67 | Log.debug("netHead(start=%d, type=%d, pkgLen=%d)" %(netHead.start, netHead.type, netHead.pkgLen)); 68 | return netHead 69 | 70 | 71 | @staticmethod 72 | def encode(netHead): 73 | start = NetHead.int2str(netHead.start) 74 | type = chr(netHead.type) 75 | pkgLen = NetHead.int2str(netHead.pkgLen) 76 | cnt = start + type + pkgLen 77 | 78 | Log.debug("netHead_encode: cnt_len: %d" %(len(cnt))) 79 | return cnt 80 | 81 | 82 | 83 | 84 | 85 | -------------------------------------------------------------------------------- /trunk/fs_datahandle/README.md: -------------------------------------------------------------------------------- 1 | The Feature Vector Preconditioning Module 2 | -------------------------------------------------------------------------------- /trunk/fs_kernel/README.md: -------------------------------------------------------------------------------- 1 | The Machine Learning Module 2 | -------------------------------------------------------------------------------- /trunk/fs_manager/README.md: -------------------------------------------------------------------------------- 1 | fshell manager 2 | -------------------------------------------------------------------------------- /trunk/fs_manager/src/base/fs_base_cfg.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # project: fshell 4 | # author: s0nnet 5 | # time: 2016-12-08 6 | # desc: base conf 7 | 8 | 9 | import os 10 | import sys 11 | 12 | ''' 13 | LOG_LEVEL 14 | DEBUG = 1 15 | INFO = 2 16 | WARINING = 3 17 | ERR = 4 18 | ALERT = 5 19 | CLOSE = 10 20 | ''' 21 | class BaseConf: 22 | 23 | LOG_LEVEL = 2 24 | LOG_DIR = "../../log/" 25 | LOG_PREFIX = "" 26 | IS_CTR_LOG = True 27 | 28 | SQL_HOST = "222.24.XX.XX" 29 | SQL_PORT = 3306 30 | SQL_USER = "root" 31 | SQL_PASSWD = "rtxxxx" 32 | SQL_DB = "fshell" 33 | 34 | 35 | # mongodb conf 36 | MG_HOST = "222.24.XX.XX" 37 | MG_PORT = 3306 38 | MG_USER = "root" 39 | MG_PASSWD = "rtxxxx" 40 | MG_DB = "fshell" 41 | 42 | -------------------------------------------------------------------------------- /trunk/fs_manager/src/base/fs_database_pid.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # project: fshell 4 | # author: s0nnet 5 | # time: 2016-12-08 6 | # desc: MySQL基础操作类封装 7 | 8 | 9 | import MySQLdb 10 | from fs_log import * 11 | 12 | class SqlCmdEnum: 13 | COMMIT_CMD = 0 14 | ROLLBACK_CMD = 1 15 | 16 | class SqlExecEnum: 17 | QUERY = 0 18 | UPDATE = 1 19 | INSERT = 2 20 | DELETE = 3 21 | 22 | 23 | def _new_conn(host, port, user, passwd, db, charset): 24 | try: 25 | conn = MySQLdb.connect(host = host, 26 | port = port, 27 | user = user, 28 | passwd = passwd, 29 | db = db, 30 | charset = charset) 31 | conn.autocommit(True) 32 | Log.debug("conn(host: %s, port: %d, user: %s, passwd: %s, db: %s)" %(host, port, user, passwd, db)) 33 | return conn 34 | except Exception, e: 35 | Log.err("ERR(host: %s, port: %d, user: %s, passwd: %s, db: %s, %s)" %(host, port, user, passwd, db, str(e))) 36 | return None 37 | 38 | class DataBase: 39 | 40 | def __init__(self, 41 | host = BaseConf.SQL_HOST, 42 | port = BaseConf.SQL_PORT, 43 | user = BaseConf.SQL_USER, 44 | passwd = BaseConf.SQL_PASSWD, 45 | db = BaseConf.SQL_DB, 46 | charset = None): 47 | self.host = host 48 | self.port = port 49 | self.user = user 50 | self.passwd = passwd 51 | self.db = db 52 | self.charset = charset 53 | 54 | self.conn = None 55 | 56 | def _check(self): 57 | try: 58 | self.conn.ping() 59 | except Exception, e: 60 | self.conn = _new_conn(self.host, self.port, self.user, self.passwd, self.db, self.charset) 61 | 62 | def _get_conn(self): 63 | if self.conn == None: 64 | self.conn = _new_conn(self.host, self.port, self.user, self.passwd, self.db, self.charset) 65 | else: 66 | self._check() 67 | 68 | def _close_conn(self): 69 | if self.conn != None: 70 | self.conn.commit() 71 | self.conn.close() 72 | self.conn = None 73 | 74 | 75 | def _get_record_list(self, cursor): 76 | 77 | def _get_fieldname_list(cursor): 78 | fieldList = [] 79 | for info in cursor.description: 80 | fieldList.append(info[0]) 81 | return fieldList 82 | 83 | fieldnameList = _get_fieldname_list(cursor) 84 | 85 | retNodeList = [] 86 | recordList = cursor.fetchall() 87 | for record in recordList: 88 | node = {} 89 | for i in range(len(fieldnameList)): 90 | node[fieldnameList[i]] = record[i] 91 | retNodeList.append(node) 92 | 93 | return retNodeList 94 | 95 | 96 | def _exec_cmdstr(self, cmdstr, param, rollback, execType): 97 | 98 | def _encode_param(param): 99 | dParam = [] 100 | for par in param: 101 | if type(par) == unicode: 102 | par = par.encode('utf-8') 103 | dParam.append(par) 104 | return dParam 105 | 106 | if param != None: param = _encode_param(param) 107 | Log.debug("%s\t%s" %(cmdstr, str(param))) 108 | 109 | self._get_conn() 110 | conn = self.conn 111 | if conn == None: 112 | Log.err("mysql not connect!") 113 | return False, "mysql not connect!" 114 | 115 | cursor = conn.cursor() 116 | try: 117 | val = cursor.execute(cmdstr, param) 118 | if execType == SqlExecEnum.QUERY: val = self._get_record_list(cursor) 119 | elif execType == SqlExecEnum.INSERT: val = int(conn.insert_id()) 120 | else: pass 121 | except Exception, e: 122 | Log.warning("sql: %s,%s ERR(%s)" %(cmdstr, str(param), str(e))) 123 | if rollback: 124 | self.cmd_submit(SqlCmdEnum.ROLLBACK_CMD) 125 | else: 126 | self._close_conn() 127 | return False, "%s:%s %s" %(cmdstr, str(param), str(e)) 128 | 129 | if not rollback: 130 | self._close_conn() 131 | 132 | return True, val 133 | 134 | def query_data(self, cmdstr, param, rollback = False): 135 | return self._exec_cmdstr(cmdstr, param, rollback, SqlExecEnum.QUERY) 136 | 137 | def update_data(self, cmdstr, param, rollback = False): 138 | return self._exec_cmdstr(cmdstr, param, rollback, SqlExecEnum.UPDATE) 139 | 140 | def insert_data(self, cmdstr, param, rollback = False): 141 | return self._exec_cmdstr(cmdstr, param, rollback, SqlExecEnum.INSERT) 142 | 143 | def delete_data(self, cmdstr, param, rollback = False): 144 | return self._exec_cmdstr(cmdstr, param, rollback, SqlExecEnum.DELETE) 145 | 146 | def cmd_submit(self, flag): 147 | conn = self.conn 148 | if not conn: self._close_conn(); return 149 | 150 | try: 151 | if flag == SqlCmdEnum.COMMIT_CMD: 152 | conn.commit() 153 | elif flag == SqlCmdEnum.ROLLBACK_CMD: 154 | conn.rollback() 155 | finally: 156 | self._close_conn() 157 | 158 | 159 | if __name__ == "__main__": 160 | 161 | dataBase = DataBase() 162 | 163 | cmdstr = "select * from tb_user limit 10" 164 | print dataBase.query_data(cmdstr, None) 165 | -------------------------------------------------------------------------------- /trunk/fs_manager/src/base/fs_email.py: -------------------------------------------------------------------------------- 1 | # -*- coding : utf-8 -*- 2 | 3 | # project: fshell 4 | # author: s0nnet 5 | # time: 2016-12-08 6 | # desc: smtp邮件发送类 7 | 8 | 9 | import sys 10 | import smtplib 11 | import socket 12 | import re 13 | 14 | from fs_log import * 15 | from fs_process import * 16 | 17 | from email.MIMEMultipart import MIMEMultipart 18 | from email.MIMEText import MIMEText 19 | 20 | class Semail: 21 | def __init__(self, userName = "you_name", 22 | passwd = "you_pswd", 23 | smtpAddr = "smtp_srv_ip", 24 | smtpPort = 25, 25 | fromAddr = "", 26 | toAddrs = ""): 27 | self.userName = userName 28 | self.passwd = passwd 29 | self.smtpAddr = smtpAddr 30 | self.smtpPort = smtpPort 31 | self.fromAddr = fromAddr 32 | self.toAddrs = toAddrs 33 | 34 | def send_email_1(self, subject, text): 35 | 36 | cmd = "echo \"%s\" | mail -s \"%s\" %s" %(text, subject, self.toAddrs) 37 | bs_system(cmd) 38 | 39 | def send_email(self, subject, text): 40 | 41 | bRet = True 42 | 43 | text = re.sub("\n", "
", text) 44 | 45 | msg = MIMEMultipart() 46 | msg["From"] = self.fromAddr 47 | msg["To"] = self.toAddrs 48 | msg['Subject'] = subject 49 | msg.attach(MIMEText(text, 'html', 'utf-8')) 50 | 51 | message = msg.as_string() 52 | try: 53 | #s = smtplib.SMTP(self.smtpAddr) 54 | s = smtplib.SMTP(host=self.smtpAddr, timeout= 3) 55 | #s.set_debuglevel(True) 56 | #s.connect(self.smtpAddr, self.smtpPort) 57 | 58 | s.ehlo() 59 | if s.has_extn('STARTTLS'): 60 | s.starttls() 61 | s.ehlo() 62 | 63 | s.login(self.userName, self.passwd) 64 | s.sendmail(self.fromAddr, self.toAddrs.split(","), message) 65 | s.quit() 66 | except Exception, e: 67 | Log.err("send email error: %s" %(str(e))) 68 | bRet = False 69 | 70 | return bRet 71 | 72 | 73 | if __name__ == "__main__": 74 | try: 75 | email = Semail() 76 | email.send_email("test", "
===test===
") 77 | except Exception, e: 78 | print e 79 | 80 | 81 | -------------------------------------------------------------------------------- /trunk/fs_manager/src/base/fs_log.py: -------------------------------------------------------------------------------- 1 | # -*- coding: UTF-8 -*- 2 | 3 | # project: fshell 4 | # author: s0nnet 5 | # time: 2016-12-08 6 | # desc: 日志模块封装 7 | 8 | 9 | from fs_syshead import * 10 | 11 | class LogLevel: 12 | DEBUG = 1 13 | INFO = 2 14 | WARNING = 3 15 | ERR = 4 16 | ALERT = 5 17 | CLOSE = 10 18 | 19 | def _get_log_level(logLevel): 20 | if logLevel == LogLevel.DEBUG: return "DEBUG" 21 | if logLevel == LogLevel.INFO: return "INFO" 22 | if logLevel == LogLevel.WARNING: return "WARNING" 23 | if logLevel == LogLevel.ERR: return "ERROR" 24 | if logLevel == LogLevel.ALERT: return "ALERT" 25 | return "CLOSE" 26 | 27 | def _get_log_tm(): 28 | "[01/Jan/2013 04:51:46]" 29 | return (datetime.datetime.now()).strftime("[%d/%m/%Y %H:%M:%S]") 30 | 31 | 32 | def _get_file(f): return f[1] 33 | def _get_function(f): return f[3] 34 | def _get_line(f): return f[2] 35 | def _get_location(stackIdx): 36 | f = inspect.stack()[stackIdx] 37 | return "%s::%s(%d)" %(_get_file(f), _get_function(f), _get_line(f)) 38 | 39 | 40 | class LogFile: 41 | 42 | @staticmethod 43 | def rm_file(path, delflag = False): 44 | 45 | if not path: return 46 | if not os.path.exists(path): return 47 | 48 | if os.path.isfile(path): 49 | os.unlink(path) 50 | return 51 | 52 | fileList = os.listdir(path) 53 | if len(fileList) != 0: 54 | for fname in fileList: 55 | filePath = "%s/%s" %(path, fname) 56 | LogFile.rm_file(filePath, True) 57 | 58 | if delflag: 59 | os.rmdir(path) 60 | 61 | def __init__(self): 62 | self.fp = None; 63 | self.maxSzie = 1024 * 1024 64 | self.fName = -1 65 | 66 | def _open_logfile(self): 67 | self.fName += 1 68 | if not os.path.exists(BaseConf.LOG_DIR): 69 | os.makedirs(BaseConf.LOG_DIR) 70 | 71 | curTm = (datetime.datetime.now()).strftime("%Y_%m_%d_%H_%M_%S") 72 | preFix = BaseConf.LOG_PREFIX + "_%s" %(curTm) 73 | tmpName = "%s/%s.txt" %(BaseConf.LOG_DIR, preFix) 74 | 75 | try: 76 | if self.fp: self.fp.close() 77 | self.fp = open(tmpName, "a") 78 | 79 | os.dup2(self.fp.fileno(), sys.stdout.fileno()) 80 | os.dup2(self.fp.fileno(), sys.stderr.fileno()) 81 | except Exception, e: 82 | sys.stderr.write("%s: %s\n" %(_get_location(1), str(e))) 83 | 84 | def _print_log(self, buf): 85 | try: 86 | if self.fp == None or self.fp.tell() >= self.maxSzie: 87 | self._open_logfile() 88 | 89 | self.fp.write(buf) 90 | self.fp.flush() 91 | except Exception, e: 92 | sys.stderr.write("%s: %s" %(_get_location(1), str(e))) 93 | sys.stderr.flush() 94 | 95 | logFile = LogFile() 96 | 97 | 98 | class Log: 99 | 100 | @staticmethod 101 | def _loglog(buf, stackIdx, logLevel): 102 | if logLevel >= BaseConf.LOG_LEVEL: 103 | msg = "%s[%s]%s: %s\n" %(_get_log_tm(), _get_log_level(logLevel), _get_location(stackIdx + 1), buf) 104 | if BaseConf.IS_CTR_LOG: sys.stderr.write(msg); sys.stderr.flush() 105 | else: logFile._print_log(msg) 106 | 107 | 108 | @staticmethod 109 | def debug(buf, stackIdx = 1): 110 | Log._loglog(buf, stackIdx + 1, LogLevel.DEBUG) 111 | 112 | @staticmethod 113 | def info(buf, stackIdx = 1): 114 | Log._loglog(buf, stackIdx + 1, LogLevel.INFO) 115 | 116 | @staticmethod 117 | def warning(buf, stackIdx = 1): 118 | Log._loglog(buf, stackIdx + 1, LogLevel.WARNING) 119 | 120 | @staticmethod 121 | def err(buf, stackIdx = 1): 122 | Log._loglog(buf, stackIdx + 1, LogLevel.ERR) 123 | 124 | @staticmethod 125 | def alert(buf, stackIdx = 1): 126 | Log._loglog(buf, stackIdx + 1, LogLevel.ALERT) 127 | 128 | 129 | if __name__ == "__main__": 130 | Log.debug("Test......") 131 | Log.info("Test.......") 132 | Log.err("Test.......") 133 | 134 | 135 | -------------------------------------------------------------------------------- /trunk/fs_manager/src/base/fs_mongodb_pid.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # project: fshell 4 | # author: s0nnet 5 | # time: 2016-12-08 6 | # desc: mongodb连接类底层封装 7 | 8 | import pymongo 9 | from fs_base_cfg import * 10 | 11 | 12 | def _now_conn(host, port, user, passwd, db): 13 | conn = None 14 | server = "mongodb://%s:%s@%s:%s/%s" % (user, passwd, host, port, db) 15 | conn = pymongo.MongoClient(server) 16 | try: 17 | Log.debug("Conn (host: %s, port: %d, user: %s, passwd: %s, db: %s)" % (host, port, user, passwd, db)) 18 | return conn[db] 19 | except Exception, e: 20 | Log.err("Error: (host: %s, port: %d, user: %s, passwd: %s, db: %s)" % (host, port, user, passwd, db)) 21 | return None 22 | 23 | 24 | class DataBase: 25 | def __init__(self, 26 | host = BaseConf.MG_HOST, 27 | port = BaseConf.MGPORT, 28 | user = BaseConf.MG_USER, 29 | passwd = BaseConf.MG_PASSWD, 30 | db = BaseConf.MG_DB): 31 | self.host = host 32 | self.port = port 33 | self.user = user 34 | self.passwd = passwd 35 | self.db = db 36 | 37 | self.conn = None 38 | 39 | def _get_conn(self): 40 | return _now_conn(self.host, self.port, self.user, self.passwd, self.db) 41 | 42 | return self.conn 43 | 44 | 45 | def _close_conn(self): 46 | if self != None: 47 | self.conn.disconnect() 48 | self.conn = None 49 | 50 | 51 | def insert_data(self, coll, datas): 52 | conn = self._get_conn() 53 | coll = conn[coll] 54 | return coll.insert(datas) 55 | 56 | def update_data(self, coll, datas): 57 | col = self.conn[coll] 58 | return col.update({'name':'steven1'},{'$set':{'realname':'测试1修改'}}, False,False) 59 | 60 | 61 | def delete_data(self, coll, datas): 62 | coll = self.conn[coll] 63 | return coll.remove({'name':'steven1'}) 64 | 65 | def query_data(self, coll, datas): 66 | pass 67 | 68 | 69 | 70 | if __name__ == '__main__': 71 | db = DataBase() 72 | datas = [{"username":"s0nnet", "pswd": "qwe@123"}, {"username":"admin", "pswd":"qwe@1234"}] 73 | print db.insert_data('user', datas) 74 | 75 | -------------------------------------------------------------------------------- /trunk/fs_manager/src/base/fs_process.py: -------------------------------------------------------------------------------- 1 | # -*- coding: UTF-8 -*- 2 | 3 | # project: fshell 4 | # author: s0nnet 5 | # time: 2016-12-08 6 | # desc: 进程创建封装 7 | 8 | 9 | from subprocess import * 10 | from fs_log import * 11 | 12 | 13 | def bs_system(cmd, stdout = False, stderr = True): 14 | bRet = False 15 | 16 | if not stderr: cmd = "%s >> /dev/null 2>&1" %(cmd) 17 | elif not stdout: cmd = "%s >> /dev/null" %(cmd) 18 | 19 | try: 20 | retState = os.system(cmd) 21 | except Exception, e: 22 | Log.err("cmd(%s) ERR(%s)" %(cmd, str(e))) 23 | retState = -1 24 | 25 | Log.debug("%s, retstate = %d" %(cmd, retState)) 26 | if retState == 0: bRet = True 27 | return bRet 28 | 29 | 30 | # 管道执行命令， 命令小输出还可以， 如果输出比较大推荐走文件输出流 31 | def exec_cmd(cmd): 32 | 33 | Log.debug("cmd: %s" %(cmd)) 34 | content = "" 35 | try: 36 | p = Popen(cmd, bufsize = 4096, stdout = PIPE, shell = True) 37 | 38 | while True: 39 | cont = p.stdout.read() 40 | if cont == "": break 41 | content += cont 42 | Log.debug("contLen: %d" %(len(cont))) 43 | time.sleep(1) 44 | retState = p.wait() 45 | 46 | return retState, content 47 | except Exception, e: 48 | Log.err("(%s)" %(traceback.format_exc())) 49 | return 255, "cmd(%s) err: %s" %(str(cmd), str(e)) 50 | 51 | 52 | # 进程池 53 | class YgProcessPool: 54 | 55 | def __init__(self, pidNum, fun, args): 56 | 57 | self.pool = [] # 进程池 58 | self.size = pidNum 59 | self.fun = fun 60 | self.args = args 61 | 62 | def start(self, wait = False): 63 | 64 | bRet = True 65 | try: 66 | for i in range(self.size): 67 | pid = os.fork() 68 | if pid == 0: 69 | retCode = 0 70 | try: 71 | bRet = self.fun(i, self.size, self.args) 72 | if not bRet: retCode = -1 73 | except Exception, e: 74 | Log.err("ERR(%s)" %(traceback.format_exc())) 75 | retCode = -1 76 | 77 | os._exit(retCode) 78 | #sys.exit(retCode) 79 | 80 | self.pool.append(pid) 81 | 82 | if wait: 83 | bRet = self.wait() 84 | 85 | except Exception, e: 86 | Log.err("ERR(%s)" %(traceback.format_exc())) 87 | bRet = False 88 | 89 | return bRet 90 | 91 | def wait(self): 92 | 93 | bRet = True 94 | for i in range(self.size): 95 | pid, retCode = os.waitpid(self.pool[i], 0) 96 | if retCode != 0: bRet = False 97 | return bRet 98 | 99 | 100 | -------------------------------------------------------------------------------- /trunk/fs_manager/src/base/fs_syshead.py: -------------------------------------------------------------------------------- 1 | # -*- coding: UTF-8 -*- 2 | 3 | # project: fshell 4 | # author: s0nnet 5 | # time: 2016-12-08 6 | # desc: 相关模块导入 7 | 8 | 9 | import sys 10 | import os 11 | import time 12 | import datetime 13 | import signal 14 | import inspect 15 | import threading 16 | import thread 17 | import signal 18 | import uuid 19 | import re 20 | import traceback 21 | from multiprocessing import Process 22 | 23 | try: 24 | import json 25 | except Exception, e: 26 | import simplejson as json 27 | 28 | from fs_base_cfg import * 29 | -------------------------------------------------------------------------------- /trunk/fs_manager/src/base/fs_time.py: -------------------------------------------------------------------------------- 1 | # -*- coding: UTF-8 -*- 2 | 3 | # project: fshell 4 | # author: s0nnet 5 | # time: 2016-12-08 6 | # desc: 时间处理类封装 7 | 8 | 9 | from fs_log import * 10 | import datetime 11 | import time 12 | 13 | 14 | def get_cur_dtime(): 15 | return int(time.time()) 16 | 17 | 18 | def get_cur_time(tmFormat=None): 19 | if tmFormat == None: tmFormat = "%Y-%m-%d %H:%M:%S" 20 | return (datetime.datetime.now()).strftime(tmFormat) 21 | 22 | 23 | def yg_tm_str_int(strTm, format="%Y-%m-%d %H:%M:%S"): 24 | if type(strTm) == datetime.datetime: 25 | strTm = strTm.strftime(format) 26 | tmTuple = time.strptime(strTm, format) 27 | return time.mktime(tmTuple) 28 | 29 | 30 | def yg_tm_int_str(tm, format="%Y-%m-%d %H:%M:%S"): 31 | # tmObj = time.localtime(tm) 32 | tmObj = time.localtime(tm) 33 | return time.strftime(format, tmObj) 34 | 35 | 36 | def get_cur_day(days=0, format="%Y%m%d"): 37 | return (datetime.datetime.now() - datetime.timedelta(days)).strftime(format) 38 | 39 | 40 | def get_cur_time_2(days=0, format="%Y-%m-%d %H:%M:%S"): 41 | return (datetime.datetime.now() - datetime.timedelta(days)).strftime(format) 42 | 43 | 44 | def get_latest_months(): 45 | latest_months = [] 46 | now = datetime.datetime.now() 47 | for i in range(9): 48 | latest_months.append((now.year, now.month)) 49 | now = now - datetime.timedelta(days=now.day) 50 | return latest_months 51 | 52 | 53 | def format_time_string(timestamp, now=None): 54 | if now is None: 55 | now = datetime.datetime.now().strftime("%s") 56 | ts_delta = int(now) - int(timestamp) 57 | if ts_delta < 60: 58 | return u"%d秒" % int(ts_delta) 59 | elif 60 <= ts_delta < 3600: 60 | return u"%d分%d秒" % (int(ts_delta // 60), int(ts_delta % 60)) 61 | elif 3600 <= ts_delta < 86400: 62 | return u"%d小时%d分" % (int(ts_delta // 3600), int((ts_delta % 3600) / 60)) 63 | elif 86400 <= ts_delta < 2678400: 64 | return u"%d天%d小时" % (int(ts_delta // 86400), int((ts_delta % 86400) / 3600)) 65 | elif 2678400 <= ts_delta < 8035200: 66 | return u"%d月%d天" % (int(ts_delta // 2678400), int((ts_delta % 2678400) / 86400)) 67 | else: 68 | return u"%d年%d月" % (int(ts_delta // 8035200), int((ts_delta % 8035200) / 2678400)) 69 | 70 | 71 | def format_time_string_v2(timestamp): 72 | now = datetime.datetime.now().strftime("%s") 73 | ts_delta = int(now) - int(timestamp) 74 | if ts_delta < 60: 75 | return u"刚刚" 76 | elif 60 <= ts_delta < 3600: 77 | return u"%d分钟前" % int(ts_delta / 60) 78 | elif 3600 <= ts_delta < 86400: 79 | return u"%d小时前" % int(ts_delta / 3600) 80 | elif 86400 <= ts_delta < 2678400: 81 | return u"%d天前" % int(ts_delta / 86400) 82 | elif 2678400 <= ts_delta < 8035200: 83 | return u"%d月前" % int(ts_delta / 2678400) 84 | else: 85 | return datetime.datetime.fromtimestamp(timestamp).strftime("%Y-%m-%d") 86 | 87 | 88 | if __name__ == "__main__": 89 | def test1(): 90 | print get_cur_time_2(1, "%Y-%m-%d 00:00:00") 91 | 92 | 93 | def test2(): 94 | print datetime.datetime.now() - datetime.timedelta(0) 95 | 96 | 97 | def test3(): 98 | tm = time.time() 99 | tm = int(tm / (24 * 3600)) * (24 * 3600) 100 | 101 | print yg_tm_int_str(tm) 102 | 103 | 104 | def test4(): 105 | print time.localtime(time.time()).tm_hour 106 | print time.gmtime(time.time() + 8 * 3600) 107 | 108 | 109 | def test5(): 110 | curTm = get_cur_dtime() 111 | print curTm 112 | print yg_tm_str_int(get_cur_time()) 113 | print get_cur_time() 114 | print yg_tm_int_str(curTm) 115 | 116 | 117 | def test6(): 118 | print yg_tm_int_str(1406131980) 119 | 120 | 121 | def test7(): 122 | print get_latest_months() 123 | 124 | def test8(): 125 | print get_cur_day(7, "%Y-%m-%d %H:%M:%S") 126 | # test7() 127 | #test8() 128 | 129 | print get_cur_day(-2,format="%Y-%m-%d") 130 | -------------------------------------------------------------------------------- /trunk/fs_manager/src/base/fs_util.py: -------------------------------------------------------------------------------- 1 | # -*- coding: UTF-8 -*- 2 | 3 | # project: fshell 4 | # author: s0nnet 5 | # time: 2016-12-08 6 | # desc: 基础常规方法封装 7 | 8 | 9 | from fs_log import * 10 | from fs_time import * 11 | 12 | import hashlib 13 | import zipfile 14 | import csv 15 | import cStringIO 16 | import urllib 17 | 18 | def get_guid(): 19 | guid = str(uuid.uuid1()) 20 | return guid 21 | 22 | 23 | # 递归的获取path下面的所有文件全名（不包括目录） 24 | def get_file(path): 25 | retList = [] 26 | if os.path.exists(path) and os.path.isdir(path): 27 | childList = os.listdir(path) 28 | for childName in childList: 29 | childFullName = "%s/%s" % (path, childName) 30 | if os.path.isfile(childFullName): 31 | retList.append(childFullName) 32 | elif os.path.isdir(childFullName): 33 | retChildList = get_file(childFullName) 34 | for retChildPath in retChildList: 35 | retList.append(retChildPath) 36 | else: 37 | Log.err("please check dir(%s) path") 38 | 39 | return retList 40 | 41 | 42 | def rm_file(path, delflag=False): 43 | if not path: return 44 | if not os.path.exists(path): return 45 | 46 | if os.path.isfile(path): 47 | os.unlink(path) 48 | return 49 | 50 | fileList = os.listdir(path) 51 | if len(fileList) != 0: 52 | for fname in fileList: 53 | filePath = "%s/%s" % (path, fname) 54 | rm_file(filePath, True) 55 | 56 | if delflag: 57 | os.rmdir(path) 58 | 59 | 60 | def cp_file(sfile, dfile): 61 | bRet = True 62 | rfp = None; 63 | wfp = None 64 | try: 65 | rfp = open(sfile, "rb") 66 | wfp = open(dfile, "wb") 67 | wfp.write(rfp.read()) 68 | except Exception, e: 69 | Log.err("_cpfile(%s, %s) ERR(%s)" % (sfile, dfile, str(e))) 70 | bRet = False 71 | finally: 72 | if rfp != None: rfp.close() 73 | if wfp != None: wfp.close() 74 | return bRet 75 | 76 | 77 | def get_file_line(fPath): 78 | cmd = "cat %s | wc -l" % (fPath) 79 | iRet, sRet = exec_cmd(cmd) 80 | if iRet != 0: return -1 81 | try: 82 | return int(sRet) 83 | except Exception, e: 84 | Log.err("%s is not valid" % (sRet)) 85 | return -1 86 | 87 | 88 | def cp_dir(fromDir, toDir): 89 | if not os.path.exists(toDir): os.makedirs(toDir) 90 | 91 | fnameList = os.listdir(fromDir) 92 | for fname in fnameList: 93 | fromFile = "%s/%s" % (fromDir, fname) 94 | toFile = "%s/%s" % (toDir, fname) 95 | if os.path.isfile(fromFile): 96 | bRet = cp_file(fromFile, toFile) 97 | if not bRet: return False 98 | if os.path.isdir(fromFile): 99 | bRet = cp_dir(fromFile, toFile) 100 | if not bRet: return False 101 | return True 102 | 103 | 104 | def get_dict(dictObj, key, default=None): 105 | if not isinstance(dictObj, dict): 106 | return None 107 | if not dictObj.has_key(key): 108 | return default 109 | return dictObj[key] 110 | 111 | 112 | def compute_md5_file(fileName): 113 | try: 114 | md5 = hashlib.md5() 115 | fp = open(fileName, "rb") 116 | 117 | while True: 118 | data = fp.read(8192) 119 | if not data: break 120 | md5.update(data) 121 | 122 | fp.close() 123 | sret = md5.hexdigest().upper() 124 | return sret 125 | 126 | except Exception, e: 127 | Log.err("fileName(%s) ERR(%s)" % (fileName, str(e))) 128 | return None 129 | 130 | 131 | def comput_md5_text(text): 132 | try: 133 | md5 = hashlib.md5() 134 | md5.update(text) 135 | return md5.hexdigest().upper() 136 | except Exception, e: 137 | Log.err("md5(%s) ERR(%s)" % (text, str(e))) 138 | return None 139 | 140 | 141 | # json load, 结果为UTF-8 142 | def json_loads_utf8(cont): 143 | def decode(jsonCont): 144 | 145 | if type(jsonCont) == unicode: 146 | return jsonCont.encode("utf-8") 147 | 148 | if type(jsonCont) == tuple or type(jsonCont) == list: 149 | for i in range(len(jsonCont)): 150 | jsonCont[i] = decode(jsonCont[i]) 151 | return jsonCont 152 | 153 | if type(jsonCont) == dict: 154 | retJsonCont = {} 155 | for key in jsonCont: 156 | retJsonCont[decode(key)] = decode(jsonCont[key]) 157 | return retJsonCont 158 | 159 | return jsonCont 160 | 161 | if cont == None or cont == "": 162 | return None 163 | 164 | jsonCont = json.loads(cont) 165 | return decode(jsonCont) 166 | 167 | 168 | # xml => json 169 | class XmlToJson: 170 | def __init__(self): 171 | self.jRootNode = {} 172 | 173 | @staticmethod 174 | def _decode(value): 175 | if type(value) == unicode: 176 | return value.encode("utf-8") 177 | return str(value) 178 | 179 | def _change(self, node): 180 | 181 | if node == None or node.tag == None: return None 182 | 183 | retJson = { 184 | "tag": re.sub("\t", " ", XmlToJson._decode(node.tag)), 185 | "text": re.sub("\t", " ", XmlToJson._decode(node.text)), 186 | "attrib": {}, 187 | "_children": [], 188 | } 189 | 190 | for childNode in node._children: 191 | childNode = self._change(childNode) 192 | retJson["_children"].append(childNode) 193 | 194 | for attrName in node.attrib: 195 | retJson["attrib"][attrName] = re.sub("\t", " ", XmlToJson._decode(node.attrib[attrName])) 196 | 197 | return retJson 198 | 199 | def change_text(self, textCnt): 200 | 201 | try: 202 | from xml.etree import ElementTree as ET 203 | rootNode = ET.fromstring(textCnt) 204 | self.jRootNode = self._change(rootNode) 205 | except Exception, e: 206 | Log.err("TEXT(%s) ERR(%s)" % (textCnt, str(e))) 207 | return False 208 | 209 | return True 210 | 211 | def get_jsontext(self): 212 | return json.dumps(self.jRootNode) 213 | 214 | def get_attr(self, jnode, key): 215 | 216 | if jnode == None: return None 217 | if type(jnode) != dict: return None 218 | if type(jnode["attrib"]) != dict: return None 219 | if key == None: return None 220 | if jnode["attrib"].has_key(key) == False: return None 221 | 222 | return jnode["attrib"][key] 223 | 224 | def get_value(self, jnode, key): 225 | 226 | if jnode == None: return None 227 | if type(jnode) != dict: return None 228 | if type(jnode["_children"]) != list: return None 229 | if key == None: return None 230 | 231 | for jchildNode in jnode["_children"]: 232 | if jchildNode["tag"] == key: 233 | return jchildNode["text"] 234 | 235 | return None 236 | 237 | 238 | # xml => json 239 | # 用于xml没有attr的特殊场景, 为了简化调用者 240 | # 为了方便，丧失顶层TAG的保存 241 | class XmlToJson2: 242 | def __init__(self): 243 | self.jRootNode = {} 244 | 245 | def _change(self, node): 246 | 247 | # 去除\t, \n, 首尾空白字符。如果不为空则用utf-8.encode. 否则返回None 248 | def _encode_cnt(value): 249 | if value == None or value == "": return None 250 | 251 | value = re.sub("\t|\n", " ", value) 252 | 253 | if type(value) == unicode: 254 | value = value.encode("utf-8") 255 | 256 | value = value.strip() 257 | if value == "": return None 258 | return value 259 | 260 | if node == None or node.tag == None: return None 261 | text = _encode_cnt(node.text) 262 | if text != None: return text 263 | 264 | retJson = {} 265 | for childNode in node._children: 266 | childTag = _encode_cnt(childNode.tag) 267 | if childTag == None: continue 268 | childVal = self._change(childNode) 269 | # if childVal == None: continue 270 | retJson[childTag] = childVal 271 | return retJson 272 | 273 | def change_text(self, textCnt): 274 | 275 | try: 276 | from xml.etree import ElementTree as ET 277 | rootNode = ET.fromstring(textCnt) 278 | self.jRootNode = self._change(rootNode) 279 | except Exception, e: 280 | Log.err("TEXT(%s) ERR(%s)" % (textCnt, str(e))) 281 | return False 282 | 283 | return True 284 | 285 | def get_jsontext(self): 286 | return json.dumps(self.jRootNode) 287 | 288 | def get_value(self, jnode, key): 289 | 290 | if not jnode.has_key(key): return None 291 | return jnode[key] 292 | 293 | 294 | def yg_zipfile_1(fPath, storePath, password=None): 295 | if not os.path.exists(fPath): 296 | Log.err("filePath = %s is not exist" % (fPath)) 297 | return False 298 | if not os.path.exists(storePath): 299 | try: 300 | os.makedirs(storePath) 301 | except Exception, e: 302 | Log.err("os.makedirs(%s) ERR(%s)" % (storePath, str(e))) 303 | else: 304 | rm_file(storePath) 305 | 306 | bRet = True 307 | f = None 308 | try: 309 | f = zipfile.ZipFile(fPath) 310 | if password != None: 311 | f.setpassword(password) 312 | f.extractall(storePath) 313 | except Exception, e: 314 | Log.err("zipfile(%s, %s) ERR(%s)" % (fPath, storePath, str(e))) 315 | bRet = False 316 | finally: 317 | if f != None: f.close() 318 | return bRet 319 | 320 | 321 | def yg_zipfile(fPath, storePath, password=None): 322 | if not os.path.exists(fPath): 323 | Log.err("filePath = %s is not exist" % (fPath)) 324 | return False 325 | if not os.path.exists(storePath): 326 | try: 327 | os.makedirs(storePath) 328 | except Exception, e: 329 | Log.err("os.makedirs(%s) ERR(%s)" % (storePath, str(e))) 330 | else: 331 | rm_file(storePath) 332 | 333 | if password != None: 334 | zipCmd = "unzip -P %s %s -d %s" % (password, fPath, storePath) 335 | else: 336 | zipCmd = "unzip %s -d %s" % (fPath, storePath) 337 | 338 | return bs_system(zipCmd) 339 | 340 | 341 | def yg_untar(fPath, storePath): 342 | if not os.path.exists(fPath): 343 | Log.err("filePath = %s is not exist" % (fPath)) 344 | return False 345 | if not os.path.exists(storePath): 346 | try: 347 | os.makedirs(storePath) 348 | except Exception, e: 349 | Log.err("os.makedirs(%s) ERR(%s)" % (storePath, str(e))) 350 | 351 | unTarCmd = "tar -zxvf %s -C %s" % (fPath, storePath) 352 | 353 | return bs_system(unTarCmd) 354 | 355 | 356 | def yg_encode_check(cnt): 357 | import chardet 358 | enc = chardet.detect(cnt) 359 | return enc['confidence'], enc['encoding'] 360 | 361 | 362 | def try_decode(cnt): 363 | def decode(cnt, coding): 364 | try: 365 | return cnt.decode(coding) 366 | except: 367 | return None 368 | 369 | dcnt = decode(cnt, "utf-8") 370 | if dcnt != None: return dcnt 371 | 372 | dcnt = decode(cnt, "gbk") 373 | if dcnt != None: return dcnt 374 | 375 | return None 376 | 377 | 378 | def try_change(cnt, coding="utf-8"): 379 | dcnt = try_decode(cnt) 380 | if dcnt == None: return None 381 | 382 | return dcnt.encode(coding) 383 | 384 | 385 | def regexp_extract(source, matchStr, num): 386 | try: 387 | matchObj = re.search(matchStr, source) 388 | if not matchObj: return None 389 | return matchObj.group(num) 390 | 391 | except Exception, e: 392 | Log.err("source(%s), matchStr(%s),num(%d) Err(%s)" % (source, matchStr, num, str(e))) 393 | return None 394 | 395 | 396 | class Page(object): 397 | def __init__(self, item_count, page_index=1, page_size=10): 398 | self.item_count = item_count 399 | self.page_size = page_size 400 | self.page_count = item_count // page_size + (1 if item_count % page_size > 0 else 0) 401 | if (item_count == 0) or (page_index < 1) or (page_index > self.page_count): 402 | self.offset = 0 403 | self.limit = 0 404 | self.page_index = 1 405 | else: 406 | self.page_index = page_index 407 | self.offset = self.page_size * (page_index - 1) 408 | self.limit = self.page_size 409 | self.has_next = self.page_index < self.page_count 410 | self.has_previous = self.page_index > 1 411 | 412 | def __str__(self): 413 | return 'item_count: %s, page_count: %s, page_index: %s, page_size: %s, offset: %s, limit: %s' % ( 414 | self.item_count, self.page_count, self.page_index, self.page_size, self.offset, self.limit) 415 | 416 | 417 | def safe_json_loads(jsonData): 418 | try: 419 | return json.loads(jsonData) 420 | except Exception, e: 421 | Log.err(traceback.format_exc()) 422 | return None 423 | 424 | 425 | def safe_datetime_to_str(tm): 426 | if type(tm) == datetime.datetime: 427 | return tm.strftime("%Y-%m-%d %H:%M:%S") 428 | return '' 429 | 430 | 431 | def safe_str_split(string, sep): 432 | try: 433 | return string.split(sep) 434 | except Exception, e: 435 | return [] 436 | 437 | def html_decode(string): 438 | return urllib.unquote(string) 439 | 440 | 441 | def check_ip_format(ip_data, format): 442 | """ 443 | check ip format: ip(10.0.0.1), (mask)10.0.0.1/24, (segment)10.0.0.1-10.0.0.54 444 | """ 445 | if format == 'ip': 446 | pic = ip_data.split('.') 447 | if len(pic) != 4: return False 448 | try: 449 | return all(0 <= int(p) < 256 for p in pic) 450 | except ValueError: 451 | return False 452 | 453 | elif format == 'mask': 454 | 455 | ips = ip_data.split('/') 456 | if len(ips) != 2: return False 457 | if (int(ips[1]) < 0) or (int(ips[1]) > 32): return False 458 | 459 | pic = ips[0].split('.') 460 | if len(pic) != 4: return False 461 | try: 462 | return all(0 <= int(p) < 256 for p in pic) 463 | except ValueError: 464 | return False 465 | 466 | elif format == 'segment': 467 | ips = ip_data.split('-') 468 | if len(ips) != 2: return False 469 | 470 | pic = ips[0].split('.') 471 | pic.extend(ips[1].split('.')) 472 | 473 | if len(pic) != 8: return False 474 | try: 475 | return all(0 <= int(p) < 256 for p in pic) 476 | except ValueError: 477 | return False 478 | 479 | else: 480 | return False 481 | 482 | class GetIpSegment(object): 483 | """ 484 | format ip/mask to an ip segment. 485 | eg: '192.168.1.5/26' to '192.168.1.0-192.168.1.63' 486 | """ 487 | 488 | def __init__(self, ip_mask): 489 | self.ip_mask = ip_mask 490 | 491 | def dec2bin(self, n): 492 | bin_str = bin(n).replace("0b", "") 493 | z_str = ''.join([str('0') for x in range(8-len(bin_str))]) 494 | return z_str + bin_str 495 | 496 | def bin2ip_str(self, bin_str): 497 | ip = [] 498 | for i in range(0, len(bin_str), 8): 499 | dec_str = str(int(bin_str[i:i+8], 2)) # bin2dec 500 | ip.append(dec_str) 501 | return ip[0]+"."+ip[1]+"."+ip[2]+"."+ip[3] 502 | 503 | def ip_str(self): 504 | 505 | ip = self.ip_mask.split("/")[0] 506 | mask = int(self.ip_mask.split("/")[1]) 507 | ip_lt = ip.split(".") 508 | ip_bin = ''.join([self.dec2bin(int(x)) for x in ip_lt]) 509 | ip_bin_p = ip_bin[0:mask] 510 | 511 | ip_min = ip_bin_p + ''.join([str('0') for i in range(32-mask)]) 512 | ip_max = ip_bin_p + ''.join([str('1') for i in range(32-mask)]) 513 | 514 | return (self.bin2ip_str(ip_min), self.bin2ip_str(ip_max)) 515 | 516 | 517 | class Echo(object): 518 | def write(self, value): 519 | return value 520 | 521 | 522 | class CSVWriter: 523 | """ 524 | A CSV writer which will write rows to CSV file "f" 525 | """ 526 | 527 | def __init__(self, f=None, dialect=csv.excel, **kwds): 528 | self.queue = cStringIO.StringIO() 529 | self.writer = csv.writer(self.queue, dialect=dialect, **kwds) 530 | self.stream = f if f else Echo() 531 | 532 | def writerow(self, row): 533 | self.writer.writerow(row) 534 | data = self.queue.getvalue() 535 | value = self.stream.write(data) 536 | self.queue.truncate(0) 537 | return value 538 | 539 | def writerows(self, rows): 540 | for row in rows: 541 | self.writerow(row) 542 | 543 | 544 | if __name__ == "__main__": 545 | def test1(): 546 | print yg_encode_check(u'我们都有一个家'.encode('gbk')) 547 | print yg_encode_check(u'我们都有一个家'.encode('utf-8')) 548 | print yg_encode_check("zxy") 549 | 550 | 551 | def test2(): 552 | print try_change(u'我们都有一个家'.encode('gbk')) 553 | 554 | 555 | def test3(): 556 | print get_file_line(os.path.abspath("../../data/ftp_root/test1")) 557 | 558 | 559 | # test2() 560 | # test3() 561 | #print GetIpSegment('192.168.1.1/0').ip_str() 562 | -------------------------------------------------------------------------------- /trunk/fs_server/conf/README.md: -------------------------------------------------------------------------------- 1 | this is conf of fs_server 2 | -------------------------------------------------------------------------------- /trunk/fs_server/src/base/fs_base_cfg.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # project: fshell 4 | # author: s0nnet 5 | # time: 2016-12-08 6 | # desc: base conf 7 | 8 | 9 | import os 10 | import sys 11 | 12 | ''' 13 | LOG_LEVEL 14 | DEBUG = 1 15 | INFO = 2 16 | WARINING = 3 17 | ERR = 4 18 | ALERT = 5 19 | CLOSE = 10 20 | ''' 21 | class BaseConf: 22 | 23 | LOG_LEVEL = 2 24 | LOG_DIR = "../../log/" 25 | LOG_PREFIX = "" 26 | IS_CTR_LOG = True 27 | 28 | SQL_HOST = "222.24.XX.XX" 29 | SQL_PORT = 3306 30 | SQL_USER = "root" 31 | SQL_PASSWD = "rtxxxx" 32 | SQL_DB = "fshell" 33 | -------------------------------------------------------------------------------- /trunk/fs_server/src/base/fs_database_pid.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # project: fshell 4 | # author: s0nnet 5 | # time: 2016-12-08 6 | # desc: MySQL基础操作类封装 7 | 8 | 9 | import MySQLdb 10 | from fs_log import * 11 | 12 | class SqlCmdEnum: 13 | COMMIT_CMD = 0 14 | ROLLBACK_CMD = 1 15 | 16 | class SqlExecEnum: 17 | QUERY = 0 18 | UPDATE = 1 19 | INSERT = 2 20 | DELETE = 3 21 | 22 | 23 | def _new_conn(host, port, user, passwd, db, charset): 24 | try: 25 | conn = MySQLdb.connect(host = host, 26 | port = port, 27 | user = user, 28 | passwd = passwd, 29 | db = db, 30 | charset = charset) 31 | conn.autocommit(True) 32 | Log.debug("conn(host: %s, port: %d, user: %s, passwd: %s, db: %s)" %(host, port, user, passwd, db)) 33 | return conn 34 | except Exception, e: 35 | Log.err("ERR(host: %s, port: %d, user: %s, passwd: %s, db: %s, %s)" %(host, port, user, passwd, db, str(e))) 36 | return None 37 | 38 | class DataBase: 39 | 40 | def __init__(self, 41 | host = BaseConf.SQL_HOST, 42 | port = BaseConf.SQL_PORT, 43 | user = BaseConf.SQL_USER, 44 | passwd = BaseConf.SQL_PASSWD, 45 | db = BaseConf.SQL_DB, 46 | charset = None): 47 | self.host = host 48 | self.port = port 49 | self.user = user 50 | self.passwd = passwd 51 | self.db = db 52 | self.charset = charset 53 | 54 | self.conn = None 55 | 56 | def _check(self): 57 | try: 58 | self.conn.ping() 59 | except Exception, e: 60 | self.conn = _new_conn(self.host, self.port, self.user, self.passwd, self.db, self.charset) 61 | 62 | def _get_conn(self): 63 | if self.conn == None: 64 | self.conn = _new_conn(self.host, self.port, self.user, self.passwd, self.db, self.charset) 65 | else: 66 | self._check() 67 | 68 | def _close_conn(self): 69 | if self.conn != None: 70 | self.conn.commit() 71 | self.conn.close() 72 | self.conn = None 73 | 74 | 75 | def _get_record_list(self, cursor): 76 | 77 | def _get_fieldname_list(cursor): 78 | fieldList = [] 79 | for info in cursor.description: 80 | fieldList.append(info[0]) 81 | return fieldList 82 | 83 | fieldnameList = _get_fieldname_list(cursor) 84 | 85 | retNodeList = [] 86 | recordList = cursor.fetchall() 87 | for record in recordList: 88 | node = {} 89 | for i in range(len(fieldnameList)): 90 | node[fieldnameList[i]] = record[i] 91 | retNodeList.append(node) 92 | 93 | return retNodeList 94 | 95 | 96 | def _exec_cmdstr(self, cmdstr, param, rollback, execType): 97 | 98 | def _encode_param(param): 99 | dParam = [] 100 | for par in param: 101 | if type(par) == unicode: 102 | par = par.encode('utf-8') 103 | dParam.append(par) 104 | return dParam 105 | 106 | if param != None: param = _encode_param(param) 107 | Log.debug("%s\t%s" %(cmdstr, str(param))) 108 | 109 | self._get_conn() 110 | conn = self.conn 111 | if conn == None: 112 | Log.err("mysql not connect!") 113 | return False, "mysql not connect!" 114 | 115 | cursor = conn.cursor() 116 | try: 117 | val = cursor.execute(cmdstr, param) 118 | if execType == SqlExecEnum.QUERY: val = self._get_record_list(cursor) 119 | elif execType == SqlExecEnum.INSERT: val = int(conn.insert_id()) 120 | else: pass 121 | except Exception, e: 122 | Log.warning("sql: %s,%s ERR(%s)" %(cmdstr, str(param), str(e))) 123 | if rollback: 124 | self.cmd_submit(SqlCmdEnum.ROLLBACK_CMD) 125 | else: 126 | self._close_conn() 127 | return False, "%s:%s %s" %(cmdstr, str(param), str(e)) 128 | 129 | if not rollback: 130 | self._close_conn() 131 | 132 | return True, val 133 | 134 | def query_data(self, cmdstr, param, rollback = False): 135 | return self._exec_cmdstr(cmdstr, param, rollback, SqlExecEnum.QUERY) 136 | 137 | def update_data(self, cmdstr, param, rollback = False): 138 | return self._exec_cmdstr(cmdstr, param, rollback, SqlExecEnum.UPDATE) 139 | 140 | def insert_data(self, cmdstr, param, rollback = False): 141 | return self._exec_cmdstr(cmdstr, param, rollback, SqlExecEnum.INSERT) 142 | 143 | def delete_data(self, cmdstr, param, rollback = False): 144 | return self._exec_cmdstr(cmdstr, param, rollback, SqlExecEnum.DELETE) 145 | 146 | def cmd_submit(self, flag): 147 | conn = self.conn 148 | if not conn: self._close_conn(); return 149 | 150 | try: 151 | if flag == SqlCmdEnum.COMMIT_CMD: 152 | conn.commit() 153 | elif flag == SqlCmdEnum.ROLLBACK_CMD: 154 | conn.rollback() 155 | finally: 156 | self._close_conn() 157 | 158 | 159 | if __name__ == "__main__": 160 | 161 | dataBase = DataBase() 162 | 163 | cmdstr = "select * from tb_user limit 10" 164 | print dataBase.query_data(cmdstr, None) 165 | -------------------------------------------------------------------------------- /trunk/fs_server/src/base/fs_log.py: -------------------------------------------------------------------------------- 1 | # -*- coding: UTF-8 -*- 2 | 3 | # project: fshell 4 | # author: s0nnet 5 | # time: 2016-12-08 6 | # desc: 日志模块封装 7 | 8 | 9 | from fs_syshead import * 10 | 11 | class LogLevel: 12 | DEBUG = 1 13 | INFO = 2 14 | WARNING = 3 15 | ERR = 4 16 | ALERT = 5 17 | CLOSE = 10 18 | 19 | def _get_log_level(logLevel): 20 | if logLevel == LogLevel.DEBUG: return "DEBUG" 21 | if logLevel == LogLevel.INFO: return "INFO" 22 | if logLevel == LogLevel.WARNING: return "WARNING" 23 | if logLevel == LogLevel.ERR: return "ERROR" 24 | if logLevel == LogLevel.ALERT: return "ALERT" 25 | return "CLOSE" 26 | 27 | def _get_log_tm(): 28 | "[01/Jan/2013 04:51:46]" 29 | return (datetime.datetime.now()).strftime("[%d/%m/%Y %H:%M:%S]") 30 | 31 | 32 | def _get_file(f): return f[1] 33 | def _get_function(f): return f[3] 34 | def _get_line(f): return f[2] 35 | def _get_location(stackIdx): 36 | f = inspect.stack()[stackIdx] 37 | return "%s::%s(%d)" %(_get_file(f), _get_function(f), _get_line(f)) 38 | 39 | 40 | class LogFile: 41 | 42 | @staticmethod 43 | def rm_file(path, delflag = False): 44 | 45 | if not path: return 46 | if not os.path.exists(path): return 47 | 48 | if os.path.isfile(path): 49 | os.unlink(path) 50 | return 51 | 52 | fileList = os.listdir(path) 53 | if len(fileList) != 0: 54 | for fname in fileList: 55 | filePath = "%s/%s" %(path, fname) 56 | LogFile.rm_file(filePath, True) 57 | 58 | if delflag: 59 | os.rmdir(path) 60 | 61 | def __init__(self): 62 | self.fp = None; 63 | self.maxSzie = 1024 * 1024 64 | self.fName = -1 65 | 66 | def _open_logfile(self): 67 | self.fName += 1 68 | if not os.path.exists(BaseConf.LOG_DIR): 69 | os.makedirs(BaseConf.LOG_DIR) 70 | 71 | curTm = (datetime.datetime.now()).strftime("%Y_%m_%d_%H_%M_%S") 72 | preFix = BaseConf.LOG_PREFIX + "_%s" %(curTm) 73 | tmpName = "%s/%s.txt" %(BaseConf.LOG_DIR, preFix) 74 | 75 | try: 76 | if self.fp: self.fp.close() 77 | self.fp = open(tmpName, "a") 78 | 79 | os.dup2(self.fp.fileno(), sys.stdout.fileno()) 80 | os.dup2(self.fp.fileno(), sys.stderr.fileno()) 81 | except Exception, e: 82 | sys.stderr.write("%s: %s\n" %(_get_location(1), str(e))) 83 | 84 | def _print_log(self, buf): 85 | try: 86 | if self.fp == None or self.fp.tell() >= self.maxSzie: 87 | self._open_logfile() 88 | 89 | self.fp.write(buf) 90 | self.fp.flush() 91 | except Exception, e: 92 | sys.stderr.write("%s: %s" %(_get_location(1), str(e))) 93 | sys.stderr.flush() 94 | 95 | logFile = LogFile() 96 | 97 | 98 | class Log: 99 | 100 | @staticmethod 101 | def _loglog(buf, stackIdx, logLevel): 102 | if logLevel >= BaseConf.LOG_LEVEL: 103 | msg = "%s[%s]%s: %s\n" %(_get_log_tm(), _get_log_level(logLevel), _get_location(stackIdx + 1), buf) 104 | if BaseConf.IS_CTR_LOG: sys.stderr.write(msg); sys.stderr.flush() 105 | else: logFile._print_log(msg) 106 | 107 | 108 | @staticmethod 109 | def debug(buf, stackIdx = 1): 110 | Log._loglog(buf, stackIdx + 1, LogLevel.DEBUG) 111 | 112 | @staticmethod 113 | def info(buf, stackIdx = 1): 114 | Log._loglog(buf, stackIdx + 1, LogLevel.INFO) 115 | 116 | @staticmethod 117 | def warning(buf, stackIdx = 1): 118 | Log._loglog(buf, stackIdx + 1, LogLevel.WARNING) 119 | 120 | @staticmethod 121 | def err(buf, stackIdx = 1): 122 | Log._loglog(buf, stackIdx + 1, LogLevel.ERR) 123 | 124 | @staticmethod 125 | def alert(buf, stackIdx = 1): 126 | Log._loglog(buf, stackIdx + 1, LogLevel.ALERT) 127 | 128 | 129 | if __name__ == "__main__": 130 | Log.debug("Test......") 131 | Log.info("Test.......") 132 | Log.err("Test.......") 133 | 134 | 135 | -------------------------------------------------------------------------------- /trunk/fs_server/src/base/fs_syshead.py: -------------------------------------------------------------------------------- 1 | # -*- coding: UTF-8 -*- 2 | 3 | # project: fshell 4 | # author: s0nnet 5 | # time: 2016-12-08 6 | # desc: 相关模块导入 7 | 8 | 9 | import sys 10 | import os 11 | import time 12 | import datetime 13 | import signal 14 | import inspect 15 | import threading 16 | import thread 17 | import signal 18 | import uuid 19 | import re 20 | import traceback 21 | from multiprocessing import Process 22 | 23 | try: 24 | import json 25 | except Exception, e: 26 | import simplejson as json 27 | 28 | from fs_base_cfg import * 29 | -------------------------------------------------------------------------------- /trunk/fs_server/src/base/fs_time.py: -------------------------------------------------------------------------------- 1 | # -*- coding: UTF-8 -*- 2 | 3 | # project: fshell 4 | # author: s0nnet 5 | # time: 2016-12-08 6 | # desc: 时间处理类封装 7 | 8 | 9 | from fs_log import * 10 | import datetime 11 | import time 12 | 13 | 14 | def get_cur_dtime(): 15 | return int(time.time()) 16 | 17 | 18 | def get_cur_time(tmFormat=None): 19 | if tmFormat == None: tmFormat = "%Y-%m-%d %H:%M:%S" 20 | return (datetime.datetime.now()).strftime(tmFormat) 21 | 22 | 23 | def yg_tm_str_int(strTm, format="%Y-%m-%d %H:%M:%S"): 24 | if type(strTm) == datetime.datetime: 25 | strTm = strTm.strftime(format) 26 | tmTuple = time.strptime(strTm, format) 27 | return time.mktime(tmTuple) 28 | 29 | 30 | def yg_tm_int_str(tm, format="%Y-%m-%d %H:%M:%S"): 31 | # tmObj = time.localtime(tm) 32 | tmObj = time.localtime(tm) 33 | return time.strftime(format, tmObj) 34 | 35 | 36 | def get_cur_day(days=0, format="%Y%m%d"): 37 | return (datetime.datetime.now() - datetime.timedelta(days)).strftime(format) 38 | 39 | 40 | def get_cur_time_2(days=0, format="%Y-%m-%d %H:%M:%S"): 41 | return (datetime.datetime.now() - datetime.timedelta(days)).strftime(format) 42 | 43 | 44 | def get_latest_months(): 45 | latest_months = [] 46 | now = datetime.datetime.now() 47 | for i in range(9): 48 | latest_months.append((now.year, now.month)) 49 | now = now - datetime.timedelta(days=now.day) 50 | return latest_months 51 | 52 | 53 | def format_time_string(timestamp, now=None): 54 | if now is None: 55 | now = datetime.datetime.now().strftime("%s") 56 | ts_delta = int(now) - int(timestamp) 57 | if ts_delta < 60: 58 | return u"%d秒" % int(ts_delta) 59 | elif 60 <= ts_delta < 3600: 60 | return u"%d分%d秒" % (int(ts_delta // 60), int(ts_delta % 60)) 61 | elif 3600 <= ts_delta < 86400: 62 | return u"%d小时%d分" % (int(ts_delta // 3600), int((ts_delta % 3600) / 60)) 63 | elif 86400 <= ts_delta < 2678400: 64 | return u"%d天%d小时" % (int(ts_delta // 86400), int((ts_delta % 86400) / 3600)) 65 | elif 2678400 <= ts_delta < 8035200: 66 | return u"%d月%d天" % (int(ts_delta // 2678400), int((ts_delta % 2678400) / 86400)) 67 | else: 68 | return u"%d年%d月" % (int(ts_delta // 8035200), int((ts_delta % 8035200) / 2678400)) 69 | 70 | 71 | def format_time_string_v2(timestamp): 72 | now = datetime.datetime.now().strftime("%s") 73 | ts_delta = int(now) - int(timestamp) 74 | if ts_delta < 60: 75 | return u"刚刚" 76 | elif 60 <= ts_delta < 3600: 77 | return u"%d分钟前" % int(ts_delta / 60) 78 | elif 3600 <= ts_delta < 86400: 79 | return u"%d小时前" % int(ts_delta / 3600) 80 | elif 86400 <= ts_delta < 2678400: 81 | return u"%d天前" % int(ts_delta / 86400) 82 | elif 2678400 <= ts_delta < 8035200: 83 | return u"%d月前" % int(ts_delta / 2678400) 84 | else: 85 | return datetime.datetime.fromtimestamp(timestamp).strftime("%Y-%m-%d") 86 | 87 | 88 | if __name__ == "__main__": 89 | def test1(): 90 | print get_cur_time_2(1, "%Y-%m-%d 00:00:00") 91 | 92 | 93 | def test2(): 94 | print datetime.datetime.now() - datetime.timedelta(0) 95 | 96 | 97 | def test3(): 98 | tm = time.time() 99 | tm = int(tm / (24 * 3600)) * (24 * 3600) 100 | 101 | print yg_tm_int_str(tm) 102 | 103 | 104 | def test4(): 105 | print time.localtime(time.time()).tm_hour 106 | print time.gmtime(time.time() + 8 * 3600) 107 | 108 | 109 | def test5(): 110 | curTm = get_cur_dtime() 111 | print curTm 112 | print yg_tm_str_int(get_cur_time()) 113 | print get_cur_time() 114 | print yg_tm_int_str(curTm) 115 | 116 | 117 | def test6(): 118 | print yg_tm_int_str(1406131980) 119 | 120 | 121 | def test7(): 122 | print get_latest_months() 123 | 124 | def test8(): 125 | print get_cur_day(7, "%Y-%m-%d %H:%M:%S") 126 | # test7() 127 | #test8() 128 | 129 | print get_cur_day(-2,format="%Y-%m-%d") 130 | -------------------------------------------------------------------------------- /trunk/fs_server/src/base/fs_util.py: -------------------------------------------------------------------------------- 1 | # -*- coding: UTF-8 -*- 2 | 3 | # project: fshell 4 | # author: s0nnet 5 | # time: 2016-12-08 6 | # desc: 基础常规方法封装 7 | 8 | 9 | from fs_log import * 10 | from fs_time import * 11 | 12 | import hashlib 13 | import zipfile 14 | import csv 15 | import cStringIO 16 | import urllib 17 | 18 | def get_guid(): 19 | guid = str(uuid.uuid1()) 20 | return guid 21 | 22 | 23 | # 递归的获取path下面的所有文件全名（不包括目录） 24 | def get_file(path): 25 | retList = [] 26 | if os.path.exists(path) and os.path.isdir(path): 27 | childList = os.listdir(path) 28 | for childName in childList: 29 | childFullName = "%s/%s" % (path, childName) 30 | if os.path.isfile(childFullName): 31 | retList.append(childFullName) 32 | elif os.path.isdir(childFullName): 33 | retChildList = get_file(childFullName) 34 | for retChildPath in retChildList: 35 | retList.append(retChildPath) 36 | else: 37 | Log.err("please check dir(%s) path") 38 | 39 | return retList 40 | 41 | 42 | def rm_file(path, delflag=False): 43 | if not path: return 44 | if not os.path.exists(path): return 45 | 46 | if os.path.isfile(path): 47 | os.unlink(path) 48 | return 49 | 50 | fileList = os.listdir(path) 51 | if len(fileList) != 0: 52 | for fname in fileList: 53 | filePath = "%s/%s" % (path, fname) 54 | rm_file(filePath, True) 55 | 56 | if delflag: 57 | os.rmdir(path) 58 | 59 | 60 | def cp_file(sfile, dfile): 61 | bRet = True 62 | rfp = None; 63 | wfp = None 64 | try: 65 | rfp = open(sfile, "rb") 66 | wfp = open(dfile, "wb") 67 | wfp.write(rfp.read()) 68 | except Exception, e: 69 | Log.err("_cpfile(%s, %s) ERR(%s)" % (sfile, dfile, str(e))) 70 | bRet = False 71 | finally: 72 | if rfp != None: rfp.close() 73 | if wfp != None: wfp.close() 74 | return bRet 75 | 76 | 77 | def get_file_line(fPath): 78 | cmd = "cat %s | wc -l" % (fPath) 79 | iRet, sRet = exec_cmd(cmd) 80 | if iRet != 0: return -1 81 | try: 82 | return int(sRet) 83 | except Exception, e: 84 | Log.err("%s is not valid" % (sRet)) 85 | return -1 86 | 87 | 88 | def cp_dir(fromDir, toDir): 89 | if not os.path.exists(toDir): os.makedirs(toDir) 90 | 91 | fnameList = os.listdir(fromDir) 92 | for fname in fnameList: 93 | fromFile = "%s/%s" % (fromDir, fname) 94 | toFile = "%s/%s" % (toDir, fname) 95 | if os.path.isfile(fromFile): 96 | bRet = cp_file(fromFile, toFile) 97 | if not bRet: return False 98 | if os.path.isdir(fromFile): 99 | bRet = cp_dir(fromFile, toFile) 100 | if not bRet: return False 101 | return True 102 | 103 | 104 | def get_dict(dictObj, key, default=None): 105 | if not isinstance(dictObj, dict): 106 | return None 107 | if not dictObj.has_key(key): 108 | return default 109 | return dictObj[key] 110 | 111 | 112 | def compute_md5_file(fileName): 113 | try: 114 | md5 = hashlib.md5() 115 | fp = open(fileName, "rb") 116 | 117 | while True: 118 | data = fp.read(8192) 119 | if not data: break 120 | md5.update(data) 121 | 122 | fp.close() 123 | sret = md5.hexdigest().upper() 124 | return sret 125 | 126 | except Exception, e: 127 | Log.err("fileName(%s) ERR(%s)" % (fileName, str(e))) 128 | return None 129 | 130 | 131 | def comput_md5_text(text): 132 | try: 133 | md5 = hashlib.md5() 134 | md5.update(text) 135 | return md5.hexdigest().upper() 136 | except Exception, e: 137 | Log.err("md5(%s) ERR(%s)" % (text, str(e))) 138 | return None 139 | 140 | 141 | # json load, 结果为UTF-8 142 | def json_loads_utf8(cont): 143 | def decode(jsonCont): 144 | 145 | if type(jsonCont) == unicode: 146 | return jsonCont.encode("utf-8") 147 | 148 | if type(jsonCont) == tuple or type(jsonCont) == list: 149 | for i in range(len(jsonCont)): 150 | jsonCont[i] = decode(jsonCont[i]) 151 | return jsonCont 152 | 153 | if type(jsonCont) == dict: 154 | retJsonCont = {} 155 | for key in jsonCont: 156 | retJsonCont[decode(key)] = decode(jsonCont[key]) 157 | return retJsonCont 158 | 159 | return jsonCont 160 | 161 | if cont == None or cont == "": 162 | return None 163 | 164 | jsonCont = json.loads(cont) 165 | return decode(jsonCont) 166 | 167 | 168 | # xml => json 169 | class XmlToJson: 170 | def __init__(self): 171 | self.jRootNode = {} 172 | 173 | @staticmethod 174 | def _decode(value): 175 | if type(value) == unicode: 176 | return value.encode("utf-8") 177 | return str(value) 178 | 179 | def _change(self, node): 180 | 181 | if node == None or node.tag == None: return None 182 | 183 | retJson = { 184 | "tag": re.sub("\t", " ", XmlToJson._decode(node.tag)), 185 | "text": re.sub("\t", " ", XmlToJson._decode(node.text)), 186 | "attrib": {}, 187 | "_children": [], 188 | } 189 | 190 | for childNode in node._children: 191 | childNode = self._change(childNode) 192 | retJson["_children"].append(childNode) 193 | 194 | for attrName in node.attrib: 195 | retJson["attrib"][attrName] = re.sub("\t", " ", XmlToJson._decode(node.attrib[attrName])) 196 | 197 | return retJson 198 | 199 | def change_text(self, textCnt): 200 | 201 | try: 202 | from xml.etree import ElementTree as ET 203 | rootNode = ET.fromstring(textCnt) 204 | self.jRootNode = self._change(rootNode) 205 | except Exception, e: 206 | Log.err("TEXT(%s) ERR(%s)" % (textCnt, str(e))) 207 | return False 208 | 209 | return True 210 | 211 | def get_jsontext(self): 212 | return json.dumps(self.jRootNode) 213 | 214 | def get_attr(self, jnode, key): 215 | 216 | if jnode == None: return None 217 | if type(jnode) != dict: return None 218 | if type(jnode["attrib"]) != dict: return None 219 | if key == None: return None 220 | if jnode["attrib"].has_key(key) == False: return None 221 | 222 | return jnode["attrib"][key] 223 | 224 | def get_value(self, jnode, key): 225 | 226 | if jnode == None: return None 227 | if type(jnode) != dict: return None 228 | if type(jnode["_children"]) != list: return None 229 | if key == None: return None 230 | 231 | for jchildNode in jnode["_children"]: 232 | if jchildNode["tag"] == key: 233 | return jchildNode["text"] 234 | 235 | return None 236 | 237 | 238 | # xml => json 239 | # 用于xml没有attr的特殊场景, 为了简化调用者 240 | # 为了方便，丧失顶层TAG的保存 241 | class XmlToJson2: 242 | def __init__(self): 243 | self.jRootNode = {} 244 | 245 | def _change(self, node): 246 | 247 | # 去除\t, \n, 首尾空白字符。如果不为空则用utf-8.encode. 否则返回None 248 | def _encode_cnt(value): 249 | if value == None or value == "": return None 250 | 251 | value = re.sub("\t|\n", " ", value) 252 | 253 | if type(value) == unicode: 254 | value = value.encode("utf-8") 255 | 256 | value = value.strip() 257 | if value == "": return None 258 | return value 259 | 260 | if node == None or node.tag == None: return None 261 | text = _encode_cnt(node.text) 262 | if text != None: return text 263 | 264 | retJson = {} 265 | for childNode in node._children: 266 | childTag = _encode_cnt(childNode.tag) 267 | if childTag == None: continue 268 | childVal = self._change(childNode) 269 | # if childVal == None: continue 270 | retJson[childTag] = childVal 271 | return retJson 272 | 273 | def change_text(self, textCnt): 274 | 275 | try: 276 | from xml.etree import ElementTree as ET 277 | rootNode = ET.fromstring(textCnt) 278 | self.jRootNode = self._change(rootNode) 279 | except Exception, e: 280 | Log.err("TEXT(%s) ERR(%s)" % (textCnt, str(e))) 281 | return False 282 | 283 | return True 284 | 285 | def get_jsontext(self): 286 | return json.dumps(self.jRootNode) 287 | 288 | def get_value(self, jnode, key): 289 | 290 | if not jnode.has_key(key): return None 291 | return jnode[key] 292 | 293 | 294 | def yg_zipfile_1(fPath, storePath, password=None): 295 | if not os.path.exists(fPath): 296 | Log.err("filePath = %s is not exist" % (fPath)) 297 | return False 298 | if not os.path.exists(storePath): 299 | try: 300 | os.makedirs(storePath) 301 | except Exception, e: 302 | Log.err("os.makedirs(%s) ERR(%s)" % (storePath, str(e))) 303 | else: 304 | rm_file(storePath) 305 | 306 | bRet = True 307 | f = None 308 | try: 309 | f = zipfile.ZipFile(fPath) 310 | if password != None: 311 | f.setpassword(password) 312 | f.extractall(storePath) 313 | except Exception, e: 314 | Log.err("zipfile(%s, %s) ERR(%s)" % (fPath, storePath, str(e))) 315 | bRet = False 316 | finally: 317 | if f != None: f.close() 318 | return bRet 319 | 320 | 321 | def yg_zipfile(fPath, storePath, password=None): 322 | if not os.path.exists(fPath): 323 | Log.err("filePath = %s is not exist" % (fPath)) 324 | return False 325 | if not os.path.exists(storePath): 326 | try: 327 | os.makedirs(storePath) 328 | except Exception, e: 329 | Log.err("os.makedirs(%s) ERR(%s)" % (storePath, str(e))) 330 | else: 331 | rm_file(storePath) 332 | 333 | if password != None: 334 | zipCmd = "unzip -P %s %s -d %s" % (password, fPath, storePath) 335 | else: 336 | zipCmd = "unzip %s -d %s" % (fPath, storePath) 337 | 338 | return bs_system(zipCmd) 339 | 340 | 341 | def yg_untar(fPath, storePath): 342 | if not os.path.exists(fPath): 343 | Log.err("filePath = %s is not exist" % (fPath)) 344 | return False 345 | if not os.path.exists(storePath): 346 | try: 347 | os.makedirs(storePath) 348 | except Exception, e: 349 | Log.err("os.makedirs(%s) ERR(%s)" % (storePath, str(e))) 350 | 351 | unTarCmd = "tar -zxvf %s -C %s" % (fPath, storePath) 352 | 353 | return bs_system(unTarCmd) 354 | 355 | 356 | def yg_encode_check(cnt): 357 | import chardet 358 | enc = chardet.detect(cnt) 359 | return enc['confidence'], enc['encoding'] 360 | 361 | 362 | def try_decode(cnt): 363 | def decode(cnt, coding): 364 | try: 365 | return cnt.decode(coding) 366 | except: 367 | return None 368 | 369 | dcnt = decode(cnt, "utf-8") 370 | if dcnt != None: return dcnt 371 | 372 | dcnt = decode(cnt, "gbk") 373 | if dcnt != None: return dcnt 374 | 375 | return None 376 | 377 | 378 | def try_change(cnt, coding="utf-8"): 379 | dcnt = try_decode(cnt) 380 | if dcnt == None: return None 381 | 382 | return dcnt.encode(coding) 383 | 384 | 385 | def regexp_extract(source, matchStr, num): 386 | try: 387 | matchObj = re.search(matchStr, source) 388 | if not matchObj: return None 389 | return matchObj.group(num) 390 | 391 | except Exception, e: 392 | Log.err("source(%s), matchStr(%s),num(%d) Err(%s)" % (source, matchStr, num, str(e))) 393 | return None 394 | 395 | 396 | class Page(object): 397 | def __init__(self, item_count, page_index=1, page_size=10): 398 | self.item_count = item_count 399 | self.page_size = page_size 400 | self.page_count = item_count // page_size + (1 if item_count % page_size > 0 else 0) 401 | if (item_count == 0) or (page_index < 1) or (page_index > self.page_count): 402 | self.offset = 0 403 | self.limit = 0 404 | self.page_index = 1 405 | else: 406 | self.page_index = page_index 407 | self.offset = self.page_size * (page_index - 1) 408 | self.limit = self.page_size 409 | self.has_next = self.page_index < self.page_count 410 | self.has_previous = self.page_index > 1 411 | 412 | def __str__(self): 413 | return 'item_count: %s, page_count: %s, page_index: %s, page_size: %s, offset: %s, limit: %s' % ( 414 | self.item_count, self.page_count, self.page_index, self.page_size, self.offset, self.limit) 415 | 416 | 417 | def safe_json_loads(jsonData): 418 | try: 419 | return json.loads(jsonData) 420 | except Exception, e: 421 | Log.err(traceback.format_exc()) 422 | return None 423 | 424 | 425 | def safe_datetime_to_str(tm): 426 | if type(tm) == datetime.datetime: 427 | return tm.strftime("%Y-%m-%d %H:%M:%S") 428 | return '' 429 | 430 | 431 | def safe_str_split(string, sep): 432 | try: 433 | return string.split(sep) 434 | except Exception, e: 435 | return [] 436 | 437 | def html_decode(string): 438 | return urllib.unquote(string) 439 | 440 | 441 | def check_ip_format(ip_data, format): 442 | """ 443 | check ip format: ip(10.0.0.1), (mask)10.0.0.1/24, (segment)10.0.0.1-10.0.0.54 444 | """ 445 | if format == 'ip': 446 | pic = ip_data.split('.') 447 | if len(pic) != 4: return False 448 | try: 449 | return all(0 <= int(p) < 256 for p in pic) 450 | except ValueError: 451 | return False 452 | 453 | elif format == 'mask': 454 | 455 | ips = ip_data.split('/') 456 | if len(ips) != 2: return False 457 | if (int(ips[1]) < 0) or (int(ips[1]) > 32): return False 458 | 459 | pic = ips[0].split('.') 460 | if len(pic) != 4: return False 461 | try: 462 | return all(0 <= int(p) < 256 for p in pic) 463 | except ValueError: 464 | return False 465 | 466 | elif format == 'segment': 467 | ips = ip_data.split('-') 468 | if len(ips) != 2: return False 469 | 470 | pic = ips[0].split('.') 471 | pic.extend(ips[1].split('.')) 472 | 473 | if len(pic) != 8: return False 474 | try: 475 | return all(0 <= int(p) < 256 for p in pic) 476 | except ValueError: 477 | return False 478 | 479 | else: 480 | return False 481 | 482 | class GetIpSegment(object): 483 | """ 484 | format ip/mask to an ip segment. 485 | eg: '192.168.1.5/26' to '192.168.1.0-192.168.1.63' 486 | """ 487 | 488 | def __init__(self, ip_mask): 489 | self.ip_mask = ip_mask 490 | 491 | def dec2bin(self, n): 492 | bin_str = bin(n).replace("0b", "") 493 | z_str = ''.join([str('0') for x in range(8-len(bin_str))]) 494 | return z_str + bin_str 495 | 496 | def bin2ip_str(self, bin_str): 497 | ip = [] 498 | for i in range(0, len(bin_str), 8): 499 | dec_str = str(int(bin_str[i:i+8], 2)) # bin2dec 500 | ip.append(dec_str) 501 | return ip[0]+"."+ip[1]+"."+ip[2]+"."+ip[3] 502 | 503 | def ip_str(self): 504 | 505 | ip = self.ip_mask.split("/")[0] 506 | mask = int(self.ip_mask.split("/")[1]) 507 | ip_lt = ip.split(".") 508 | ip_bin = ''.join([self.dec2bin(int(x)) for x in ip_lt]) 509 | ip_bin_p = ip_bin[0:mask] 510 | 511 | ip_min = ip_bin_p + ''.join([str('0') for i in range(32-mask)]) 512 | ip_max = ip_bin_p + ''.join([str('1') for i in range(32-mask)]) 513 | 514 | return (self.bin2ip_str(ip_min), self.bin2ip_str(ip_max)) 515 | 516 | 517 | class Echo(object): 518 | def write(self, value): 519 | return value 520 | 521 | 522 | class CSVWriter: 523 | """ 524 | A CSV writer which will write rows to CSV file "f" 525 | """ 526 | 527 | def __init__(self, f=None, dialect=csv.excel, **kwds): 528 | self.queue = cStringIO.StringIO() 529 | self.writer = csv.writer(self.queue, dialect=dialect, **kwds) 530 | self.stream = f if f else Echo() 531 | 532 | def writerow(self, row): 533 | self.writer.writerow(row) 534 | data = self.queue.getvalue() 535 | value = self.stream.write(data) 536 | self.queue.truncate(0) 537 | return value 538 | 539 | def writerows(self, rows): 540 | for row in rows: 541 | self.writerow(row) 542 | 543 | 544 | if __name__ == "__main__": 545 | def test1(): 546 | print yg_encode_check(u'我们都有一个家'.encode('gbk')) 547 | print yg_encode_check(u'我们都有一个家'.encode('utf-8')) 548 | print yg_encode_check("zxy") 549 | 550 | 551 | def test2(): 552 | print try_change(u'我们都有一个家'.encode('gbk')) 553 | 554 | 555 | def test3(): 556 | print get_file_line(os.path.abspath("../../data/ftp_root/test1")) 557 | 558 | 559 | # test2() 560 | # test3() 561 | #print GetIpSegment('192.168.1.1/0').ip_str() 562 | -------------------------------------------------------------------------------- /trunk/fs_server/src/bean/README.md: -------------------------------------------------------------------------------- 1 | this is bean module 2 | -------------------------------------------------------------------------------- /trunk/fs_server/src/dao/README.md: -------------------------------------------------------------------------------- 1 | this is dao module 2 | -------------------------------------------------------------------------------- /trunk/fs_server/src/fss_cfg.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf--8 -*- 2 | 3 | # project: fshell 4 | # author: s0nnet 5 | # time: 2016-12-09 6 | # desc: fs_server conf 7 | 8 | 9 | from fs_base_cfg import * 10 | 11 | class EnvEnum: 12 | T_DEV = "dev" 13 | T_ONLINE = "online" 14 | 15 | 16 | CUR_ENV = EnvEnum.T_DEV 17 | 18 | if CUR_ENV == EnvEnum.T_DEV: 19 | 20 | BaseConf.IS_CTR_LOG = True 21 | BaseConf.LOG_LEVEL = 1 22 | BaseConf.SQL_HOST = "127.0.0.1" 23 | BaseConf.SQL_PORT = 3306 24 | BaseConf.SQL_USER = "xxx" 25 | BaseConf.SQL_PASSWD = "xxx" 26 | BaseConf.SQL_DB = "db_fshell" 27 | 28 | SERVER_IP = "127.0.0.1" 29 | SERVER_PORT = 8000 30 | 31 | else: 32 | BaseConf.IS_CTR_LOG = True 33 | BaseConf.LOG_LEVEL = 1 34 | BaseConf.SQL_HOST = "127.0.0.1" 35 | BaseConf.SQL_PORT = 3306 36 | BaseConf.SQL_USER = "xxxx" 37 | BaseConf.SQL_PASSWD = "xxx" 38 | BaseConf.SQL_DB = "db_fshell" 39 | 40 | SERVER_IP = "127.0.0.1" 41 | SERVER_PORT = 8000 42 | 43 | 44 | # with local cfg, so do not modify the file when test in local env 45 | if os.path.exists(os.path.join(os.path.dirname(__file__), 'fss_local_cfg.py')): 46 | from fss_local_cfg import * 47 | -------------------------------------------------------------------------------- /trunk/fs_server/src/net/fss_net.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # project: fshell 4 | # author: s0nnet 5 | # time: 2016-12-09 6 | # desc: 完成网络接收，并存入mysql 7 | 8 | 9 | import select 10 | from fss_cfg import * 11 | from fss_nethead import * 12 | 13 | 14 | ''' 15 | 目的：监听端口，完成解码进行回调 16 | server, 提供事件驱动，但采用阻塞模式 17 | ''' 18 | 19 | class FsNetMode: 20 | T_SYN_MODE = 0 # 请求应答模式 21 | T_ASY_MODE = 1 # 异步模式 22 | 23 | 24 | class FsNetConf: 25 | 26 | def __init__(self): 27 | self.ip = None 28 | self.port = None 29 | self.mode = None 30 | self.acceptFun = None 31 | self.pkgFun = None 32 | self.closeFun = None 33 | 34 | class FsSession: 35 | 36 | class Status: 37 | T_ACCEPT = 0 38 | T_CONNECTING = 1 39 | 40 | def __init__(self): 41 | self.ip = None 42 | self.port = None 43 | self.sock = -1 44 | self.conf =None 45 | self.status = None 46 | self.extra = None 47 | 48 | @staticmethod 49 | def new_session(sock, ip, port, status, conf): 50 | session = FsSession() 51 | session.ip = ip 52 | session.port = port 53 | session.sock = sock 54 | session.status = status 55 | session.conf = conf 56 | session.buf = "" 57 | 58 | return session 59 | 60 | 61 | class FsNet: 62 | 63 | READ_ONLY = (select.POLLIN | select.POLLPRI | select.POLLHUP | select.POLLERR) 64 | 65 | def __init__(self): 66 | self.confList = [] 67 | self.sessionMap = {} 68 | self.poller = select.poll() 69 | 70 | 71 | def append(self, ip, port, mode, acceptFun, pkgFun, closeFun): 72 | node = FsNetConf() 73 | node.ip = ip 74 | node.port = port 75 | node.mode = mode 76 | node.acceptFun = acceptFun 77 | node.pkgFun = pkgFun 78 | node.closeFun = closeFun 79 | 80 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM); 81 | sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 82 | sock.bind((ip, port)) 83 | sock.listen(1024) 84 | session = FsSession.new_session(sock, ip, port, FsSession.Status.T_ACCEPT, node) 85 | self.sessionMap[session.sock.fileno()] = session 86 | 87 | self.poller.register(session.sock, FsNet.READ_ONLY) 88 | self.confList.append(node) 89 | 90 | return True, "" 91 | 92 | 93 | def _accept_conn(self, session, conf): 94 | conn, clientAddr = session.sock.accept() 95 | ip, port = clientAddr 96 | 97 | cliSession = FsSession.new_session(conn, ip, port, FsSession.Status.T_CONNECTING, conf) 98 | 99 | if conf.acceptFun != None: 100 | bAccept = conf.acceptFun(session, cliSession) 101 | if not bAccept: 102 | conn.close() 103 | Log.info("srv(%s, %d) not accept(%s, %d)" %(session.ip, session.port, ip, port)) 104 | return True, "" 105 | 106 | self.sessionMap[conn.fileno()] = cliSession 107 | self.poller.register(cliSession.sock, FsNet.READ_ONLY) 108 | 109 | return True, "" 110 | 111 | 112 | def _close_session(self, session, conf): 113 | if session.status == FsSession.Status.T_ACCEPT: 114 | Log.err("server sock(%s, %d) ERR!...." %(session.ip, session.fd)) 115 | return 116 | 117 | if conf.closeFun != None: 118 | conf.closeFun(session) 119 | 120 | if session.sock != None: 121 | self.poller.unregister(session.sock) 122 | del self.sessionMap[session.sock.fileno()] 123 | session.sock.close() 124 | session.sock = None 125 | 126 | 127 | def _deal_pkg(self, session, conf, packet, bodyType): 128 | try: 129 | #Log.debug("deal_pkg: %s" %(packet)) 130 | packet = unicode(packet, errors='ignore') 131 | reqJson = json.loads(packet) 132 | if conf.pkgFun != None: 133 | return conf.pkgFun(session, reqJson) 134 | 135 | except Exception, e: 136 | Log.err(traceback.format_exc()) 137 | return False, str(e) 138 | 139 | return True, "" 140 | 141 | # 接收一个完整的数据包， 如果包不合法直接短链接 142 | def _read_pkg(self, session, conf): 143 | 144 | sock = session.sock 145 | buf = sock.recv(16 * 1024) 146 | if buf == None or len(buf) == 0: 147 | self._close_session(session, conf) 148 | return False, "client close connection" 149 | 150 | buf = session.buf + buf 151 | session.buf = "" 152 | 153 | while True: 154 | if len(buf) < NetHead.size(): 155 | session.buf = buf 156 | #self._close_session(session, conf) 157 | return True, "" 158 | #return False, "buf_len: %d < len(NetHead)" %(len(buf)) 159 | 160 | netHead = NetHead.decode(buf) 161 | if netHead == None: 162 | self._close_session(session, conf) 163 | return False, "NetHead.decode ERR" 164 | 165 | pkgLen = netHead.pkgLen 166 | while len(buf) < pkgLen: 167 | sock.settimeout(3) 168 | val = sock.recv(16 * 1024) 169 | sock.settimeout(None) 170 | if val == None or len(val) == 0: 171 | self._close_session(session, conf) 172 | return False, "client close connection" 173 | 174 | buf = buf + val 175 | 176 | packet = buf[NetHead.size() : pkgLen] 177 | bRet, sRet = self._deal_pkg(session, conf, packet, netHead.type) 178 | if not bRet: 179 | Log.err("_deal_pkg ERR(%s)!" %(str(sRet))) 180 | self._close_session(session, conf) 181 | return False, "_deal_pkg ERR(%s)!" %(str(sRet)) 182 | 183 | if len(buf) > pkgLen: 184 | buf = buf[pkgLen : len(buf)] 185 | continue 186 | 187 | return True, "" 188 | 189 | def send_req(self, session, cmdInfo): 190 | 191 | netHead = NetHead() 192 | netHead.start = NetHead.START_CODE 193 | netHead.type = 0 194 | netHead.pkgLen = len(cmdInfo) + NetHead.size() 195 | headStr = NetHead.encode(netHead) 196 | rspStr = headStr + cmdInfo 197 | 198 | sendlen = session.sock.send(rspStr) 199 | Log.debug("sendLen: %d" %(sendlen)) 200 | 201 | return True, "" 202 | 203 | 204 | def run(self): 205 | 206 | while True: 207 | events = self.poller.poll(1000) 208 | for fd, flag in events: 209 | try: 210 | #print fd, flag 211 | if not self.sessionMap.has_key(fd): 212 | continue 213 | 214 | session = self.sessionMap[fd] 215 | conf = session.conf 216 | 217 | if flag & select.POLLIN or flag & flag & select.POLLPRI: 218 | if session.status == FsSession.Status.T_ACCEPT: 219 | 220 | bRet, sRet = self._accept_conn(session, conf) 221 | if not bRet: 222 | Log.err("accept_conn ERR: %s" %(sRet)) 223 | else: 224 | bRet, sRet = self._read_pkg(session, conf) 225 | if not bRet: 226 | Log.err("_read_req(%s, %d) conf:(%s, %d) ERR: %s" %(session.ip, session.port, conf.ip, conf.port, sRet)) 227 | 228 | if conf.mode == FsNetMode.T_SYN_MODE: 229 | self._close_session(session, conf) 230 | 231 | 232 | elif flag & select.POLLHUP or flag & select.POLLERR: 233 | self._close_session(session, conf) 234 | 235 | except Exception, e: 236 | Log.err("%s, %s" %(traceback.format_exc(), str(e))) 237 | 238 | 239 | def close_session(self, session): 240 | conf = session.conf 241 | self._close_session(session, conf) 242 | 243 | 244 | 245 | g_fsNet = FsNet() 246 | -------------------------------------------------------------------------------- /trunk/fs_server/src/net/fss_nethead.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # project: fshell 4 | # author: s0nnet 5 | # time: 2016-12-09 6 | # desc: 完成数据包编/解码 7 | 8 | 9 | import socket 10 | import sys 11 | import os 12 | import time 13 | 14 | from fs_log import * 15 | 16 | 17 | class NetHead: 18 | 19 | START_CODE = 0x11111111 20 | 21 | def __init__(self): 22 | self.start = 0 23 | self.type = 0 24 | self.pkgLen = 0 25 | 26 | @staticmethod 27 | def size(): 28 | return 9 29 | 30 | @staticmethod 31 | def str2int(s): 32 | if len(s) < 4: return None 33 | 34 | a = s[0 : 1] 35 | b = s[1 : 2] 36 | c = s[2 : 3] 37 | d = s[3 : 4] 38 | 39 | ts = (ord(a) << 24) + (ord(b) << 16) + (ord(c) << 8) + ord(d) 40 | return ts 41 | 42 | @staticmethod 43 | def int2str(n): 44 | 45 | d = n & 0xff; n = n >> 8; 46 | c = n & 0xff; n = n >> 8; 47 | b = n & 0xff; n = n >> 8; 48 | a = n & 0xff; n = n >> 8; 49 | 50 | ts = chr(a) + chr(b) + chr(c) + chr(d) 51 | return ts 52 | 53 | 54 | @staticmethod 55 | def decode(data): 56 | 57 | if data == None or len(data) < 9: 58 | Log.err("data: %s is not valid!" %(str(data))) 59 | return None 60 | 61 | netHead = NetHead() 62 | netHead.start = NetHead.str2int(data[0 : 4]) 63 | netHead.type = ord(data[4 : 5]) 64 | netHead.pkgLen = NetHead.str2int(data[5 : 9]) 65 | 66 | if netHead.start != NetHead.START_CODE: 67 | Log.err("netHead not valid(%d, %d, %d)" %(netHead.start, netHead.type, netHead.pkgLen)); 68 | return None 69 | 70 | Log.debug("netHead(start=%d, type=%d, pkgLen=%d)" %(netHead.start, netHead.type, netHead.pkgLen)); 71 | return netHead 72 | 73 | 74 | @staticmethod 75 | def encode(netHead): 76 | start = NetHead.int2str(netHead.start) 77 | type = chr(netHead.type) 78 | pkgLen = NetHead.int2str(netHead.pkgLen) 79 | cnt = start + type + pkgLen 80 | 81 | Log.debug("netHead_encode: cnt_len: %d" %(len(cnt))) 82 | return cnt 83 | -------------------------------------------------------------------------------- /trunk/fs_server/src/route/fss_proto_type.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # project: fshell 4 | # author: s0nnet 5 | # time: 2016-12-11 6 | # desc: 通信协议定义 7 | 8 | 9 | class FsProtoProtoEnum: 10 | T_HP_EVIL_UP = 0x01 11 | T_HP_STAT_REPORT = 0x02 12 | T_HP_CMD_REQ = 0x03 13 | T_HP_CMD_RSP = 0x04 14 | 15 | 16 | class FsProtoTypeEnum: 17 | # 获取配置 18 | T_HPC_ROUTE_SYN_IP_REQ = 0x01 19 | T_HPC_ROUTE_ASY_IP_REQ = 0x02 20 | T_HPC_GET_PLUGIN_ALL_CONF = 0x03 21 | 22 | # 服务端命令下发(0x-60 ~ 0xa0) 23 | T_HPC_PLUGIN_NEW = 0x60 24 | T_HPC_PLUGIN_START = 0x61 25 | T_HPC_PLUGIN_STOP = 0x62 26 | T_HPC_PROXY_RULE_UPDATE = 0x63 27 | T_HPC_PLUGIN_ADD_IP = 0x64 28 | T_HPC_PLUGIN_DESTROY = 0x65 29 | T_HPC_PLUGIN_DEL_IP = 0x66 30 | T_HPC_CLIENT_RESET = 0x67 31 | 32 | # 心跳和错误上报 33 | T_HPC_HEART_BEAT = 0xa0 34 | T_HPC_CLIENT_REPORT = 0xa1 35 | T_HPC_PLUGIN_REPORT = 0xa2 36 | T_HPC_KVM_ERR = 0xa3 37 | T_HP_ERR = 0xff 38 | 39 | -------------------------------------------------------------------------------- /trunk/fs_server/src/route/fss_srv.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # project: fshell 4 | # author: s0nnet 5 | # time: 2016-12-09 6 | # desc: server 数据接入模块主函数 7 | 8 | 9 | import os 10 | path = os.path.dirname(os.path.realpath(__file__)) 11 | os.chdir(path) 12 | 13 | import sys 14 | reload(sys) 15 | sys.setdefaultencoding('utf-8') 16 | 17 | sys.path.append("..") 18 | sys.path.append("../net") 19 | sys.path.append("../base") 20 | sys.path.append("../dao") 21 | sys.path.append("../bean") 22 | 23 | from fss_cfg import * 24 | from fss_net import * 25 | from fss_srv_manager import * 26 | 27 | 28 | class FsSrv: 29 | 30 | def __init__(self): 31 | 32 | self.fsNet = FsNet() 33 | self.managerSrv = FsManagerSrv() 34 | 35 | 36 | def init(self): 37 | 38 | self.fsNet = g_fsNet 39 | self.fsNet.append(self.managerSrv.ip, self.managerSrv.port, self.managerSrv.mode, 40 | None, self.managerSrv.deal_pkg, None) 41 | 42 | return True, None 43 | 44 | 45 | def run(self): 46 | 47 | self.fsNet.run() 48 | 49 | 50 | if __name__ == "__main__": 51 | 52 | fsSrv = FsSrv() 53 | bRet = fsSrv.init() 54 | if bRet: 55 | fsSrv.run() 56 | 57 | 58 | -------------------------------------------------------------------------------- /trunk/fs_server/src/route/fss_srv_base.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # project: fshell 4 | # author: s0nnet 5 | # time: 2016-12-11 6 | # desc: sRetv base 7 | 8 | 9 | from fss_cfg import * 10 | from fss_net import * 11 | 12 | class FsContentProto: 13 | 14 | @staticmethod 15 | def check_valid(reqJson): 16 | def _check_valid(reqJson): 17 | if not reqJson.has_key("task_id"): 18 | return False, "task_id is not exist!" 19 | 20 | if not reqJson.has_key("dev_name"): 21 | return False, "dev_name is not exist!" 22 | 23 | if not reqJson.has_key("agent_id"): 24 | return False, "agent_id is not exist" 25 | 26 | if not reqJson.has_key("msg_protocol"): 27 | return False, "msg_protocol is not exist!" 28 | 29 | if not reqJson.has_key("msg_type"): 30 | return False, "msg_type is not exist!" 31 | 32 | if not reqJson.has_key("data"): 33 | return False, "data is not exist!" 34 | 35 | return True, "" 36 | 37 | bRet, sRet = _check_valid(reqJson) 38 | if not bRet: 39 | Log.err("reqJson: %s not valid(%s)" %(reqJson, sRet)) 40 | return bRet 41 | 42 | 43 | @staticmethod 44 | def response_packet(session, reqJson, rspProto, rspType, rspData): 45 | rspJson = {} 46 | rspJson['task_id'] = reqJson['task_id'] 47 | rspJson['dev_name'] = reqJson['dev_name'] 48 | rspJson['agent_id'] = reqJson['agent_id'] 49 | rspJson['msg_protocol'] = rspProto 50 | rspJson['msg_type'] = rspType 51 | rspJson['data'] = rspData 52 | rspStr = json.dumps(rspJson) 53 | g_fsNet.send_req(session, rspStr) 54 | 55 | return True, "" 56 | -------------------------------------------------------------------------------- /trunk/fs_server/src/route/fss_srv_manager.py: -------------------------------------------------------------------------------- 1 | # -*- coding: UTF-8 -*- 2 | 3 | # project: fshell 4 | # author: s0nnet 5 | # time: 2016-12-11 6 | # desc: srv manager 7 | 8 | 9 | 10 | from fss_cfg import * 11 | from fss_net import * 12 | from fss_srv_base import * 13 | from fss_proto_type import * 14 | 15 | #from fs_report_info_dao import * 16 | 17 | 18 | class FsManagerSrv: 19 | 20 | def __init__(self): 21 | self.ip = SERVER_IP 22 | self.port = SERVER_PORT 23 | self.mode = FsNetMode.T_SYN_MODE 24 | 25 | 26 | def deal_pkg(self, session, reqJson): 27 | 28 | bRet = FsContentProto.check_valid(reqJson) 29 | if not bRet: return False, "" 30 | 31 | # 记录心跳 32 | """ 33 | HpsReportInfoDao.insert_node( 34 | reqJson['task_id'], 35 | reqJson['dev_name'], 36 | reqJson['plugin_id'], 37 | reqJson['msg_protocol'], 38 | reqJson['msg_type'], 39 | reqJson['data']) 40 | """ 41 | 42 | proto = reqJson['msg_type'] 43 | rspProto = "" 44 | rspData = "" 45 | 46 | """ 47 | if proto == HpProtoTypeEnum.T_HPC_ROUTE_SYN_IP_REQ: 48 | rspProto = HpProtoTypeEnum.T_HPC_ROUTE_SYN_IP_REQ 49 | rspJson = { 50 | 'route_syn_ip' : ROUTE_SYN_IP, 51 | 'route_syn_port' : ROUTE_SYN_PORT 52 | } 53 | 54 | rspData = json.dumps(rspJson) 55 | 56 | elif proto == HpProtoTypeEnum.T_HPC_ROUTE_ASY_IP_REQ: 57 | rspProto = HpProtoTypeEnum.T_HPC_ROUTE_ASY_IP_REQ 58 | rspJson = { 59 | 'route_asy_ip' : ROUTE_ASY_IP, 60 | 'route_asy_port' : ROUTE_ASY_PORT 61 | } 62 | 63 | rspData = json.dumps(rspJson) 64 | 65 | else: 66 | Log.err("proto: %d not valid!" %(proto)) 67 | rspProto = HpProtoTypeEnum.T_HP_ERR 68 | rspData = "" 69 | """ 70 | 71 | 72 | return FsContentProto.response_packet(session, reqJson, FsProtoProtoEnum.T_HP_CMD_RSP, rspProto, rspData) 73 | --------------------------------------------------------------------------------