├── .gitignore ├── README.md ├── synWatch.py ├── log.py └── ftp.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | *.so 3 | *.egg 4 | *.egg-info 5 | *.bak 6 | logs/* 7 | .idea 8 | .DS_Store 9 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 毕业在即，闲着无聊，偶然发现腾讯云主机正在搞活动，于是买了一台云主机。想用跑一些日常小任务，如爬虫。然而在云主机上利用vim进行程序开发实在麻烦，本地开发完程序还需手动同步到服务端，太过麻烦。 2 | [搭建samba](https://blog.csdn.net/linhai1028/article/details/80200256)后发现mac访问腾讯云主机上的samba速度很慢，ftp速度还是比较快的。因此就想着自己写一个自动同步本地代码到云主机的python脚本。 3 | 4 | ----- 5 | 6 | ## 一、搭建ftp服务器 7 | 搭建ftp服务器见文章[ubuntu16.04搭建ftp服务器](https://blog.csdn.net/linhai1028/article/details/80197254) 8 | 9 | ## 二、思路简介 10 | 本文工具主要利用watchdog对文件夹做监听，如发现文件夹中的文件有移动、创建、重命名、修改操作，那么就把对应的修改上传到ftp服务器。如文件夹某文件删除了，那么不对服务端的数据做处理。也就是对ftp服务器的数据做增量更新。 11 | 12 | ## 三、模块简介 13 | ### 1. 日志处理模块 14 | 日志处理模块主要是设置日志格式，日志输出等，其代码对应于文log.py 15 | ### 2. ftp模块 16 | 主要是基于ftplib库，利用python代码实现文件从ftp服务器上传、下载、删除、移动等，主要代码ftp.py 17 | ### 3. watchdog监听模块 18 | 利用watchdog对文件夹做监听，如发现文件夹中的文件有移动、创建 19 | 、重命名、修改操作，那么就把对应的修改上传到ftp服务器。如文件夹某文件删除了，那么不对服务端的数据做处理。 20 | 21 | ## 四、使用 22 | 23 | ### 1. [下载项目代码](https://github.com/hailinli/synWatch/archive/7080c7d58b7852927caa448dfef0bc41e38dfe38.zip) 24 | 修改synWatch.py中的配置信息，配置信息有ftp服务器的ip地址，ftp用户名、密码。 25 | 26 | ``` 27 | ip = 'x.x.x.x' 28 | username = 'xxxx' 29 | passwd = 'xxxx' 30 | ``` 31 | ### 2. 进行文件监听 32 | ``` 33 | # yourpath为你要监听的本地目录 34 | python synWatch.py yourpath 35 | ``` 36 | 37 | ------ 38 | 39 | ## 环境 40 | 1. MacOS X 10.11.6 41 | 2. python2.7 42 | 3. watchdog 0.83 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /synWatch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | __author__ = 'lihailin' 4 | __mail__ = '415787837@qq.com' 5 | __date__ = '2018-05-03' 6 | __version__ = 1.0 7 | 8 | import sys 9 | import time 10 | import ftp 11 | # 配置日志 12 | import logging 13 | import log 14 | log.initLogConf() 15 | _logging = logging.getLogger(__file__) 16 | from watchdog.observers import Observer 17 | from watchdog.events import FileSystemEventHandler 18 | reload(sys) 19 | sys.setdefaultencoding('utf-8') 20 | 21 | 22 | # 配置ftp 23 | ip = 'x.x.x.x' 24 | username = 'xxxx' 25 | passwd = 'xxxx' 26 | 27 | class ToServer(FileSystemEventHandler): 28 | ''' 29 | 将本地文件变化事件记录到日志中，并上传到服务器 30 | ''' 31 | def __init__(self, path, ip, username, passwd): 32 | super(ToServer, self).__init__() 33 | # 配置ftp服务器 34 | self.xfer = ftp.Xfer() 35 | self.xfer.setFtpParams(ip, username, passwd) 36 | # 开启服务时上传一遍文件至远程文件夹 37 | self.xfer.upload(path) 38 | 39 | 40 | def on_any_event(self, event): 41 | # 将发生过的事件写入日志 42 | if event.is_directory: 43 | is_d = 'directory' 44 | else: 45 | is_d = 'file' 46 | log_s = "%s %s: %s " % (event.event_type, is_d, event.src_path) 47 | # print log_s 48 | _logging.info(log_s) 49 | 50 | 51 | def on_created(self, event): 52 | # 仅上传文件 53 | if event.is_directory: 54 | return 55 | self.xfer.upload(event.src_path) 56 | 57 | 58 | def on_modified(self, event): 59 | self.on_created(event) 60 | 61 | 62 | def on_moved(self, event): 63 | ''' 64 | 服务器中的文件进行移动 65 | ''' 66 | self.xfer.rename(event.src_path, event.dest_path) 67 | # log_s = "move file: %s to %s" % (event.src_path, event.dest_path) 68 | # _logging.info(log_s) 69 | 70 | 71 | if __name__ == "__main__": 72 | path = sys.argv[1] if len(sys.argv) > 1 else '.' 73 | event_handler = ToServer(path, ip, username, passwd) 74 | observer = Observer() 75 | observer.schedule(event_handler, path, recursive=True) 76 | observer.start() 77 | try: 78 | while True: 79 | time.sleep(1) 80 | except KeyboardInterrupt: 81 | observer.stop() 82 | observer.join() 83 | -------------------------------------------------------------------------------- /log.py: -------------------------------------------------------------------------------- 1 | #coding=utf-8 2 | import os 3 | import datetime 4 | import logging 5 | import logging.config 6 | 7 | def genLogDict(logDir, logFile): 8 | ''' 9 | 配置日志格式的字典 10 | ''' 11 | logDict = { 12 | "version": 1, 13 | "disable_existing_loggers": False, 14 | "formatters": { 15 | "simple": { 16 | 'format': '%(asctime)s [%(name)s:%(lineno)d] [%(levelname)s]- %(message)s' 17 | }, 18 | 'standard': { 19 | 'format': '%(asctime)s [%(threadName)s:%(thread)d] [%(name)s:%(lineno)d] [%(levelname)s]- %(message)s' 20 | }, 21 | }, 22 | 23 | "handlers": { 24 | "console": { 25 | "class": "logging.StreamHandler", 26 | "level": "DEBUG", 27 | "formatter": "simple", 28 | "stream": "ext://sys.stdout" 29 | }, 30 | 31 | "default": { 32 | "class": "logging.handlers.RotatingFileHandler", 33 | "level": "INFO", 34 | "formatter": "simple", 35 | "filename": os.path.join(logDir, logFile), 36 | 'mode': 'w+', 37 | "maxBytes": 1024*1024*5, # 5 MB 38 | "backupCount": 20, 39 | "encoding": "utf8" 40 | }, 41 | }, 42 | 43 | # "loggers": { 44 | # "app_name": { 45 | # "level": "INFO", 46 | # "handlers": ["console"], 47 | # "propagate": "no" 48 | # } 49 | # }, 50 | 51 | "root": { 52 | 'handlers': ['default'], 53 | 'level': "INFO", 54 | 'propagate': False 55 | } 56 | } 57 | return logDict 58 | 59 | 60 | def initLogConf(): 61 | """ 62 | 配置日志 63 | """ 64 | baseDir = os.path.dirname(os.path.abspath(__file__)) 65 | logDir = os.path.join(baseDir, "logs") 66 | if not os.path.exists(logDir): 67 | os.makedirs(logDir) # 创建路径 68 | 69 | logFile = datetime.datetime.now().strftime("%Y-%m-%d") + ".log" 70 | logDict = genLogDict(logDir, logFile) 71 | logging.config.dictConfig(logDict) 72 | 73 | if __name__ == '__main__': 74 | initLogConf() 75 | log = logging.getLogger(__file__) 76 | print "print A" 77 | log.info("log B") -------------------------------------------------------------------------------- /ftp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | __author__ = 'lihailin' 4 | __mail__ = '415787837@qq.com' 5 | __date__ = '2018-05-03' 6 | __version__ = 1.0 7 | 8 | import sys 9 | import os 10 | from ftplib import FTP 11 | import logging 12 | import log 13 | log.initLogConf() 14 | _logging = logging.getLogger(__file__) 15 | reload(sys) 16 | sys.setdefaultencoding('utf-8') 17 | 18 | 19 | import socket 20 | timeout = 1000 21 | socket.setdefaulttimeout(timeout) 22 | 23 | class Xfer(object): 24 | ''''' 25 | 上传本地文件或目录递归上传到FTP服务器 26 | ''' 27 | def __init__(self): 28 | self.ftp = None 29 | 30 | 31 | def setFtpParams(self, ip, uname, pwd, port = 21, timeout = 60): 32 | # 设置ftp参数 33 | self.ip = ip 34 | self.uname = uname 35 | self.pwd = pwd 36 | self.port = port 37 | self.timeout = timeout 38 | 39 | 40 | def initEnv(self): 41 | # 链接ftp 42 | if self.ftp is None: 43 | self.ftp = FTP() 44 | _logging.debug('### connect ftp server: %s ...'%self.ip) 45 | self.ftp.connect(self.ip, self.port, self.timeout) 46 | self.ftp.login(self.uname, self.pwd) 47 | _logging.debug(self.ftp.getwelcome()) 48 | 49 | 50 | def clearEnv(self): 51 | # ftp断开链接 52 | if self.ftp: 53 | self.ftp.close() 54 | # self.ftp.quit(), close和quit只能选一个 55 | _logging.debug('### disconnect ftp server: %s!'%self.ip) 56 | self.ftp = None 57 | 58 | 59 | def uploadFile(self, localpath, remotepath='./'): 60 | ''' 61 | 上传文件至服务端，为了保存本地和服务器路径一致 62 | loacalpath路径文件夹需是单层，如'a/b/c.txt'是不行的。 63 | ''' 64 | if not os.path.isfile(localpath): 65 | return 66 | _logging.info('+++ uploading %s to %s:%s'%(localpath, self.ip, remotepath)) 67 | try: 68 | self.ftp.storbinary('STOR ' + remotepath, open(localpath, 'rb')) 69 | _logging.info('upload success: %s '%localpath) 70 | except Exception as e: 71 | _logging.info('upload fail %s : %s'%(localpath, e)) 72 | 73 | 74 | def uploadDir(self, localdir='./', remotedir='./'): 75 | ''' 76 | localdir 里面的文件以及全部同步到服务器的remotedir 77 | 为了保存本地和服务器路径一致，localdir路径需是单层，如'a/b/c'是不行的 78 | ''' 79 | # print sdfsdf 为了产生bug方便测试 80 | 81 | if not os.path.isdir(localdir): 82 | return 83 | self.ftp.cwd(remotedir) 84 | for file in os.listdir(localdir): 85 | src = os.path.join(localdir, file) 86 | if os.path.isfile(src): 87 | self.uploadFile(src, file) 88 | elif os.path.isdir(src): 89 | try: 90 | self.ftp.mkd(file) 91 | except: 92 | # sys.stderr.write('the dir is exists %s'%file) 93 | # _logging.error('the dir is exists %s'%file) 94 | pass 95 | self.uploadDir(src, file) 96 | self.ftp.cwd('..') 97 | 98 | 99 | def walkLastServer(self, src): 100 | ''' 101 | 在服务端递归创建文件夹，如'a/b/c' 102 | 并将cd到最里层文件夹 103 | ''' 104 | deldictorys = src.split('/') 105 | for d in deldictorys[:-1]: 106 | try: 107 | self.ftp.mkd(d) 108 | except: 109 | pass 110 | self.ftp.cwd(d) 111 | return deldictorys[-1] 112 | 113 | 114 | def _upload(self, src): 115 | ''' 116 | 可同时上传文件以及文件夹 117 | 文件夹及文件路径可含有多层，'desk/za/a.txt'是可行的 118 | ''' 119 | self.initEnv() 120 | desSrc = self.walkLastServer(src) 121 | if os.path.isdir(desSrc): 122 | try: 123 | self.ftp.mkd(src) 124 | except: 125 | pass 126 | try: 127 | log_s = "+++ uploading directory %s" % desSrc 128 | _logging.info(log_s) 129 | self.uploadDir(desSrc, desSrc) 130 | log_s = "upload directory sucess: %s" % desSrc 131 | _logging.info(log_s) 132 | except Exception as e: 133 | _logging.error(e) 134 | log_s = "upload directory fail: %s" % desSrc 135 | _logging.info(log_s) 136 | elif os.path.isfile(src): 137 | self.uploadFile(src, desSrc) 138 | self.clearEnv() 139 | 140 | 141 | def upload(self, src, times=3): 142 | ''' 143 | 上传文件是一个耗时的操作，防止timeout 144 | ''' 145 | try: 146 | self._upload(src) 147 | except Exception as e: 148 | _logging.error(e) 149 | times -= 1 150 | if times>0: 151 | _logging.info('======重传文件: %s ======' % src) 152 | self.upload(src,times) 153 | 154 | 155 | def rename(self, fromname, toname): 156 | self.initEnv() 157 | logging.info('+++ rename %s to %s'%(fromname, toname)) 158 | try: 159 | self.ftp.rename(fromname, toname) 160 | logging.info('rename success: %s' %fromname) 161 | except Exception as e: 162 | _logging.error(e) 163 | logging.info('rename fail %s : %s' %(fromname, e)) 164 | 165 | self.clearEnv() 166 | 167 | 168 | # def delFile(self, remotefile): 169 | # ''' 170 | # 删除文件，路径可是多层 171 | # ''' 172 | # if not os.path.isfile(remotefile): 173 | # return 174 | # self.initEnv() 175 | # self.ftp.delete(remotefile) 176 | # self.clearEnv() 177 | 178 | 179 | # def delDictory(self, remotedic): 180 | # ''' 181 | # 删除文件夹，路径可是多层 182 | # ''' 183 | # self.initEnv() 184 | # for a,b,c in os.walk(remotedic,topdown=False): 185 | # for ai in a: 186 | # for ci in c: 187 | # self.delfile(os.path.join(ai, ci)) 188 | # self.ftp.rmd(ai) 189 | # self.clearEnv() 190 | 191 | 192 | if __name__ == '__main__': 193 | 194 | ip = 'xxxx.xxxx.xxxx.xxxx' 195 | username = 'xxxx' 196 | passwd = 'xxx' 197 | 198 | # to = 'offer/tencent-实习offer' 199 | # src = 'offer/相关资料/tencent-实习offer' 200 | # xfer = Xfer() 201 | # xfer.setFtpParams(ip, username, passwd) 202 | # xfer.rename(src, to) 203 | 204 | # #测试 205 | # srcFile = r'offer' 206 | # srcFile = 'offer/相关资料/成绩单.docx' 207 | # xfer = Xfer() 208 | # xfer.setFtpParams(ip, username, passwd) 209 | # xfer.upload(srcFile) --------------------------------------------------------------------------------