├── .gitattributes ├── .gitignore ├── readme.md ├── archive ├── qCloud_COS_Sync_v4.py ├── qCloud_COS_Sync_v4_py3.py └── qCloud_COS_Sync_v5_function.py └── qCloud_COS_Sync.py /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | 4 | # Custom for Visual Studio 5 | *.cs diff=csharp 6 | 7 | # Standard to msysgit 8 | *.doc diff=astextplain 9 | *.DOC diff=astextplain 10 | *.docx diff=astextplain 11 | *.DOCX diff=astextplain 12 | *.dot diff=astextplain 13 | *.DOT diff=astextplain 14 | *.pdf diff=astextplain 15 | *.PDF diff=astextplain 16 | *.rtf diff=astextplain 17 | *.RTF diff=astextplain 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # My Ignore List 2 | *private* 3 | 4 | 5 | 6 | # Byte-compiled / optimized / DLL files 7 | __pycache__/ 8 | *.py[cod] 9 | *$py.class 10 | 11 | # C extensions 12 | *.so 13 | 14 | # Distribution / packaging 15 | .Python 16 | env/ 17 | build/ 18 | develop-eggs/ 19 | dist/ 20 | downloads/ 21 | eggs/ 22 | .eggs/ 23 | lib/ 24 | lib64/ 25 | parts/ 26 | sdist/ 27 | var/ 28 | *.egg-info/ 29 | .installed.cfg 30 | *.egg 31 | 32 | # PyInstaller 33 | # Usually these files are written by a python script from a template 34 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 35 | *.manifest 36 | *.spec 37 | 38 | # Installer logs 39 | pip-log.txt 40 | pip-delete-this-directory.txt 41 | 42 | # Unit test / coverage reports 43 | htmlcov/ 44 | .tox/ 45 | .coverage 46 | .coverage.* 47 | .cache 48 | nosetests.xml 49 | coverage.xml 50 | *,cover 51 | .hypothesis/ 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Django stuff: 58 | *.log 59 | local_settings.py 60 | 61 | # Flask instance folder 62 | instance/ 63 | 64 | # Scrapy stuff: 65 | .scrapy 66 | 67 | # Sphinx documentation 68 | docs/_build/ 69 | 70 | # PyBuilder 71 | target/ 72 | 73 | # IPython Notebook 74 | .ipynb_checkpoints 75 | 76 | # pyenv 77 | .python-version 78 | 79 | # celery beat schedule file 80 | celerybeat-schedule 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | venv/ 87 | ENV/ 88 | 89 | # Spyder project settings 90 | .spyderproject 91 | 92 | # Rope project settings 93 | .ropeproject 94 | 95 | # ========================= 96 | # Operating System Files 97 | # ========================= 98 | 99 | # OSX 100 | # ========================= 101 | 102 | .DS_Store 103 | .AppleDouble 104 | .LSOverride 105 | 106 | # Thumbnails 107 | ._* 108 | 109 | # Files that might appear in the root of a volume 110 | .DocumentRevisions-V100 111 | .fseventsd 112 | .Spotlight-V100 113 | .TemporaryItems 114 | .Trashes 115 | .VolumeIcon.icns 116 | 117 | # Directories potentially created on remote AFP share 118 | .AppleDB 119 | .AppleDesktop 120 | Network Trash Folder 121 | Temporary Items 122 | .apdisk 123 | 124 | # Windows 125 | # ========================= 126 | 127 | # Windows image file caches 128 | Thumbs.db 129 | ehthumbs.db 130 | 131 | # Folder config file 132 | Desktop.ini 133 | 134 | # Recycle Bin used on file shares 135 | $RECYCLE.BIN/ 136 | 137 | # Windows Installer files 138 | *.cab 139 | *.msi 140 | *.msm 141 | *.msp 142 | 143 | # Windows shortcuts 144 | *.lnk 145 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # 功能 2 | * 这是一个腾讯云COS的同步工具（仅上传更新过的文件）。 3 | * 可指定更新个别目录。 4 | * 同步源 **以本地文件为准，镜像到COS，让COS上的文件和本地一致。** 5 | * 会删除COS上多余的文件。 **不会改动本地文件**。 6 | * 我是用来把本地生成的静态页面同步到COS的。 7 | 8 | **【重要】使用前务必做好备份！做好备份！做好备份！** 9 | 10 | 11 | 12 | # 安装环境 13 | 首先，安装官方的SDK。 14 | ```python 15 | pip install cos-python-sdk-v5 16 | ``` 17 | [官方 Python SDK 文档](https://cloud.tencent.com/document/product/436/12270) 18 | 19 | 20 | 21 | # 配置参数 22 | 看代码最下端 23 | ## 初始化客户端 24 | 在【密钥管理】找到appid和配套的key，填写下面3个值。 25 | ```python 26 | appid = 88888888 # your appid 27 | secret_id = 'your_id' 28 | secret_key = 'your_key' 29 | ``` 30 | 31 | ## 地区列表 32 | 可以看自己的bucket下的域名管理/静态地址/`cos.`后面那段。 33 | ```python 34 | region_info = 'ap-shanghai' 35 | ``` 36 | 37 | ## 填写同步目录 38 | 设定本地特定目录为root，对应bucket根目录。还可以指定仅更新某个目录。 39 | 40 | **bucket** 前面appid对应项目下的bucket name。 41 | ```python 42 | bucket_name = 'your_bucket_name' 43 | ``` 44 | 45 | **root** 本地根目录，对应bucket根目录。我两台电脑，所以适配了win和mac。 46 | ```python 47 | if os.name == 'nt': 48 | root = 'D:/OneDrive/yourRoot' # PC 49 | else: 50 | root = '/Users/Erimus/OneDrive/yourRoot' # MAC 51 | ``` 52 | 53 | **subFolder** 仅更新root下指定目录 54 | ```python 55 | subFolder = '' #仅更新root下指定目录 56 | subFolder = 'notebook' #无需指定的话直接注释本行 57 | ``` 58 | 59 | **maxAge** 可以设置浏览器缓存时间了 60 | ```python 61 | maxAge = 0 # header的缓存过期时间 0为不设置 62 | ``` 63 | 64 | ## 忽略文件/文件夹 65 | **ignoreFiles / ignoreFolders** 这两个函数可以自己去设定。 66 | 目前默认排除了git相关文件，还有exe、py、psd等等。具体看一下代码就明白了。 67 | 默认排除项目见顶部的两个 **DEFAULT_IGNORE** 。 68 | ```python 69 | ignoreFiless = [] # 忽略的文件结尾字符(扩展名) 70 | ignoreFolders = [] # 需要忽略的文件夹 71 | ``` 72 | 上述两个列表，接受字符串，也可以直接传入自定义规则的 function。 73 | 示例： 74 | ```python 75 | def my_rule(fn): 76 | if os.path.getsize(fn) > 10000000: # 忽略大文件 77 | return True 78 | ignoreFiless = ['exe', 'py', my_rule] 79 | ``` 80 | 81 | **ignoreFiles** 82 | 判断的对象是含完整路径的文件名，如 `C:/path/file.ext`。 83 | 传入字符串时，默认识别结尾字符串(扩展名)，符合的忽略（不同步）。 84 | 传入函数时，以文件名为参数。返回 `True` 时，忽略该文件（不同步）。 85 | 86 | **ignoreFolders** 87 | 判断的对象是相对根目录的**完整路径**，如`/path`。 88 | 传入字符串时，只要上述路径中的某一级等于该字符串，就忽略（不同步）。 89 | 传入函数时，以上述路径为参数。返回 `True` 时，忽略该目录（不同步）。 90 | 91 | 92 | 93 | # 工作流程 94 | * 扫描指定的本地目录，搜集基于root的相对地址及文件名，还有修改时间。并搜集空文件夹信息。 95 | * 排除符合ignore规则的文件。(git、exe、py、psd、ai等) 96 | * 读取整个bucket(中指定的目录)，搜集含相对地址的文件名和修改时间。并搜集空文件夹信息。 97 | * 比较本地和COS上文件的修改时间，仅上传较新的文件。 98 | * 删除本地不存在(或ignore)，但COS上存在的文件。 99 | * 比较空文件夹，同步。逻辑同文件。(暂无效) 100 | 新版本SDK会自动删除空文件夹。所以本地空文件夹，不会在cos上出现。 101 | 102 | 103 | 104 | # 使用示例 105 | ## 配置文件 106 | `my_bucket1.py` 用来统一配置 bucket 的链接密钥和本地对应目录等。 107 | 因为每次初始化会连接，建议不同 bucket 分不同文件。 108 | ```python 109 | from qCloud_COS_Sync import COS 110 | 111 | my_cos_config = { 112 | 'appid': 888888, 113 | 'secret_id': '', 114 | 'secret_key': '', 115 | 'region_info': 'ap-shanghai', 116 | 'bucket_name': '', 117 | 'maxAge': 0 118 | } 119 | 120 | ignore = {'ignoreFiles': [], 121 | 'ignoreFolders': ['python']} 122 | 123 | root = _root + '/OneDrive/site' # PC 124 | # subFolder = 'notebook' # 无需指定的话 直接注释本行 125 | MY_BUCKET1 = COS(**my_cos_config, root=root, **ignore) # 初始化 126 | ``` 127 | 128 | ## 使用场景 129 | `my_func.py` 直接调用上述配置好的 bucket 连接。 130 | 还可以中途修改过滤规则，目录，默认maxAge，等等。 131 | ```python 132 | from cos.my_bucket1 import MY_BUCKET1 133 | 134 | here = os.path.abspath(os.path.dirname(__file__)) 135 | sub_folder = 'test_folder' 136 | file_full_path = os.path.join(here, sub_folder, 'test_file.html') 137 | local_file = file_full_path[len(here):] 138 | 139 | # 创建本地测试文件 140 | if not os.path.exists(new:=os.path.join(here, sub_folder)): 141 | os.mkdir(new) 142 | if not os.path.exists(file_full_path): 143 | with open(file_full_path, 'w', encoding='utf-8') as f: 144 | f.write('test file') 145 | 146 | 147 | def title(x): print('\n' + x.center(30, '=')) 148 | 149 | 150 | title('修改配置') 151 | MY_BUCKET1.config({'root': here}) 152 | [print(f'{k}: {v}') for k, v in MY_BUCKET1.cfg.items()] 153 | 154 | title('上传') 155 | MY_BUCKET1.upload(local_file) 156 | cos_files = MY_BUCKET1.read(sub_folder) 157 | [print(i) for i in cos_files] 158 | 159 | title('删除') 160 | MY_BUCKET1.delete(local_file) 161 | cos_files = MY_BUCKET1.read(sub_folder) 162 | [print(i) for i in cos_files] 163 | 164 | title('同步') 165 | MY_BUCKET1.sync(sub_folder) 166 | cos_files = MY_BUCKET1.read(sub_folder) 167 | [print(i) for i in cos_files] 168 | 169 | title('删除') 170 | MY_BUCKET1.delete(local_file) 171 | cos_files = MY_BUCKET1.read(sub_folder) 172 | [print(i) for i in cos_files] 173 | 174 | ``` 175 | 176 | 177 | `腾讯云 COS 同步 网站 静态 Python SDK` 178 | 179 | --- 180 | 181 | # Update log 182 | 183 | - 2020-03-08 184 | - 改为 Class 的形式。（原函数形式移入 archive） 185 | 这样配置文件比较容易分离调用，在有多个bucket或服务器时也会比较方便，且易于区分。 186 | 187 | - 2019-11-19 188 | - 阻止 sdk 自带的 logging，它的 info 级信息过多。 189 | - 缩减打印信息。 190 | - 扩展忽略规则，接受自定义函数。 191 | -------------------------------------------------------------------------------- /archive/qCloud_COS_Sync_v4.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python2.7 2 | #-*- coding: utf-8 -*- 3 | __author__ = 'Erimus' 4 | import os,sys,time,re,json,urllib 5 | from datetime import datetime,timedelta 6 | from qcloud_cos import CosClient,UploadFileRequest,StatFileRequest,ListFolderRequest,DelFileRequest,DelFolderRequest,CreateFolderRequest 7 | 8 | """ 9 | 这是一个腾讯云COS的同步工具（仅上传更新部分）。可指定更新个别目录。 10 | 同步源以本地文件为准，镜像到COS，会删除COS上多余的文件。我是用来把本地生成的静态页面同步到COS的。 11 | 12 | 【重要】使用前务必做好备份！做好备份！做好备份！这是一个美工写的工具，请谨慎使用。 13 | 14 | 准备工作，安装SDK。 15 | pip install qcloud_cos_v4 16 | 17 | 设定本地特定目录为root，对应bucket根目录。还可以指定仅更新某个目录。 18 | 会比较本地和COS上文件的修改时间，仅上传较新的文件。 19 | 如果某个文件，本地没有，但COS对应路径下有，会自动删COS上的文件。 20 | 会忽略部分文件，具体搜ignoreFiles部分。 21 | """ 22 | 23 | # Draw title with Frame 24 | def drawTitle(string): 25 | width = (len(string)+len(string.decode('utf-8')))/2 26 | hr = '─'*(width+10) 27 | print '\n┌%s┐\n│ >>> %s <<< │\n└%s┘'%(hr,string,hr) 28 | # Print format JSON 29 | def formatJSON(obj,indent=4): 30 | return json.dumps(obj,encoding="UTF-8", ensure_ascii=False,indent=indent) 31 | 32 | 33 | 34 | 35 | 36 | def readLocalFiles(root,subFolder=u''): 37 | start = datetime.now() 38 | drawTitle('Reading local files') 39 | 40 | #斜杠转换(windows) & 去头尾斜杠 41 | root=re.sub(r'\\',u'/',root).rstrip('\\/') 42 | subFolder=re.sub(r'\\',u'/',subFolder).lstrip('\\/').rstrip('\\/') 43 | 44 | localFilesDict,localEmptyFolders = {},[] 45 | ignoreFoldersNum=ignoreFilesNum=0 #ignore计数 46 | for path, dirs, files in os.walk(root+'/'+subFolder): 47 | if ignoreFolder(path[len(root):]): 48 | ignoreFoldersNum += 1 49 | continue 50 | 51 | total = len(files) 52 | for i,fileName in enumerate(files[:]): 53 | if ignoreFile(fileName): 54 | ignoreFilesNum += 1 55 | continue 56 | 57 | localFile = (path+'/'+fileName)[len(root):] 58 | localFile = re.sub(r'\\',u'/',localFile) #斜杠转换 59 | localFile = re.sub(r'//',u'/',localFile) #根目录重复斜杠 60 | # print '%s'%localFile 61 | modifyTime = int(os.stat(path+'/'+fileName).st_mtime) 62 | localFilesDict[localFile] = modifyTime #输出结果 {'正斜杠相对地址文件名':'int文件修改时间'} 63 | # print '%s / %s'%(i+1,total) 64 | 65 | if not os.listdir(path): 66 | emptyFolder = path[len(root):] 67 | emptyFolder = re.sub(r'\\',u'/',emptyFolder) + '/' #斜杠转换 68 | # print emptyFolder 69 | localEmptyFolders.append(emptyFolder) 70 | 71 | print 'Local Files: %s\nLocal Empty Folders: %s'%(len(localFilesDict),len(localEmptyFolders)) 72 | print '\n-Ignored folders: %s\n-Ignored files: %s'%(ignoreFoldersNum,ignoreFilesNum) 73 | # print 'localFilesDict'+formatJSON(localFilesDict.keys()[:20]) 74 | if localEmptyFolders: 75 | print '---\nlocalEmptyFolders'+formatJSON(localEmptyFolders) 76 | print '---\n%s: %s'%('Used',datetime.now()-start) 77 | return localFilesDict,localEmptyFolders 78 | 79 | 80 | 81 | def ignoreFile(fileName): 82 | ignore = False 83 | 84 | if fileName[0]=='.': #file start with '.' 85 | ignore = True 86 | 87 | try: 88 | extension = re.findall(r'(?<=.)(?!.*\.).*',fileName)[0].lower() 89 | except: 90 | extension = '' 91 | # print extension 92 | ignoreExts = ['exe','py','psd','ai'] #ignore extension list 93 | for ext in ignoreExts: 94 | if extension==ext: 95 | ignore = True 96 | 97 | return ignore 98 | 99 | 100 | 101 | def ignoreFolder(path): 102 | path=re.sub(r'\\',u'/',path) 103 | 104 | ignore = False 105 | 106 | if '.git' in path: 107 | ignore = True 108 | 109 | return ignore 110 | 111 | 112 | 113 | def readCosFiles(cos_client,bucket,subFolder=''): 114 | start = datetime.now() 115 | drawTitle('Reading COS files') 116 | 117 | cosFilesDict,cosEmptyFolders = {},[] 118 | if subFolder: 119 | folderPool = ['/' + re.sub(r'\\',u'/',subFolder).lstrip('\\/').rstrip('\\/') + '/'] 120 | else: 121 | folderPool = [u'/'] 122 | while len(folderPool)>0: 123 | folder = folderPool.pop() 124 | 125 | cosFilesList,listover,context = [],False,'' #每次请求有199限制 神tm199为毛不是200 126 | while listover==False: 127 | succes = times = 0 128 | while succes==0 and times<10: 129 | times += 1 130 | request = ListFolderRequest(bucket, folder) 131 | if context: 132 | request.set_context(context) #文档里虽然给了参数 但怎么用你猜（文档不会告诉你用set的哦） 133 | try: 134 | list_folder_ret = cos_client.list_folder(request) 135 | # print formatJSON(list_folder_ret) 136 | if list_folder_ret['message']==u'SUCCESS': 137 | succes = 1 138 | cosFilesList += list_folder_ret['data']['infos'] 139 | listover = list_folder_ret['data']['listover'] 140 | context = list_folder_ret['data']['context'] 141 | except: 142 | print 'ListFolderRequest Failed.\nFolder: %s\nContext:%s'%(folder,context) 143 | if times==10: 144 | print '===Error===: %s info load failed'%(folder) 145 | continue 146 | 147 | for item in cosFilesList: 148 | if item['mtime']==0: #folder with files 149 | folderPool.append(folder+item['name']) 150 | # print folder+item['name'] 151 | 152 | if ('filesize' not in item) and (item['mtime']!=0): #empty folder 153 | cosEmptyFolders.append(folder+item['name']) 154 | 155 | if 'filesize' in item: #file 156 | cosFile = re.sub(r'^.*?qcloud.com','',item['source_url']) #取含路径的文件名 157 | cosFile = urllib.unquote(cosFile.encode('utf-8')).decode('utf-8') #url中文转码 158 | # print cosFile,type(cosFile) 159 | cosFilesDict[cosFile] = item['mtime'] 160 | 161 | print 'COS files: %s\nCOS empty folders: %s'%(len(cosFilesDict),len(cosEmptyFolders)) 162 | if cosEmptyFolders: 163 | print '---\ncosEmptyFolders'+formatJSON(cosEmptyFolders) 164 | print '---\n%s: %s'%('Used',datetime.now()-start) 165 | return cosFilesDict,cosEmptyFolders 166 | 167 | 168 | 169 | # def getCosFileModifyTime(cos_client,cosFile): #StatFileRequest instead by readCosFiles 170 | # # try 10 times, if fail then print error. 171 | # succes = times = mtime = 0 172 | # while succes==0 and times<10: 173 | # times += 1 174 | # try: 175 | # request = StatFileRequest(bucket, cosFile) 176 | # stat_file_ret = cos_client.stat_file(request) 177 | # if stat_file_ret['message']==u'SUCCESS': 178 | # mtime = stat_file_ret['data']['mtime'] 179 | # succes = 1 180 | # except: 181 | # print 'Error\n%-30s: %s'%(cosFile,stat_file_ret['message']) 182 | # if times==10: 183 | # print '===Error===: %s get state failed'%(cosFile) 184 | # return mtime 185 | 186 | 187 | 188 | def filterModifiedLocalFiles(localFilesDict,cosFilesDict): 189 | drawTitle('Filtering modified local files') 190 | 191 | modifiedLocalFiles=[] 192 | for i,file in enumerate(localFilesDict): 193 | # print 'read cos file state: %s / %s'%(i+1,len(localFilesDict)) 194 | # cosFileModifyTime = getCosFileModifyTime(file) #old method 195 | try: 196 | cosFileModifyTime = cosFilesDict[file] 197 | except: 198 | cosFileModifyTime = 0 199 | if localFilesDict[file]>cosFileModifyTime: 200 | modifiedLocalFiles.append(file) 201 | 202 | if len(modifiedLocalFiles)==0: 203 | print 'All files on COS are the newest.\nNo files need to be uploaded.' 204 | else: 205 | print 'Modified Files: %s'%len(modifiedLocalFiles) 206 | 207 | return modifiedLocalFiles 208 | 209 | 210 | 211 | def uploadToCos(cos_client,bucket,root,localFile): 212 | # try 10 times, if fail then print error. 213 | succes = times = 0 214 | request = UploadFileRequest(bucket, localFile, root+localFile) 215 | request.set_insert_only(0) # 0是允许覆盖 1是不允许 216 | while succes==0 and times<10: 217 | times += 1 218 | try: 219 | upload_file_ret = cos_client.upload_file(request) 220 | if upload_file_ret['message']==u'SUCCESS': 221 | succes = 1 222 | print 'Upload | %-10s | %s'%(upload_file_ret['message'],localFile) 223 | except: 224 | pass 225 | if times==10: 226 | print '===Error===: %s upload failed'%(localFile) 227 | 228 | 229 | 230 | def filterExtraCosFiles(localFilesDict,cosFilesDict): 231 | drawTitle('Filtering extra files on COS') 232 | 233 | extraCosFiles = [] 234 | for file in cosFilesDict: 235 | if file not in localFilesDict: 236 | extraCosFiles.append(file) 237 | # print extraCosFiles 238 | if len(extraCosFiles)==0: 239 | print 'No files need to be deleted.' 240 | else: 241 | print 'Extra Files: %s'%len(extraCosFiles) 242 | 243 | return extraCosFiles 244 | 245 | 246 | 247 | def deleteCosFile(cos_client,bucket,cosFile): 248 | succes = times = 0 249 | while succes==0 and times<10: 250 | times += 1 251 | try: 252 | del_ret = cos_client.del_file(DelFileRequest(bucket, cosFile)) 253 | # print cosFile+formatJSON(del_ret) 254 | if del_ret['message']==u'SUCCESS': 255 | succes = 1 256 | print 'Delete | %-10s | %s'%(del_ret['message'],cosFile) 257 | except: 258 | pass 259 | if times==10: 260 | print '===Error===: %s delete failed'%(cosFile) 261 | 262 | 263 | 264 | def deleteCosFolder(cos_client,bucket,folder): 265 | try: 266 | request = DelFolderRequest(bucket, folder) 267 | delete_folder_ret = cos_client.del_folder(request) 268 | print 'Delete | %-10s | %s'%(delete_folder_ret['message'],folder) 269 | except: 270 | print 'Error: deleteCosFolder | %s'%folder 271 | 272 | 273 | 274 | def createCosFolder(cos_client,bucket,folder): 275 | try: 276 | request = CreateFolderRequest(bucket, folder) 277 | create_folder_ret = cos_client.create_folder(request) 278 | print 'Create | %-10s | %s'%(create_folder_ret['message'],folder) 279 | except: 280 | print 'Error: createCosFolder | %s'%folder 281 | 282 | 283 | 284 | def syncEmptyFolders(cos_client,bucket,localEmptyFolders,cosEmptyFolders): 285 | start = datetime.now() 286 | drawTitle('Sync empty folders') 287 | 288 | createFolderNum=0 289 | for folder in localEmptyFolders: 290 | if folder not in cosEmptyFolders: 291 | createCosFolder(cos_client,bucket,folder) 292 | createFolderNum += 1 293 | else: 294 | cosEmptyFolders.remove(folder) 295 | print 'Create folder(s): %s'%(createFolderNum) 296 | 297 | for folder in cosEmptyFolders: 298 | deleteCosFolder(cos_client,bucket,folder) 299 | print 'Delete folder(s): %s'%len(cosEmptyFolders) 300 | 301 | print '---\n%s: %s'%('Used',datetime.now()-start) 302 | 303 | 304 | 305 | def syncLocalToCOS(appid,secret_id,secret_key,bucket,region_info,root,subFolder=''): 306 | cos_client = CosClient(appid, secret_id, secret_key, region=region_info) 307 | # check cos_client & bucket 308 | list_folder_ret = cos_client.list_folder(ListFolderRequest(bucket, u'/')) 309 | if list_folder_ret['message']!='SUCCESS': 310 | raise Exception('\n'+drawTitle('Check your appid / secret_id / secret_key / bucket_name')) 311 | 312 | localFilesDict,localEmptyFolders = readLocalFiles(root,subFolder) #读取本地需要更新的目录 313 | cosFilesDict,cosEmptyFolders = readCosFiles(cos_client,bucket,subFolder) #读取cos上需要更新的目录 314 | # # print 'localFilesDict'+formatJSON(localFilesDict) 315 | # # print 'cosFilesDict'+formatJSON(cosFilesDict) 316 | 317 | # 比较文件修改时间，筛选出需要更新的文件，并且上传。 318 | modifiedLocalFiles = filterModifiedLocalFiles(localFilesDict,cosFilesDict) 319 | if modifiedLocalFiles: 320 | start = datetime.now() 321 | drawTitle('Uploading files') 322 | for file in modifiedLocalFiles: 323 | uploadToCos(cos_client,bucket,root,file) 324 | print '---\n%s: %s'%('Used',datetime.now()-start) 325 | 326 | # 筛选出cos上有，但本地已经不存在的文件，删除COS上的文件。 327 | extraCosFiles = filterExtraCosFiles(localFilesDict,cosFilesDict) 328 | if extraCosFiles: 329 | start = datetime.now() 330 | drawTitle('Deleting COS files') 331 | for file in extraCosFiles: 332 | deleteCosFile(cos_client,bucket,file) 333 | print '---\n%s: %s'%('Used',datetime.now()-start) 334 | 335 | # 同步空文件夹 336 | syncEmptyFolders(cos_client,bucket,localEmptyFolders,cosEmptyFolders) 337 | 338 | 339 | 340 | 341 | 342 | if __name__ == '__main__': 343 | # 初始化客户端。在【云key密钥/项目密钥】找到appid和配套的key，填写下面4个值。 344 | appid = 88888888 345 | secret_id = u'YOUR_ID' 346 | secret_key = u'YOUR_KEY' 347 | region_info = 'sh' #'sh'=华东,'gz'=华南,'tj'=华北 ,('sgp'~=新加坡) 348 | 349 | 350 | # 填写同步目录。COS端的bucket，本地的root，可以指定root下的单个目录(optional)。 351 | bucket = u'YOUR_BUCKET_NAME' #上述appid对应项目下的bucket name 352 | # 本机根目录 对应bucket根目录 353 | if os.name == 'nt': 354 | root = u'D:\\OneDrive\\site' #PC 355 | else: 356 | root = u'/Users/Erimus/OneDrive/site' #MAC 357 | subFolder = u'' #仅更新root下指定目录 358 | # subFolder = u'bilibili' #无需指定的话直接注释本行 359 | 360 | # Main Progress 361 | syncLocalToCOS(appid,secret_id,secret_key,bucket,region_info,root,subFolder) 362 | -------------------------------------------------------------------------------- /archive/qCloud_COS_Sync_v4_py3.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | __author__ = 'Erimus' 4 | import os 5 | import re 6 | import json 7 | import urllib 8 | import time 9 | from datetime import datetime 10 | from qcloud_cos import * 11 | 12 | """ 13 | 这是一个腾讯云COS的同步工具（仅上传更新部分）。可指定更新个别目录。 14 | 同步源以本地文件为准，镜像到COS，会删除COS上多余的文件。我是用来把本地生成的静态页面同步到COS的。 15 | 16 | 【重要】使用前务必做好备份！做好备份！做好备份！这是一个美工写的工具，请谨慎使用。 17 | 18 | 准备工作，安装以下SDK（已修改支持py3）。 19 | https://github.com/a270443177/cos-python3-sdk-v4 20 | 21 | 设定本地特定目录为root，对应bucket根目录。还可以指定仅更新某个目录。 22 | 会比较本地和COS上文件的修改时间，仅上传较新的文件。 23 | 如果某个文件，本地没有，但COS对应路径下有，会自动删COS上的文件。 24 | 会忽略部分文件，具体搜ignoreFiles部分。 25 | """ 26 | 27 | 28 | # Draw title with Frame 29 | def drawTitle(string): 30 | width = (len(string) + len(string.encode('utf-8'))) // 2 31 | hr = '=' * (width + 8) 32 | print('\n%s\n>>> %s <<<\n%s' % (hr, string, hr)) 33 | 34 | 35 | # Print format JSON 36 | def formatJSON(obj, indent=4): 37 | return json.dumps(obj, ensure_ascii=False, indent=indent) 38 | 39 | 40 | def formatPath(path): 41 | path = path.replace('\\', '/') 42 | while '//' in path: 43 | path = path.replace('//', '/') 44 | return path.rstrip('/') 45 | 46 | 47 | # ==================== 48 | 49 | 50 | def readLocalFiles(root, subFolder=''): 51 | start = datetime.now() 52 | drawTitle('Reading local files') 53 | 54 | localFilesDict, localEmptyFolders = {}, [] 55 | ignoreFoldersNum = ignoreFilesNum = 0 # ignore计数 56 | for path, dirs, files in os.walk(os.path.join(root, subFolder)): 57 | if isIgnoreFolder(path[len(root):]): # 忽略目录 58 | ignoreFoldersNum += 1 59 | continue 60 | 61 | # total = len(files) 62 | for i, fileName in enumerate(files[:]): 63 | if isIgnoreFile(fileName): # 忽略文件 64 | ignoreFilesNum += 1 65 | continue 66 | 67 | localFile = formatPath(os.path.join(path, fileName))[len(root):] 68 | # print('local: %s'%localFile) 69 | modifyTime = int(os.stat(os.path.join(path, fileName)).st_mtime) 70 | # 输出结果 {'正斜杠相对地址文件名':'int文件修改时间'} 71 | localFilesDict[localFile] = modifyTime 72 | # print('%s / %s'%(i+1,total)) 73 | 74 | if not os.listdir(path): # 标注空文件夹 75 | emptyFolder = formatPath(path)[len(root):] 76 | # print(emptyFolder) 77 | localEmptyFolders.append(emptyFolder) 78 | 79 | # 打印详情 80 | print('Local Files: %s\nLocal Empty Folders: %s' 81 | % (len(localFilesDict), len(localEmptyFolders))) 82 | print('\n-Ignored folders (& inner files): %s\n-Ignored files: %s' 83 | % (ignoreFoldersNum, ignoreFilesNum)) 84 | # print('localFilesDict'+formatJSON(localFilesDict.keys()[:20])) 85 | 86 | if localEmptyFolders: 87 | print('\n---\nlocalEmptyFolders: ' + formatJSON(localEmptyFolders)) 88 | print('\n---\n%s: %s\n' % ('Used', datetime.now() - start)) 89 | 90 | return localFilesDict, localEmptyFolders 91 | 92 | 93 | def isIgnoreFolder(path): # 忽略文件夹 94 | # path = formatPath(path) 95 | for k in ['.git', '.svn', '__pycache__', 'miniprogram']: 96 | if k in path: 97 | return True 98 | 99 | 100 | def isIgnoreFile(fileName): # 忽略文件 101 | # file start with '.' 隐藏文件 102 | if fileName[0] == '.': 103 | return True 104 | 105 | ignoreExts = ['exe', 'py', 'pyc', 'psd', 'ai', 'xlsx'] # ignore extension list 106 | extension = fileName.split('.')[-1].lower() # 获取扩展名 107 | if extension in ignoreExts: 108 | return True 109 | 110 | # 个人网页专用忽略项 111 | if extension in ['less']: 112 | return True 113 | if fileName.endswith('.html'): 114 | if fileName == 'index.html': 115 | return 116 | if fileName.endswith('.min.html'): 117 | return 118 | return True # 过滤非首页且没有Minify过的网页源文件 119 | 120 | 121 | # ==================== 122 | 123 | 124 | def readCosFiles(cos_client, bucket, subFolder=''): 125 | start = datetime.now() 126 | drawTitle('Reading COS files') 127 | 128 | cosFilesDict, cosEmptyFolders = {}, [] 129 | folderPool = ['/' + subFolder + '/'] if subFolder else ['/'] 130 | while len(folderPool) > 0: 131 | folder = folderPool.pop() 132 | 133 | cosFilesList, listover, context = [], False, '' 134 | while not listover: 135 | succes = times = 0 136 | while not succes and times < 10: 137 | times += 1 138 | request = ListFolderRequest(bucket, folder) 139 | # 设置请求头(下一页) 140 | if context: 141 | request.set_context(context) 142 | try: 143 | # 每次请求有199限制 144 | list_folder_ret = cos_client.list_folder(request) 145 | # print(formatJSON(list_folder_ret)) 146 | if list_folder_ret['message'] == 'SUCCESS': 147 | succes = 1 148 | cosFilesList += list_folder_ret['data']['infos'] 149 | listover = list_folder_ret['data']['listover'] 150 | context = list_folder_ret['data']['context'] 151 | except Exception: 152 | print('ListFolderRequest Failed.\nFolder: %s\nContext:%s' 153 | % (folder, context)) 154 | if times == 10: 155 | print('===Error===: %s info load failed' % (folder)) 156 | continue 157 | 158 | for item in cosFilesList: 159 | if item['mtime'] == 0: # folder with files 160 | folderPool.append(folder + item['name']) 161 | # print(folder+item['name']) 162 | 163 | if ('filesize' not in item) and (item['mtime'] != 0): # empty folder 164 | cosEmptyFolders.append(folder + item['name']) 165 | 166 | if 'filesize' in item: # file 167 | # 取含路径的文件名 168 | cosFile = re.sub(r'^.*?qcloud.com', '', item['source_url']) 169 | cosFile = urllib.parse.unquote(cosFile) # url编码中文解码 170 | # print(cosFile,type(cosFile)) 171 | cosFilesDict[cosFile] = item['mtime'] 172 | 173 | print('COS files: %s\nCOS empty folders: %s' 174 | % (len(cosFilesDict), len(cosEmptyFolders))) 175 | if cosEmptyFolders: 176 | print('---\ncosEmptyFolders' + formatJSON(cosEmptyFolders)) 177 | print('---\n%s: %s' % ('Used', datetime.now() - start)) 178 | 179 | return cosFilesDict, cosEmptyFolders 180 | 181 | 182 | # ==================== 183 | 184 | 185 | def filterModifiedLocalFiles(localFilesDict, cosFilesDict): 186 | drawTitle('Filtering modified local files') 187 | 188 | modifiedLocalFiles = [] 189 | for i, file in enumerate(localFilesDict): 190 | cosFileModifyTime = cosFilesDict.get(file, 0) # cos上没有的文件 191 | if localFilesDict[file] > cosFileModifyTime: 192 | modifiedLocalFiles.append(file) 193 | 194 | if modifiedLocalFiles: 195 | print('Modified Files: %s' % len(modifiedLocalFiles)) 196 | else: 197 | print('All files on COS are the newest.\nNo files need to be uploaded.') 198 | 199 | return modifiedLocalFiles 200 | 201 | 202 | def uploadToCos(cos_client, bucket, root, localFile): 203 | # try 10 times, if fail then print(error.) 204 | succes = times = 0 205 | request = UploadFileRequest(bucket, localFile, root + localFile) 206 | request.set_insert_only(0) # 0是允许覆盖 1是不允许 207 | while not succes and times < 10: 208 | times += 1 209 | try: 210 | upload_file_ret = cos_client.upload_file(request) 211 | if upload_file_ret['message'] == 'SUCCESS': 212 | succes = 1 213 | print('Upload | %-10s | %s' 214 | % (upload_file_ret['message'], localFile)) 215 | except Exception: 216 | pass 217 | if times == 10: 218 | print('===Error===: %s upload failed' % (localFile)) 219 | 220 | 221 | # ==================== 222 | 223 | 224 | def filterExtraCosFiles(localFilesDict, cosFilesDict): 225 | drawTitle('Filtering extra files on COS') 226 | 227 | extraCosFiles = [] 228 | for file in cosFilesDict: 229 | if file not in localFilesDict: 230 | extraCosFiles.append(file) 231 | # print(extraCosFiles) 232 | if extraCosFiles: 233 | print('Extra Files: %s' % len(extraCosFiles)) 234 | else: 235 | print('No files need to be deleted.') 236 | 237 | return extraCosFiles 238 | 239 | 240 | def deleteCosFile(cos_client, bucket, cosFile): 241 | succes = times = 0 242 | while succes == 0 and times < 10: 243 | times += 1 244 | try: 245 | del_ret = cos_client.del_file(DelFileRequest(bucket, cosFile)) 246 | # print(cosFile+formatJSON(del_ret)) 247 | if del_ret['message'] == 'SUCCESS': 248 | succes = 1 249 | print('Delete | %-10s | %s' % (del_ret['message'], cosFile)) 250 | except Exception: 251 | pass 252 | if times == 10: 253 | print('===Error===: %s delete failed' % (cosFile)) 254 | 255 | 256 | # ==================== 257 | 258 | 259 | def deleteCosFolder(cos_client, bucket, folder): 260 | try: 261 | request = DelFolderRequest(bucket, folder) 262 | delete_folder_ret = cos_client.del_folder(request) 263 | print('Delete | %-10s | %s' % (delete_folder_ret['message'], folder)) 264 | except Exception: 265 | print('Error: deleteCosFolder | %s' % folder) 266 | 267 | 268 | def createCosFolder(cos_client, bucket, folder): 269 | if folder[-1] != '/': 270 | folder += '/' 271 | try: 272 | request = CreateFolderRequest(bucket, folder) 273 | create_folder_ret = cos_client.create_folder(request) 274 | print('Create | %-10s | %s' % (create_folder_ret['message'], folder)) 275 | except Exception: 276 | print('Error: createCosFolder | %s' % folder) 277 | 278 | 279 | def syncEmptyFolders(cos_client, bucket, 280 | localEmptyFolders, cosEmptyFolders): 281 | start = datetime.now() 282 | drawTitle('Sync empty folders') 283 | 284 | if not localEmptyFolders + cosEmptyFolders: 285 | print('No empty folder.') 286 | 287 | createFolderNum = 0 288 | for folder in localEmptyFolders: 289 | if folder not in cosEmptyFolders: 290 | createCosFolder(cos_client, bucket, folder) 291 | createFolderNum += 1 292 | else: 293 | cosEmptyFolders.remove(folder) 294 | if createFolderNum: 295 | print('Create folder(s): %s' % (createFolderNum)) 296 | 297 | for folder in cosEmptyFolders: 298 | deleteCosFolder(cos_client, bucket, folder) 299 | if cosEmptyFolders: 300 | print('Delete folder(s): %s' % len(cosEmptyFolders)) 301 | 302 | print('---\n%s: %s' % ('Used', datetime.now() - start)) 303 | 304 | 305 | # ==================== 306 | 307 | 308 | def syncLocalToCOS(appid, secret_id, secret_key, bucket, region_info, 309 | root, subFolder='', debug=0): 310 | root = formatPath(root) 311 | subFolder = formatPath(subFolder) 312 | cos_client = CosClient(appid, secret_id, secret_key, region=region_info) 313 | 314 | # check cos_client & bucket 315 | success = 0 # 偶尔会失联 隔一段时间自动重试 316 | while not success: 317 | list_folder_ret = cos_client.list_folder(ListFolderRequest(bucket, '/')) 318 | if list_folder_ret.get('message') == 'SUCCESS': 319 | success = 1 320 | else: 321 | if debug: 322 | raise Exception('>>> Check your appid / secret_id / ' 323 | 'secret_key / bucket_name <<<') 324 | else: 325 | print('>>>%s<<<\nConnection maybe has some problem, ' 326 | 'or id / password / bucket wrong.' % datetime.now()) 327 | time.sleep(30) 328 | 329 | # 读取本地需要更新的目录 330 | localFilesDict, localEmptyFolders = readLocalFiles(root, subFolder) 331 | 332 | # 读取cos上需要更新的目录 333 | cosFilesDict, cosEmptyFolders = readCosFiles(cos_client, bucket, subFolder) 334 | 335 | # 比较文件修改时间，筛选出需要更新的文件，并且上传。 336 | modifiedLocalFiles = filterModifiedLocalFiles(localFilesDict, cosFilesDict) 337 | if modifiedLocalFiles: 338 | start = datetime.now() 339 | if debug: 340 | drawTitle('Uploading files') 341 | for file in modifiedLocalFiles: 342 | uploadToCos(cos_client, bucket, root, file) 343 | if debug: 344 | print('---\n%s: %s' % ('Used', datetime.now() - start)) 345 | 346 | # 筛选出cos上有，但本地已经不存在的文件，删除COS上的文件。 347 | extraCosFiles = filterExtraCosFiles(localFilesDict, cosFilesDict) 348 | if extraCosFiles: 349 | start = datetime.now() 350 | if debug: 351 | drawTitle('Deleting COS files') 352 | for file in extraCosFiles: 353 | deleteCosFile(cos_client, bucket, file) 354 | if debug: 355 | print('---\n%s: %s' % ('Used', datetime.now() - start)) 356 | 357 | # 同步空文件夹 358 | syncEmptyFolders(cos_client, bucket, localEmptyFolders, cosEmptyFolders) 359 | 360 | 361 | # ==================== 362 | 363 | 364 | if __name__ == '__main__': 365 | # 初始化客户端。在【云key密钥/项目密钥】找到appid和配套的key，填写下面4个值。 366 | appid = 88888888 367 | secret_id = 'YOUR_ID' 368 | secret_key = 'YOUR_KEY' 369 | region_info = 'sh' # 'sh'=华东,'gz'=华南,'tj'=华北 ,('sgp'~=新加坡) 370 | 371 | # 填写同步目录。COS端的bucket，本地的root，可以指定root下的单个目录(可选)。 372 | bucket = 'YOUR_BUCKET_NAME' # 上述appid对应项目下的bucket name 373 | # 本机根目录 对应bucket根目录 374 | if os.name == 'nt': 375 | root = 'D:\\OneDrive\\site' # PC 376 | else: 377 | root = '/Users/Erimus/OneDrive/site' # MAC 378 | subFolder = '' # 仅更新root下指定目录 379 | # subFolder = 'bilibili' #无需指定的话直接注释本行 380 | 381 | # Main Progress 382 | syncLocalToCOS(appid, secret_id, secret_key, bucket, 383 | region_info, root, subFolder, debug=1) 384 | -------------------------------------------------------------------------------- /archive/qCloud_COS_Sync_v5_function.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | __author__ = 'Erimus' 4 | """ 5 | 这是一个腾讯云COS的同步工具（仅上传更新部分）。可指定更新个别目录。 6 | 同步源以本地文件为准，镜像到COS，会删除COS上多余的文件。我是用来把本地生成的静态页面同步到COS的。 7 | 8 | 【重要】使用前务必做好备份！做好备份！做好备份！这是一个美工写的工具，请谨慎使用。 9 | 10 | 准备工作，安装以下SDK。 11 | pip install -U cos-python-sdk-v5 12 | 13 | 设定本地特定目录为root，对应bucket根目录。还可以指定仅更新某个目录。 14 | 会比较本地和COS上文件的修改时间，仅上传较新的文件。 15 | 如果某个文件，本地没有，但COS对应路径下有，会自动删COS上的文件。 16 | 会忽略部分文件，具体搜ignoreFiles部分。 17 | """ 18 | 19 | import os 20 | import json 21 | import time 22 | import logging 23 | from datetime import datetime 24 | from qcloud_cos import * 25 | 26 | logging.getLogger(__name__) # 阻止SDK打印的log 27 | # ==================== 28 | DEFAULT_IGNORE_FOLDERS = ['.git', '.svn', '__pycache__'] 29 | DEFAULT_IGNORE_FILES = ['exe', 'py', 'pyc', 'psd', 'psb', 'ai', 'xlsx'] 30 | # ==================== 31 | 32 | 33 | # Draw title with Frame 34 | def drawTitle(string): 35 | print('-' * 20 + f'\n>>> {string} >>>') 36 | 37 | 38 | # Print format JSON 39 | def formatJSON(obj, indent=4): 40 | return json.dumps(obj, ensure_ascii=False, indent=indent) 41 | 42 | 43 | def formatPath(path): 44 | return path.replace('\\', '/').strip('/') 45 | 46 | 47 | def ts2uft(ts): 48 | return datetime.utcfromtimestamp(ts).strftime('%Y-%m-%dT%H:%M:%S.000Z') 49 | 50 | 51 | # ==================== 52 | 53 | 54 | def readLocalFiles(root, subFolder='', ignoreFiles=[], ignoreFolders=[]): 55 | start = datetime.now() 56 | drawTitle('Read local files') 57 | 58 | localFilesDict, localEmptyFolders = {}, [] 59 | ignoreFoldersNum = ignoreFilesNum = 0 # ignore计数 60 | for path, dirs, files in os.walk(os.path.join(root, subFolder)): 61 | # 忽略部分目录 不上传 62 | if isIgnoreFolder(path[len(root):], ignoreFolders): 63 | ignoreFoldersNum += 1 64 | continue 65 | 66 | # 忽略部分文件 不上传 67 | for i, fileName in enumerate(files[:]): 68 | if isIgnoreFile(os.path.join(path, fileName), ignoreFiles): 69 | ignoreFilesNum += 1 70 | continue 71 | 72 | localFile = formatPath(os.path.join(path, fileName)[len(root):]) 73 | # print('local: %s'%localFile) 74 | modifyTime = int(os.stat(os.path.join(path, fileName)).st_mtime) 75 | modifyTime = ts2uft(modifyTime) 76 | # 输出结果 {'/相对地址文件名':'文件修改时间int'} 77 | localFilesDict[localFile] = modifyTime 78 | 79 | if not os.listdir(path): # 标注空文件夹 80 | emptyFolder = formatPath(path[len(root):]) 81 | # print(emptyFolder) 82 | localEmptyFolders.append(emptyFolder) 83 | 84 | # 打印详情 85 | print(f'Local Files: {len(localFilesDict)} / ' 86 | f'Empty Folders: {len(localEmptyFolders)}') 87 | # if localEmptyFolders: 88 | # print('localEmptyFolders: ' + formatJSON(localEmptyFolders)) 89 | print(f'Ignored folders: {ignoreFoldersNum} / ' 90 | f'Files: {ignoreFilesNum}') 91 | # print(f'localFilesDict: \n{formatJSON(list(localFilesDict.items())[:20])}') 92 | 93 | print(f'Used: {datetime.now() - start}') 94 | return localFilesDict, localEmptyFolders 95 | 96 | 97 | def isIgnoreFolder(path, ignoreFolders=[]): # 忽略文件夹 98 | path = formatPath(path) 99 | for k in DEFAULT_IGNORE_FOLDERS + ignoreFolders: 100 | if isinstance(k, str) and k in path.split('/'): 101 | return True 102 | if callable(k) and k(path): 103 | return True 104 | 105 | 106 | def isIgnoreFile(fileName, ignoreFiles=[]): # 忽略文件 107 | # file start with '.' 隐藏文件 108 | if fileName[0] == '.': 109 | return True 110 | 111 | for ext in DEFAULT_IGNORE_FILES + ignoreFiles: 112 | if isinstance(ext, str) and fileName.lower().endswith(ext): 113 | return True 114 | if callable(ext) and ext(fileName): # function 115 | return True 116 | 117 | 118 | # ==================== 119 | 120 | 121 | def readCosFiles(cos_client, bucket, subFolder=''): 122 | start = datetime.now() 123 | drawTitle('Read COS files') 124 | cosFilesDict, cosEmptyFolders = {}, [] 125 | end, marker = 0, '' 126 | while not end: 127 | times = 0 128 | while times < 100: 129 | times += 1 130 | try: 131 | # 每次请求有1000限制 132 | res = cos_client.list_objects( 133 | Bucket=bucket, 134 | Prefix=subFolder, 135 | Marker=marker, # 设置请求头(下一页) 136 | # Delimiter="/", # 目录分隔符 不设置可自动获取全部文件 137 | # MaxKeys=100, # 默认单次返回1000条 138 | ) 139 | break # 访问成功则退出 140 | except Exception: 141 | print(f'ListFolderRequest Failed. [{times}]\nMarder: {marker}') 142 | time.sleep(3) 143 | if times == 100: 144 | print(f'===Error===: {folder} info load failed') 145 | res = {} 146 | 147 | if 'NextMarker' in res: 148 | marker = res['NextMarker'] 149 | else: 150 | end = 1 151 | 152 | # print(formatJSON(res)) 153 | 154 | # 获取文件 155 | for k in res.get('Contents', []): 156 | fn = k['Key'] 157 | if fn.endswith('/'): # 空文件夹 158 | cosEmptyFolders.append(fn) 159 | else: # 文件 160 | mt = k['LastModified'] 161 | if '.000Z' not in mt: 162 | print('Time format changed!!!', mt) 163 | raise 164 | cosFilesDict[fn] = mt 165 | 166 | print(f'COS files: {len(cosFilesDict)} / ' 167 | f'Empty folders: {len(cosEmptyFolders)}') 168 | if cosEmptyFolders: 169 | print('---\ncosEmptyFolders: ' + formatJSON(cosEmptyFolders)) 170 | # print(f'cosFilesDict: \n{formatJSON(list(cosFilesDict.items())[:20])}') 171 | 172 | print(f'Used: {datetime.now() - start}') 173 | return cosFilesDict, cosEmptyFolders 174 | 175 | 176 | # ==================== 177 | 178 | 179 | def filterModifiedLocalFiles(localFilesDict, cosFilesDict): 180 | drawTitle('Filter modified local files') 181 | 182 | modifiedLocalFiles = [] 183 | for i, file in enumerate(localFilesDict): 184 | cosFileModifyTime = cosFilesDict.get(file, '') # cos上没有的文件 185 | if localFilesDict[file] > cosFileModifyTime: 186 | # print(f'{file}\n' 187 | # f'LOC Modify Time: {localFilesDict[file]}\n' 188 | # f'COS Modify Time: {cosFileModifyTime}') 189 | modifiedLocalFiles.append(file) 190 | 191 | if modifiedLocalFiles: 192 | print(f'Modified Files: {len(modifiedLocalFiles)}') 193 | else: 194 | print('All files on COS are the latest version.') 195 | 196 | # print(f'modifiedLocalFiles: \n{formatJSON(modifiedLocalFiles[:20])}') 197 | return modifiedLocalFiles 198 | 199 | 200 | def uploadToCos(cos_client, bucket, root, localFile, maxAge=0): 201 | times = 0 202 | while times < 10: 203 | times += 1 204 | try: 205 | cos_client.put_object_from_local_file( 206 | Bucket=bucket, 207 | LocalFilePath=os.path.join(root, localFile), 208 | Key=localFile, 209 | CacheControl=f'max-age={maxAge}' if maxAge else '' 210 | ) 211 | print(f'Upload | {localFile}') 212 | break 213 | except CosServiceError as e: 214 | print(repr(e)) 215 | pass 216 | if times == 10: 217 | print(f'Error: Upload failed! | {localFile}') 218 | 219 | 220 | # ==================== 221 | 222 | 223 | def filterExtraCosFiles(localFilesDict, cosFilesDict): 224 | drawTitle('Filter extra files on COS') 225 | 226 | extraCosFiles = [] 227 | for file in cosFilesDict: 228 | if file not in localFilesDict: 229 | extraCosFiles.append(file) 230 | # print(extraCosFiles) 231 | if extraCosFiles: 232 | print(f'Extra Files: {len(extraCosFiles)}') 233 | else: 234 | print('No files need to be deleted.') 235 | 236 | return extraCosFiles 237 | 238 | 239 | def deleteCosFiles(cos_client, bucket, cosFiles): 240 | while cosFiles: 241 | once, cosFiles = cosFiles[:500], cosFiles[500:] # 号称最高1000 242 | delObjects = {'Object': [{'Key': i} for i in once], 'Quiet': 'false'} 243 | times = 0 244 | while times < 10: 245 | times += 1 246 | try: 247 | res = cos_client.delete_objects(Bucket=bucket, Delete=delObjects) 248 | if 'Error' in res: 249 | print(res['Error']) 250 | raise 251 | print(f'Delete | {len(once)} files') 252 | print(formatJSON(once)) 253 | break 254 | except Exception: 255 | pass 256 | if times == 10: 257 | print(f'Error: delete failed!\n{formatJSON(once)}') 258 | return 259 | 260 | 261 | # ==================== 262 | 263 | 264 | def deleteCosFolder(cos_client, bucket, folder): 265 | try: 266 | cos_client.delete_object(Bucket=bucket, Key=folder) 267 | print(f'Delete | {folder}') 268 | except Exception: 269 | print(f'Error: deleteCosFolder | {folder}') 270 | 271 | 272 | def createCosFolder(cos_client, bucket, folder): # 新版本中好像无法建空文件夹 273 | if folder[-1] != '/': 274 | folder += '/' 275 | try: 276 | request = CreateFolderRequest(bucket, folder) 277 | create_folder_ret = cos_client.create_folder(request) 278 | print(f'Create | {create_folder_ret["message"]} | {folder}') 279 | except Exception: 280 | print(f'Error: createCosFolder | {folder}') 281 | 282 | 283 | def syncEmptyFolders(cos_client, bucket, 284 | localEmptyFolders, cosEmptyFolders): 285 | start = datetime.now() 286 | 287 | if localEmptyFolders + cosEmptyFolders: 288 | drawTitle('Sync empty folders') 289 | else: 290 | return 291 | 292 | createFolderNum = 0 293 | for folder in localEmptyFolders: 294 | if folder not in cosEmptyFolders: 295 | createCosFolder(cos_client, bucket, folder) 296 | createFolderNum += 1 297 | else: 298 | cosEmptyFolders.remove(folder) 299 | if createFolderNum: 300 | print('Create folder(s): %s' % (createFolderNum)) 301 | 302 | for folder in cosEmptyFolders: 303 | deleteCosFolder(cos_client, bucket, folder) 304 | if cosEmptyFolders: 305 | print('Delete folder(s): %s' % len(cosEmptyFolders)) 306 | 307 | print(f'Used: {datetime.now() - start}') 308 | 309 | 310 | # ==================== 311 | 312 | 313 | def syncLocalToCOS(appid, secret_id, secret_key, bucket_name, region_info, 314 | root, subFolder, ignoreFiles, ignoreFolders, maxAge): 315 | _start = datetime.now() 316 | print(f'Sync [{subFolder if subFolder else os.path.dirname(root)}] to COS') 317 | 318 | subFolder = subFolder.strip('/') 319 | bucket = f'{bucket_name}-{appid}' 320 | 321 | # check cos_client & bucket 322 | while True: # 偶尔会失联 隔一段时间自动重试 323 | cos_config = CosConfig(SecretId=secret_id, 324 | SecretKey=secret_key, 325 | Region=region_info) 326 | cos_client = CosS3Client(cos_config) 327 | try: 328 | cos_client.head_bucket(Bucket=bucket) 329 | break 330 | except Exception as e: 331 | print(repr(e)) 332 | print('Check your appid / secret_id / secret_key / bucket_name\n' 333 | 'Retry after 30 seconds.') 334 | time.sleep(30) 335 | 336 | # 读取本地需要更新的目录 337 | localFilesDict, localEmptyFolders = readLocalFiles(root, subFolder, 338 | ignoreFiles, 339 | ignoreFolders) 340 | 341 | # 读取cos上需要更新的目录 342 | cosFilesDict, cosEmptyFolders = readCosFiles(cos_client, bucket, subFolder) 343 | 344 | # 比较文件修改时间，筛选出需要更新的文件，并且上传。 345 | modifiedLocalFiles = filterModifiedLocalFiles(localFilesDict, cosFilesDict) 346 | if modifiedLocalFiles: 347 | start = datetime.now() 348 | drawTitle('Uploading files') 349 | for file in modifiedLocalFiles: 350 | uploadToCos(cos_client, bucket, root, file, maxAge=maxAge) 351 | print(f'Used: {datetime.now() - start}') 352 | 353 | # 筛选出cos上有，但本地已经不存在的文件，删除COS上的文件。 354 | extraCosFiles = filterExtraCosFiles(localFilesDict, cosFilesDict) 355 | if extraCosFiles: 356 | start = datetime.now() 357 | drawTitle('Deleting COS files') 358 | deleteCosFiles(cos_client, bucket, extraCosFiles) 359 | print(f'Used: {datetime.now() - start}') 360 | 361 | # 同步空文件夹（这个功能暂停） 362 | # 这个版本的SDK在删除文件后，会自动删除空文件夹，所以COS上不会存在空文件夹。 363 | # syncEmptyFolders(cos_client, bucket, localEmptyFolders, cosEmptyFolders) 364 | 365 | print('-' * 20 + f'\nTotal used: {datetime.now() - _start}') 366 | 367 | 368 | # ==================== 369 | 370 | 371 | if __name__ == '__main__': 372 | # 初始化客户端。在【密钥管理】找到appid和配套的key，填写下面3个值。 373 | appid = 88888888 # your appid 374 | secret_id = 'your_id' 375 | secret_key = 'your_key' 376 | 377 | # 地区列表，可以看自己的bucket下的域名管理/静态地址/'cos.'后面那段。 378 | region_info = 'ap-shanghai' 379 | 380 | # 填写同步目录。 381 | # COS端的bucket，对应本地的root。 382 | bucket_name = 'your_bucket_name' # 上述appid对应项目下的bucket name 383 | 384 | # 本机root根目录 385 | if os.name == 'nt': 386 | root = 'D:\\OneDrive\\yourRoot' # PC 387 | else: 388 | root = '/Users/Erimus/OneDrive/yourRoot' # MAC 389 | 390 | subFolder = '' # 仅更新root下指定目录（可选） 391 | # subFolder = 'bilibili' # 无需指定的话 直接注释本行 392 | 393 | # 忽略以下内容，不进行上传。(请参考顶部两组 default ignore) 394 | ignoreFiles = [] # 忽略的文件结尾字符(扩展名) 395 | ignoreFolders = [] # 需要忽略的文件夹 396 | ''' 397 | 上述两个列表，接受字符串，也可以直接传入自定义规则的 function。 398 | ignoreFiles。默认识别结尾字符串(扩展名)，或以含路径文件名作为参数的函数。 399 | ignoreFolders。识别完整匹配文件夹名的字符，或以一个文件夹名为参数的函数。 400 | 示例： 401 | def my_rule(fn): 402 | if os.path.getsize(fn) > 10000000: # 忽略大文件 403 | return True 404 | ignoreFiles = ['exe', 'py', my_rule] 405 | ''' 406 | 407 | maxAge = 0 # header的缓存过期时间 0为不设置 408 | 409 | # Main Progress 410 | syncLocalToCOS(appid, secret_id, secret_key, bucket_name, region_info, 411 | root, subFolder, ignoreFiles, ignoreFolders, maxAge) 412 | -------------------------------------------------------------------------------- /qCloud_COS_Sync.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | __author__ = 'Erimus' 4 | """ 5 | 这是一个腾讯云COS的同步工具（仅上传更新部分）。可指定更新个别目录。 6 | 同步源以本地文件为准，镜像到COS，会删除COS上多余的文件。我是用来把本地生成的静态页面同步到COS的。 7 | 8 | 【重要】使用前务必做好备份！做好备份！做好备份！这是一个美工写的工具，请谨慎使用。 9 | 10 | 准备工作，安装以下SDK。 11 | pip install -U cos-python-sdk-v5 12 | 13 | 设定本地特定目录为root，对应bucket根目录。还可以指定仅更新某个目录。 14 | 会比较本地和COS上文件的修改时间，仅上传较新的文件。 15 | 如果某个文件，本地没有，但COS对应路径下有，会自动删COS上的文件。 16 | 会忽略部分文件，具体搜ignoreFiles部分。 17 | """ 18 | 19 | import os 20 | import json 21 | import time 22 | import logging 23 | from datetime import datetime 24 | from qcloud_cos import * 25 | 26 | logging.getLogger(__name__) # 阻止SDK打印的log 27 | 28 | # ==================== 默认不同步的文件/文件夹 29 | 30 | 31 | DEFAULT_IGNORE_FILES = [ 32 | 'exe', 'py', 'pyc', 'psd', 'psb', 'ai', 'xlsx', 33 | lambda x:x.split('/')[-1][0] == '.', # 隐藏文件 34 | ] 35 | DEFAULT_IGNORE_FOLDERS = [ 36 | '__pycache__', 37 | lambda x:any(fd[0] == '.' for fd in x.split('/')), # 隐藏目录 38 | ] 39 | 40 | 41 | # ==================== 工具 42 | 43 | 44 | # Draw title with Frame 45 | def drawTitle(string): 46 | print('-' * 20 + f'\n>>> {string} >>>') 47 | 48 | 49 | # Print format JSON 50 | def formatJSON(obj, indent=4): 51 | return json.dumps(obj, ensure_ascii=False, indent=indent) 52 | 53 | 54 | def formatPath(path): 55 | return path.replace('\\', '/').strip('/') 56 | 57 | 58 | def ts2uft(ts): 59 | return datetime.utcfromtimestamp(ts).strftime('%Y-%m-%dT%H:%M:%S.000Z') 60 | 61 | 62 | # ==================== 读取本地文件 63 | 64 | 65 | def readLocalFiles(root, subFolder='', ignoreFiles=None, ignoreFolders=None): 66 | start = datetime.now() 67 | drawTitle('Read local files') 68 | 69 | ignoreFiles = [] if ignoreFiles is None else ignoreFiles 70 | ignoreFolders = [] if ignoreFolders is None else ignoreFolders 71 | 72 | localFilesDict, localEmptyFolders = {}, [] 73 | ignoreFoldersNum = ignoreFilesNum = 0 # ignore计数 74 | for path, dirs, files in os.walk(os.path.join(root, subFolder), 75 | followlinks=True): 76 | # 忽略部分目录 不上传 77 | if isIgnoreFolder(path[len(root):], ignoreFolders): 78 | ignoreFoldersNum += 1 79 | continue 80 | 81 | # 忽略部分文件 不上传 82 | for i, fileName in enumerate(files[:]): 83 | this_file = os.path.join(path, fileName) 84 | if isIgnoreFile(this_file, ignoreFiles): 85 | ignoreFilesNum += 1 86 | continue 87 | 88 | if os.path.islink(this_file): 89 | print(f'This is link: {this_file}') 90 | continue 91 | 92 | localFile = formatPath(this_file[len(root):]) 93 | # print('local: %s'%localFile) 94 | modifyTime = int(os.stat(this_file).st_mtime) 95 | modifyTime = ts2uft(modifyTime) 96 | # 输出结果 {'/相对地址文件名':'文件修改时间int'} 97 | localFilesDict[localFile] = modifyTime 98 | 99 | if not os.listdir(path): # 标注空文件夹 100 | emptyFolder = formatPath(path[len(root):]) 101 | # print(emptyFolder) 102 | localEmptyFolders.append(emptyFolder) 103 | 104 | # 打印详情 105 | print(f'Local Files: {len(localFilesDict)} / ' 106 | f'Empty Folders: {len(localEmptyFolders)}') 107 | # if localEmptyFolders: 108 | # print('localEmptyFolders: ' + formatJSON(localEmptyFolders)) 109 | print(f'Ignored folders: {ignoreFoldersNum} / ' 110 | f'Files: {ignoreFilesNum}') 111 | # print(f'localFilesDict: \n{formatJSON(list(localFilesDict.items())[:20])}') 112 | 113 | print(f'Used: {datetime.now() - start}') 114 | return localFilesDict, localEmptyFolders 115 | 116 | 117 | def isIgnoreFile(file, ignoreFiles): # 忽略文件 118 | file = formatPath(file) 119 | for ext in ignoreFiles: 120 | if isinstance(ext, str) and file.lower().endswith(ext): 121 | return True 122 | if callable(ext) and ext(file): # 传入为函数 且结果为真 123 | return True 124 | 125 | 126 | def isIgnoreFolder(path, ignoreFolders): # 忽略文件夹 127 | path = formatPath(path) 128 | for k in ignoreFolders: 129 | if isinstance(k, str) and k in path.split('/'): 130 | return True 131 | if callable(k) and k(path): # 传入为函数 且结果为真 132 | return True 133 | 134 | 135 | # ==================== 文件比较 136 | 137 | 138 | def filterModifiedLocalFiles(localFilesDict, cosFilesDict): 139 | drawTitle('Filter modified local files') 140 | 141 | modifiedLocalFiles = [] 142 | for i, file in enumerate(localFilesDict): 143 | cosFileModifyTime = cosFilesDict.get(file, '') # cos上没有的文件 144 | if localFilesDict[file] > cosFileModifyTime: 145 | # print(f'{file}\n' 146 | # f'LOC Modify Time: {localFilesDict[file]}\n' 147 | # f'COS Modify Time: {cosFileModifyTime}') 148 | modifiedLocalFiles.append(file) 149 | 150 | if modifiedLocalFiles: 151 | print(f'Modified Files: {len(modifiedLocalFiles)}') 152 | else: 153 | print('All files on COS are the latest version.') 154 | 155 | # print(f'modifiedLocalFiles: \n{formatJSON(modifiedLocalFiles[:20])}') 156 | return modifiedLocalFiles 157 | 158 | 159 | def filterExtraCosFiles(localFilesDict, cosFilesDict): 160 | drawTitle('Filter extra files on COS') 161 | 162 | extraCosFiles = [] 163 | for file in cosFilesDict: 164 | if file not in localFilesDict: 165 | extraCosFiles.append(file) 166 | # print(extraCosFiles) 167 | if extraCosFiles: 168 | print(f'Extra Files: {len(extraCosFiles)}') 169 | else: 170 | print('No files need to be deleted.') 171 | 172 | return extraCosFiles 173 | 174 | 175 | # ==================== COS读取操作 176 | 177 | 178 | def readCosFiles(cos_client, bucket, subFolder=''): 179 | start = datetime.now() 180 | drawTitle('Read COS files') 181 | cosFilesDict, cosEmptyFolders = {}, [] 182 | end, marker = 0, '' 183 | while not end: 184 | times = 0 185 | while times < 100: 186 | times += 1 187 | try: 188 | # 每次请求有1000限制 189 | res = cos_client.list_objects( 190 | Bucket=bucket, 191 | Prefix=subFolder, 192 | Marker=marker, # 设置请求头(下一页) 193 | # Delimiter="/", # 目录分隔符 不设置可自动获取全部文件 194 | # MaxKeys=100, # 默认单次返回1000条 195 | ) 196 | break # 访问成功则退出 197 | except Exception: 198 | print(f'ListFolderRequest Failed. [{times}]\nMarder: {marker}') 199 | time.sleep(3) 200 | if times == 100: 201 | print(f'===Error===: {folder} info load failed') 202 | res = {} 203 | 204 | if 'NextMarker' in res: 205 | marker = res['NextMarker'] 206 | else: 207 | end = 1 208 | 209 | # print(formatJSON(res)) 210 | 211 | # 获取文件 212 | for k in res.get('Contents', []): 213 | fn = k['Key'] 214 | if fn.endswith('/'): # 空文件夹 215 | cosEmptyFolders.append(fn) 216 | else: # 文件 217 | mt = k['LastModified'] 218 | if '.000Z' not in mt: 219 | print('Time format changed!!!', mt) 220 | raise 221 | cosFilesDict[fn] = mt 222 | 223 | print(f'COS files: {len(cosFilesDict)} / ' 224 | f'Empty folders: {len(cosEmptyFolders)}') 225 | if cosEmptyFolders: 226 | print('---\ncosEmptyFolders: ' + formatJSON(cosEmptyFolders)) 227 | # print(f'cosFilesDict: \n{formatJSON(list(cosFilesDict.items())[:20])}') 228 | 229 | print(f'Used: {datetime.now() - start}') 230 | # return cosFilesDict, cosEmptyFolders # 这个版本不含空文件夹 231 | return cosFilesDict 232 | 233 | 234 | # ==================== 主程序 235 | 236 | 237 | class COS(): 238 | def __init__(self, *, 239 | appid, secret_id, secret_key, bucket_name, region_info, 240 | retry_limit=10, # 重试次数 偶尔会连接不上 241 | root, ignoreFiles=None, ignoreFolders=None, maxAge=0 242 | ): 243 | # 初始化 cos_client 244 | self.bucket = f'{bucket_name}-{appid}' 245 | while True: # 偶尔会失联 隔一段时间自动重试 246 | cos_config = CosConfig(SecretId=secret_id, 247 | SecretKey=secret_key, 248 | Region=region_info) 249 | self.cos_client = CosS3Client(cos_config) 250 | try: 251 | self.cos_client.head_bucket(Bucket=self.bucket) 252 | break 253 | except Exception as e: 254 | print(repr(e), 255 | '\nCheck your appid / secret_id / secret_key / bucket_name' 256 | '\nRetry after 30 seconds.') 257 | time.sleep(30) 258 | 259 | if ignoreFiles is None: 260 | ignoreFiles = DEFAULT_IGNORE_FILES 261 | if ignoreFolders is None: 262 | ignoreFolders = DEFAULT_IGNORE_FOLDERS 263 | 264 | # 初始化常量 265 | self.cfg = { 266 | 'retry_limit': retry_limit, 267 | 'root': root, 268 | 'ignoreFiles': ignoreFiles, 269 | 'ignoreFolders': ignoreFolders, 270 | 'maxAge': maxAge, 271 | } 272 | 273 | def config(self, dicts): # 传入字典修改配置。例{'root':'C:/test'} 274 | assert isinstance(dicts, dict) 275 | for k, v in dicts.items(): 276 | if k in self.cfg: 277 | self.cfg[k] = v 278 | else: 279 | raise KeyError(f'[{k}] not in cfg:{self.cfg.keys()}') 280 | 281 | def upload(self, localFile, maxAge=None): 282 | # localFile 是相对根目录的完整路径。 283 | localFile = formatPath(localFile) 284 | maxAge = self.cfg['maxAge'] if maxAge is None else maxAge 285 | 286 | times = 0 287 | while times < self.cfg['retry_limit']: 288 | times += 1 289 | try: 290 | self.cos_client.put_object_from_local_file( 291 | Bucket=self.bucket, 292 | LocalFilePath=os.path.join(self.cfg['root'], localFile), 293 | Key=localFile, 294 | CacheControl=f'max-age={maxAge}' if maxAge else '' 295 | ) 296 | print(f'Upload | {localFile}') 297 | break 298 | except CosServiceError as e: 299 | print(repr(e)) 300 | pass 301 | else: 302 | print(f'Error: Upload failed! | {localFile}') 303 | 304 | def delete(self, cosFiles): 305 | # cosFiles应该是文件的相对于根目录的路径。 306 | # 如果传入单个文件，自动转化为list。 307 | if isinstance(cosFiles, str): 308 | cosFiles = [cosFiles] 309 | assert isinstance(cosFiles, list) 310 | cosFiles = [formatPath(i) for i in cosFiles] 311 | 312 | while cosFiles: # 单次删除的数量有限，需要分批处理。 313 | once, cosFiles = cosFiles[:500], cosFiles[500:] # 号称最高1000 314 | delObjects = {'Object': [{'Key': i} for i in once], 'Quiet': 'false'} 315 | times = 0 316 | while times < self.cfg['retry_limit']: 317 | times += 1 318 | try: 319 | res = self.cos_client.delete_objects(Bucket=self.bucket, 320 | Delete=delObjects) 321 | if 'Error' in res: 322 | print(res['Error']) 323 | raise 324 | print(f'Delete | {len(once)} files') 325 | print(formatJSON(once)) 326 | break 327 | except Exception as e: 328 | print(f'Delete tried: {times} times\n{repr(e)}') 329 | pass 330 | else: 331 | print(f'Error: delete failed!\n{formatJSON(once)}') 332 | 333 | def read(self, subFolder): 334 | return readCosFiles(self.cos_client, self.bucket, subFolder) 335 | 336 | def sync(self, subFolder='', *, maxAge=None): 337 | maxAge = self.cfg['maxAge'] if maxAge is None else maxAge 338 | _start = datetime.now() 339 | _folder = subFolder or self.cfg['root'].strip('/').split('/')[-1] 340 | print(f'\n{"="*20}\nSYNC [{_folder}] TO COS') 341 | 342 | subFolder = subFolder.strip('/') 343 | 344 | # 读取本地需要更新的目录 345 | localFilesDict, localEmptyFolders = readLocalFiles( 346 | self.cfg['root'], subFolder, 347 | self.cfg['ignoreFiles'], self.cfg['ignoreFolders']) 348 | 349 | # 读取cos上需要更新的目录 350 | cosFilesDict = self.read(subFolder) 351 | 352 | # 比较文件修改时间，筛选出需要更新的文件，并且上传。 353 | modifiedLocalFiles = filterModifiedLocalFiles(localFilesDict, cosFilesDict) 354 | if modifiedLocalFiles: 355 | start = datetime.now() 356 | drawTitle('Uploading files') 357 | for file in modifiedLocalFiles: 358 | self.upload(file, maxAge=maxAge) 359 | print(f'Used: {datetime.now() - start}') 360 | 361 | # 筛选出cos上有，但本地已经不存在的文件，删除COS上的文件。 362 | extraCosFiles = filterExtraCosFiles(localFilesDict, cosFilesDict) 363 | if extraCosFiles: 364 | start = datetime.now() 365 | drawTitle('Deleting COS files') 366 | self.delete(extraCosFiles) 367 | print(f'Used: {datetime.now() - start}') 368 | 369 | # 同步空文件夹（这个功能暂停） 370 | # 这个版本的SDK在删除文件后，会自动删除空文件夹，所以COS上不会存在空文件夹。 371 | 372 | print('-' * 20 + f'\nTotal used: {datetime.now() - _start}') 373 | 374 | 375 | # ==================== 376 | 377 | 378 | if __name__ == '__main__': 379 | 380 | # ========== 服务器端初始化参数 ========== 381 | # appid/secret_id/secret_key 详见COS/密钥管理。 382 | # region_info 地区列表，在bucket下的域名管理/静态地址/'cos.'后面那段。 383 | # bucket_name 是COS端的bucket，对应本地的root。 384 | my_cos_config = { 385 | 'appid': 88888888, 386 | 'secret_id': 'your_id', 387 | 'secret_key': 'your_key', 388 | 'region_info': 'ap-shanghai', 389 | 'bucket_name': 'your_bucket_name' 390 | } 391 | 392 | # ========== 本机需要同步的参数 ========== 393 | # root 本机根目录 394 | if os.name == 'nt': 395 | root = 'D:/OneDrive/yourRoot' # PC 396 | else: 397 | root = '/Users/Erimus/OneDrive/yourRoot' # MAC 398 | 399 | # ========== 可选的参数 ========== 400 | subFolder = '' # 仅更新root下指定目录（可选） 401 | # subFolder = 'notebook' # 示例 402 | 403 | # 不需要进行上传的内容，通过以下规则忽略。(最下有详细说明) 404 | ignore = { 405 | 'ignoreFiles': [], # 忽略的文件结尾字符(扩展名) 406 | 'ignoreFolders': [] # 需要忽略的文件夹 407 | } 408 | 409 | maxAge = 0 # header 可设置文件缓存的过期时间。upload和sync可用该参数。 410 | 411 | # ========== 使用示例 ========== 412 | my_bucket = COS(**my_cos_config, root=root, **ignore) # 初始化 413 | my_bucket.sync(subFolder) # 同步 414 | 415 | # 其它用法 416 | # my_bucket.upload(root, localFile='notebook/index.html', maxAge) # 上传 417 | # my_bucket.delete(localFile='notebook/index.html') # 删除 418 | 419 | # ========== 忽略规则说明 ========== 420 | ''' 421 | 上述两个列表，接受字符串，也可以直接传入自定义规则的 function。 422 | 423 | ignoreFiles 判断的对象是含完整路径的文件名，如'C:/path/file.ext'。 424 | 传入字符串时，默认识别结尾字符串(扩展名)，符合的忽略（不同步）。 425 | 传入函数时，以文件名为参数。返回True时，忽略该文件（不同步）。 426 | 427 | ignoreFolders 判断的对象是相对根目录的完整路径，如'/path'。 428 | 传入字符串时，只要上述路径中的某一级等于该字符串，就忽略（不同步）。 429 | 传入函数时，以上述路径为参数。返回True时，忽略该目录（不同步）。 430 | 431 | 示例： 432 | def my_rule(fn): 433 | if os.path.getsize(fn) > 10000000: # 忽略大文件 434 | return True 435 | ignoreFiles = ['exe', 'py', my_rule] 436 | 437 | 另请参考顶部两组 default ignore 438 | ''' 439 | --------------------------------------------------------------------------------