├── .gitignore ├── LICENSE ├── README.md └── qiniusync.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | *.egg-info/ 23 | .installed.cfg 24 | *.egg 25 | 26 | # PyInstaller 27 | # Usually these files are written by a python script from a template 28 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 29 | *.manifest 30 | *.spec 31 | 32 | # Installer logs 33 | pip-log.txt 34 | pip-delete-this-directory.txt 35 | 36 | # Unit test / coverage reports 37 | htmlcov/ 38 | .tox/ 39 | .coverage 40 | .coverage.* 41 | .cache 42 | nosetests.xml 43 | coverage.xml 44 | *,cover 45 | 46 | # Translations 47 | *.mo 48 | *.pot 49 | 50 | # Django stuff: 51 | *.log 52 | 53 | # Sphinx documentation 54 | docs/_build/ 55 | 56 | # PyBuilder 57 | target/ 58 | 59 | # vim 60 | *.swp 61 | 62 | # Mac crap 63 | .DS_Store 64 | *.DS_Store 65 | 66 | 67 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, and 10 | distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by the copyright 13 | owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all other entities 16 | that control, are controlled by, or are under common control with that entity. 17 | For the purposes of this definition, "control" means (i) the power, direct or 18 | indirect, to cause the direction or management of such entity, whether by 19 | contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the 20 | outstanding shares, or (iii) beneficial ownership of such entity. 21 | 22 | "You" (or "Your") shall mean an individual or Legal Entity exercising 23 | permissions granted by this License. 24 | 25 | "Source" form shall mean the preferred form for making modifications, including 26 | but not limited to software source code, documentation source, and configuration 27 | files. 28 | 29 | "Object" form shall mean any form resulting from mechanical transformation or 30 | translation of a Source form, including but not limited to compiled object code, 31 | generated documentation, and conversions to other media types. 32 | 33 | "Work" shall mean the work of authorship, whether in Source or Object form, made 34 | available under the License, as indicated by a copyright notice that is included 35 | in or attached to the work (an example is provided in the Appendix below). 36 | 37 | "Derivative Works" shall mean any work, whether in Source or Object form, that 38 | is based on (or derived from) the Work and for which the editorial revisions, 39 | annotations, elaborations, or other modifications represent, as a whole, an 40 | original work of authorship. For the purposes of this License, Derivative Works 41 | shall not include works that remain separable from, or merely link (or bind by 42 | name) to the interfaces of, the Work and Derivative Works thereof. 43 | 44 | "Contribution" shall mean any work of authorship, including the original version 45 | of the Work and any modifications or additions to that Work or Derivative Works 46 | thereof, that is intentionally submitted to Licensor for inclusion in the Work 47 | by the copyright owner or by an individual or Legal Entity authorized to submit 48 | on behalf of the copyright owner. For the purposes of this definition, 49 | "submitted" means any form of electronic, verbal, or written communication sent 50 | to the Licensor or its representatives, including but not limited to 51 | communication on electronic mailing lists, source code control systems, and 52 | issue tracking systems that are managed by, or on behalf of, the Licensor for 53 | the purpose of discussing and improving the Work, but excluding communication 54 | that is conspicuously marked or otherwise designated in writing by the copyright 55 | owner as "Not a Contribution." 56 | 57 | "Contributor" shall mean Licensor and any individual or Legal Entity on behalf 58 | of whom a Contribution has been received by Licensor and subsequently 59 | incorporated within the Work. 60 | 61 | 2. Grant of Copyright License. 62 | 63 | Subject to the terms and conditions of this License, each Contributor hereby 64 | grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, 65 | irrevocable copyright license to reproduce, prepare Derivative Works of, 66 | publicly display, publicly perform, sublicense, and distribute the Work and such 67 | Derivative Works in Source or Object form. 68 | 69 | 3. Grant of Patent License. 70 | 71 | Subject to the terms and conditions of this License, each Contributor hereby 72 | grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, 73 | irrevocable (except as stated in this section) patent license to make, have 74 | made, use, offer to sell, sell, import, and otherwise transfer the Work, where 75 | such license applies only to those patent claims licensable by such Contributor 76 | that are necessarily infringed by their Contribution(s) alone or by combination 77 | of their Contribution(s) with the Work to which such Contribution(s) was 78 | submitted. If You institute patent litigation against any entity (including a 79 | cross-claim or counterclaim in a lawsuit) alleging that the Work or a 80 | Contribution incorporated within the Work constitutes direct or contributory 81 | patent infringement, then any patent licenses granted to You under this License 82 | for that Work shall terminate as of the date such litigation is filed. 83 | 84 | 4. Redistribution. 85 | 86 | You may reproduce and distribute copies of the Work or Derivative Works thereof 87 | in any medium, with or without modifications, and in Source or Object form, 88 | provided that You meet the following conditions: 89 | 90 | You must give any other recipients of the Work or Derivative Works a copy of 91 | this License; and 92 | You must cause any modified files to carry prominent notices stating that You 93 | changed the files; and 94 | You must retain, in the Source form of any Derivative Works that You distribute, 95 | all copyright, patent, trademark, and attribution notices from the Source form 96 | of the Work, excluding those notices that do not pertain to any part of the 97 | Derivative Works; and 98 | If the Work includes a "NOTICE" text file as part of its distribution, then any 99 | Derivative Works that You distribute must include a readable copy of the 100 | attribution notices contained within such NOTICE file, excluding those notices 101 | that do not pertain to any part of the Derivative Works, in at least one of the 102 | following places: within a NOTICE text file distributed as part of the 103 | Derivative Works; within the Source form or documentation, if provided along 104 | with the Derivative Works; or, within a display generated by the Derivative 105 | Works, if and wherever such third-party notices normally appear. The contents of 106 | the NOTICE file are for informational purposes only and do not modify the 107 | License. You may add Your own attribution notices within Derivative Works that 108 | You distribute, alongside or as an addendum to the NOTICE text from the Work, 109 | provided that such additional attribution notices cannot be construed as 110 | modifying the License. 111 | You may add Your own copyright statement to Your modifications and may provide 112 | additional or different license terms and conditions for use, reproduction, or 113 | distribution of Your modifications, or for any such Derivative Works as a whole, 114 | provided Your use, reproduction, and distribution of the Work otherwise complies 115 | with the conditions stated in this License. 116 | 117 | 5. Submission of Contributions. 118 | 119 | Unless You explicitly state otherwise, any Contribution intentionally submitted 120 | for inclusion in the Work by You to the Licensor shall be under the terms and 121 | conditions of this License, without any additional terms or conditions. 122 | Notwithstanding the above, nothing herein shall supersede or modify the terms of 123 | any separate license agreement you may have executed with Licensor regarding 124 | such Contributions. 125 | 126 | 6. Trademarks. 127 | 128 | This License does not grant permission to use the trade names, trademarks, 129 | service marks, or product names of the Licensor, except as required for 130 | reasonable and customary use in describing the origin of the Work and 131 | reproducing the content of the NOTICE file. 132 | 133 | 7. Disclaimer of Warranty. 134 | 135 | Unless required by applicable law or agreed to in writing, Licensor provides the 136 | Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, 137 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, 138 | including, without limitation, any warranties or conditions of TITLE, 139 | NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are 140 | solely responsible for determining the appropriateness of using or 141 | redistributing the Work and assume any risks associated with Your exercise of 142 | permissions under this License. 143 | 144 | 8. Limitation of Liability. 145 | 146 | In no event and under no legal theory, whether in tort (including negligence), 147 | contract, or otherwise, unless required by applicable law (such as deliberate 148 | and grossly negligent acts) or agreed to in writing, shall any Contributor be 149 | liable to You for damages, including any direct, indirect, special, incidental, 150 | or consequential damages of any character arising as a result of this License or 151 | out of the use or inability to use the Work (including but not limited to 152 | damages for loss of goodwill, work stoppage, computer failure or malfunction, or 153 | any and all other commercial damages or losses), even if such Contributor has 154 | been advised of the possibility of such damages. 155 | 156 | 9. Accepting Warranty or Additional Liability. 157 | 158 | While redistributing the Work or Derivative Works thereof, You may choose to 159 | offer, and charge a fee for, acceptance of support, warranty, indemnity, or 160 | other liability obligations and/or rights consistent with this License. However, 161 | in accepting such obligations, You may act only on Your own behalf and on Your 162 | sole responsibility, not on behalf of any other Contributor, and only if You 163 | agree to indemnify, defend, and hold each Contributor harmless for any liability 164 | incurred by, or claims asserted against, such Contributor by reason of your 165 | accepting any such warranty or additional liability. 166 | 167 | END OF TERMS AND CONDITIONS 168 | 169 | APPENDIX: How to apply the Apache License to your work 170 | 171 | To apply the Apache License to your work, attach the following boilerplate 172 | notice, with the fields enclosed by brackets "{}" replaced with your own 173 | identifying information. (Don't include the brackets!) The text should be 174 | enclosed in the appropriate comment syntax for the file format. We also 175 | recommend that a file or class name and description of purpose be included on 176 | the same "printed page" as the copyright notice for easier identification within 177 | third-party archives. 178 | 179 | Copyright 2015 hqp 180 | 181 | Licensed under the Apache License, Version 2.0 (the "License"); 182 | you may not use this file except in compliance with the License. 183 | You may obtain a copy of the License at 184 | 185 | http://www.apache.org/licenses/LICENSE-2.0 186 | 187 | Unless required by applicable law or agreed to in writing, software 188 | distributed under the License is distributed on an "AS IS" BASIS, 189 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 190 | See the License for the specific language governing permissions and 191 | limitations under the License. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | #qiniu_sync 2 | 3 | 基于七牛Python SDK写的一个同步脚本。 4 | 支持批量比较文件，差异增量更新、批量更新。 5 | 6 | ## 使用方式 7 | 8 | * 安装七牛Python SDK 9 | `pip install qiniu` 10 | 11 | * 填写脚本文件(qiniusync.py)的配置信息 12 | ``` 13 | access_key = '' 14 | secret_key = '' 15 | bucket_name = '' 16 | bucket_domain = '' 17 | ``` 18 | >注册后可以拿到对应的信息 19 | 20 | * 将脚本文件(qiniusync.py)拷贝到待同步根目录 21 | 22 | * 运行脚本 23 | ``` 24 | python qiniusync.py 25 | python qiniusync.py down 26 | python qiniusync.py down [文件路径前缀] 27 | ``` 28 | -------------------------------------------------------------------------------- /qiniusync.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | #-*- coding:utf-8 -*- 3 | # 4 | # AUTHOR = "heqingpan" 5 | # AUTHOR_EMAIL = "heqingpan@126.com" 6 | # URL = "http://git.oschina.net/hqp/qiniu_sync" 7 | 8 | import qiniu 9 | from qiniu import Auth 10 | from qiniu import BucketManager 11 | import os 12 | import re 13 | import urllib2 14 | import sys 15 | 16 | access_key = '' 17 | secret_key = '' 18 | bucket_name = '' 19 | bucket_domain = '' 20 | 21 | q = Auth(access_key, secret_key) 22 | bucket = BucketManager(q) 23 | basedir=os.path.realpath(os.path.dirname(__file__)) 24 | #同步目录 25 | #basedir="" 26 | filename=__file__ 27 | ignore_paths=[filename,"{0}c".format(filename)] 28 | ignore_names=[".DS_Store",".git",".gitignore","qiniusync.py","README.md","LICENSE"] 29 | charset="utf8" 30 | diff_time=2*60 31 | 32 | 33 | def list_all(bucket_name, bucket=None, prefix="", limit=100): 34 | rlist=[] 35 | if bucket is None: 36 | bucket = BucketManager(q) 37 | marker = None 38 | eof = False 39 | while eof is False: 40 | ret, eof, info = bucket.list(bucket_name, prefix=prefix, marker=marker, limit=limit) 41 | marker = ret.get('marker', None) 42 | for item in ret['items']: 43 | rlist.append(item["key"]) 44 | if eof is not True: 45 | # 错误处理 46 | #print "error" 47 | pass 48 | return rlist 49 | 50 | def get_files(basedir="",fix="",rlist=None,ignore_paths=[],ignore_names=[]): 51 | if rlist is None: 52 | rlist=[] 53 | for subfile in os.listdir(basedir): 54 | temp_path=os.path.join(basedir,subfile) 55 | tp=os.path.join(fix,subfile) 56 | if tp in ignore_names: 57 | continue 58 | if tp in ignore_paths: 59 | continue 60 | if os.path.isfile(temp_path): 61 | rlist.append(tp) 62 | elif os.path.isdir(temp_path): 63 | get_files(temp_path,tp,rlist,ignore_paths,ignore_names) 64 | return rlist 65 | 66 | def get_valid_key_files(subdir=""): 67 | basedir=subdir or basedir 68 | files = get_files(basedir=basedir,ignore_paths=ignore_paths,ignore_names=ignore_names) 69 | return map(lambda f:(f.replace("\\","/"),f),files) 70 | 71 | 72 | def sync(): 73 | qn_keys=list_all(bucket_name,bucket) 74 | qn_set=set(qn_keys) 75 | l_key_files=get_valid_key_files(basedir) 76 | k2f={} 77 | update_keys=[] 78 | u_count=500 79 | u_index=0 80 | for k,f in l_key_files: 81 | k2f[k]=f 82 | str_k=k 83 | if isinstance(k,str): 84 | k=k.decode(charset) 85 | if k in qn_set: 86 | update_keys.append(str_k) 87 | u_index+=1 88 | if u_index > u_count: 89 | u_index-=u_count 90 | update_file(k2f,update_keys) 91 | update_keys=[] 92 | else: 93 | # upload 94 | upload_file(k,os.path.join(basedir,f)) 95 | if update_keys: 96 | update_file(k2f,update_keys) 97 | print "sync end" 98 | 99 | def update_file(k2f,ulist): 100 | ops=qiniu.build_batch_stat(bucket_name,ulist) 101 | rets,infos = bucket.batch(ops) 102 | for i in xrange(len(ulist)): 103 | k=ulist[i] 104 | f=k2f.get(k) 105 | ret=rets[i]["data"] 106 | size=ret.get("fsize",None) 107 | put_time = int(ret.get("putTime")/10000000) 108 | local_size=os.path.getsize(f) 109 | local_time=int(os.path.getatime(f)) 110 | if local_size==size: 111 | continue 112 | if put_time >= local_time - diff_time: 113 | # is new 114 | continue 115 | # update 116 | upload_file(k,os.path.join(basedir,f)) 117 | 118 | def upload_file(key,localfile): 119 | print "upload_file:" 120 | print key 121 | token = q.upload_token(bucket_name, key) 122 | mime_type = get_mime_type(localfile) 123 | params = {'x:a': 'a'} 124 | progress_handler = lambda progress, total: progress 125 | ret, info = qiniu.put_file(token, key, localfile, params, mime_type, progress_handler=progress_handler) 126 | 127 | def get_mime_type(path): 128 | mime_type = "text/plain" 129 | return mime_type 130 | 131 | def down_file(key,basedir="",is_private=1,expires=3600): 132 | if isinstance(key,unicode): 133 | key=key.encode(charset) 134 | url = 'http://%s/%s' % (bucket_domain, key) 135 | if is_private: 136 | url=q.private_download_url(url, expires=expires) 137 | c=urllib2.urlopen(url) 138 | fpath=key.replace("/",os.sep) 139 | savepath=os.path.join(basedir,fpath).decode(charset) 140 | dir_=os.path.dirname(savepath) 141 | if not os.path.isdir(dir_): 142 | os.makedirs(dir_) 143 | elif os.path.isfile(savepath): 144 | os.remove(savepath) 145 | f = file(savepath, 'wb') 146 | f.write(c.read()) 147 | f.close() 148 | 149 | def down_all(prefix=""): 150 | import traceback 151 | for key in list_all(bucket_name,bucket,prefix=prefix): 152 | try: 153 | down_file(key,basedir=basedir) 154 | print "down:\t"+key 155 | except: 156 | print "error down:\t"+key 157 | print traceback.format_exc() 158 | print "down end" 159 | 160 | 161 | def main(): 162 | if len(sys.argv)>1: 163 | if sys.argv[1]=="down": 164 | prefix=len(sys.argv)>2 and sys.argv[2] or "" 165 | down_all(prefix=prefix) 166 | return 167 | sync() 168 | 169 | if __name__=="__main__": 170 | main() 171 | 172 | --------------------------------------------------------------------------------