├── .gitignore ├── COPYRIGHT ├── README.md ├── config ├── __init__.py └── setting.default.py ├── cover.png ├── init.py ├── inotify_thread.py ├── ossync.py ├── ossync ├── __init__.py ├── lib │ ├── __init__.py │ ├── helper.py │ └── queue_model.py ├── sdk │ ├── __init__.py │ ├── oss_api.py │ ├── oss_cmd.py │ ├── oss_fs.py │ ├── oss_sample.py │ ├── oss_util.py │ └── oss_xml_handler.py └── tests │ ├── __init__.py │ ├── api_test.py │ ├── filehash.py │ ├── helper_test.py │ ├── path_test.py │ ├── pyinotify_test.py │ ├── queue_model_test.py │ └── walkdir_test.py ├── queue_thread.py ├── runtest.py ├── setup.py └── sync_thread.py /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.pyc 3 | doc/ 4 | docs/ 5 | db/ 6 | *.db 7 | *.log 8 | downloads/ 9 | uploads/ 10 | backup/ 11 | logs/ 12 | *.bak 13 | setting.py 14 | *.key 15 | -------------------------------------------------------------------------------- /COPYRIGHT: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2012 Wu Tangsheng(lanbaba) 2 | 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | 10 | # The above copyright notice and this permission notice shall be included in 11 | # all copies or substantial portions of the Software. 12 | 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | # THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # OSSync 2 | 3 | OSSync是一款开源的、基于inotify机制的阿里云同步上载工具，采用Python开发。License : [MIT](http://rem.mit-license.org/). 4 | 5 | ## 主要特色 6 | 7 | * **基于inotify机制** - Inotify 是一个 Linux 内核特性，它实时监控文件系统的变化，比如删除、读、写,移动等操作。利用这个机制可以实现文件及时、高效同步到阿里云，避免频繁轮询等低效操作文件方式，减轻系统负载。 8 | * **可以一次同步多个本地文件夹和多个bucket** - 只要定义好本地文件夹和bucket的映射关系，可以同时将多个本地文件夹同步到多个bucket. 9 | * **基于消息队列的多线程快速同步** - 采用消息队列和多线程机制，实现快速同步. 10 | * **安全准确同步** - 文件上传校验和失败重传确保文件完整同步。 11 | 12 | ## 安装 13 | 将本程序解压到任意目录, 并进入该目录，运行： 14 | 15 | sudo python setup.py 16 | 17 | 如果提示：“Installation complete successfully!”，表明安装成功。否则，请检查是否满足以下条件并手动安装pyinotify模块。 18 | 19 | * Python版本大于2.6(建议使用python2.7, 暂不支持python3) 20 | * 检查和系统是否有/proc/sys/fs/inotify/目录，以确定内核是否支持inotify，即linux内核版本号大于2.6.13。 21 | * 安装pyinotify模块，[https://github.com/seb-m/pyinotify](https://github.com/seb-m/pyinotify)。 22 | 23 | 24 | ## 运行 25 | * 请复制config/setting.default.py并命名为setting.py，修改setting.py中的配置，请参考配置文件中的说明文字. 26 | * 在程序根目录下运行: 27 | 28 | nohup python ossync.py >/dev/null 2>&1 & 29 | 30 | 注：请查看logs目录下的日志文件以了解系统运行状况。 31 | 32 | -------------------------------------------------------------------------------- /config/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lanbaba/ossync/7fe9e6ceaac62b0ccba956bc1e09c730cf75ea81/config/__init__.py -------------------------------------------------------------------------------- /config/setting.default.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Copyright (c) 2012 Wu Tangsheng(lanbaba) 4 | 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | 12 | # The above copyright notice and this permission notice shall be included in 13 | # all copies or substantial portions of the Software. 14 | 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | # THE SOFTWARE. 22 | 23 | # OSS 连接参数 24 | #################### 25 | # OSS # 26 | #################### 27 | 28 | HOST = "oss.aliyuncs.com" 29 | ACCESS_ID = "" 30 | SECRET_ACCESS_KEY = "" 31 | 32 | # OSS Bucket和本地目录同步映射关系，一个Bucket对应一个或多个本地目录(local_folder)。 33 | # 可以定义多个bucket。示例： 34 | # oss_mappers = [{'bucket': 'dzdata', 'local_folders': ['/root/testdata/audios', '/root/testdata/docs']}, 35 | # {'bucket': 'privdata', 'local_folders': ['/root/testdata/images', '/root/testdata/pdfs']}] 36 | #################### 37 | # OSS MAP # 38 | #################### 39 | oss_mappers = [{'bucket': 'dzdata', 'local_folders': ['/root/testdata/audios', '/root/testdata/docs']}] 40 | 41 | # 日志选项 42 | #################### 43 | # LOGGING SETTING # 44 | #################### 45 | MAX_LOGFILE_SIZE = 104857600 # 默认日志文件大小为100M，每次达大小限制时，会自动加后缀生成备份文件 46 | MAX_BACKUP_COUNT = 5 # 默认备份文件为5个 47 | 48 | # 上传文件或者删除object的最大重试次数 49 | #################### 50 | # MAX_RETRIES # 51 | #################### 52 | MAX_RETRIES = 10 53 | 54 | # 上传文件线程数 55 | #################### 56 | # MAX_RETRIES # 57 | #################### 58 | NTHREADS = 5 59 | -------------------------------------------------------------------------------- /cover.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lanbaba/ossync/7fe9e6ceaac62b0ccba956bc1e09c730cf75ea81/cover.png -------------------------------------------------------------------------------- /init.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Copyright (c) 2012 Wu Tangsheng(lanbaba) 4 | 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | 12 | # The above copyright notice and this permission notice shall be included in 13 | # all copies or substantial portions of the Software. 14 | 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | # THE SOFTWARE. 22 | 23 | import sys 24 | import os 25 | import os.path 26 | from config.setting import * 27 | from ossync.lib import queue_model 28 | import logging 29 | import logging.handlers 30 | from ossync.sdk.oss_api import * 31 | 32 | def set_sys_to_utf8(): 33 | reload(sys) 34 | sys.setdefaultencoding('utf-8') 35 | 36 | def get_logger(): 37 | LOG_FILENAME = 'logs/app.log' 38 | format = logging.Formatter("%(levelname)-10s %(asctime)s %(message)s") 39 | logging.basicConfig(level = logging.INFO) 40 | logger = logging.getLogger('app') 41 | handler1 = logging.handlers.RotatingFileHandler(LOG_FILENAME, maxBytes = MAX_LOGFILE_SIZE, backupCount = MAX_BACKUP_COUNT) 42 | handler2 = logging.StreamHandler(sys.stdout) 43 | handler1.setFormatter(format) 44 | handler2.setFormatter(format) 45 | logger.addHandler(handler1) 46 | # logger.addHandler(handler2) 47 | return logger 48 | 49 | def check_config(logger): 50 | if len(HOST) == 0 or len(ACCESS_ID) == 0 or len(SECRET_ACCESS_KEY) == 0: 51 | msg = "Please set HOST and ACCESS_ID and SECRET_ACCESS_KEY" 52 | #print msg 53 | logger.critical(msg) 54 | exit(0) 55 | if len(oss_mappers) == 0: 56 | msg = "please set OSS Mappers" 57 | #print msg 58 | logger.critical(msg) 59 | exit(0) 60 | oss = OssAPI(HOST, ACCESS_ID, SECRET_ACCESS_KEY) 61 | for oss_mapper in oss_mappers: 62 | bucket = oss_mapper['bucket'] 63 | acl = '' 64 | headers = {} 65 | res = oss.create_bucket(bucket, acl, headers) 66 | if (res.status / 100) != 2: 67 | msg = "Bucket: " + bucket + " is not existed or create bucket failure, please rename your bucket." 68 | #print msg 69 | logger.critical(msg) 70 | exit(0) 71 | local_folders = oss_mapper['local_folders'] 72 | if len(local_folders) > 0: 73 | for folder in local_folders: 74 | if not os.path.exists(folder) or not os.path.isdir(folder): 75 | msg = "Local folder: " + folder + " is not existed or is not a direcotry.Please check you setting." 76 | #print msg 77 | logger.critical(msg) 78 | exit(0) 79 | else: 80 | msg = "please at least set one local folder for each bucket" 81 | #print msg 82 | logger.critical(msg) 83 | exit(0) 84 | 85 | def queue_unprocessed(queue, logger): 86 | dbpath = 'db/ossync.db' 87 | qm = queue_model.QueueModel(dbpath) 88 | qm.open() 89 | items = qm.find_all(status = 0) 90 | if items: 91 | for item in items: 92 | if int(item['retries']) < MAX_RETRIES: 93 | el = item['bucket'] + '::' + item['root'] + '::' + item['relpath'] + '::' + item['action'] 94 | queue.put(el, block = True, timeout = 1) 95 | msg = 'queue unprocessed element:' + el 96 | logger.info(msg) 97 | qm.close() -------------------------------------------------------------------------------- /inotify_thread.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Copyright (c) 2012 Wu Tangsheng(lanbaba) 4 | 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | 12 | # The above copyright notice and this permission notice shall be included in 13 | # all copies or substantial portions of the Software. 14 | 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | # THE SOFTWARE. 22 | 23 | import os, sys, threading, logging 24 | from Queue import * 25 | from ossync.lib import queue_model 26 | from ossync.lib import helper 27 | try: 28 | import pyinotify 29 | except ImportError as e: 30 | print e.message 31 | exit(0) 32 | 33 | class EventHandler(pyinotify.ProcessEvent): 34 | """事件处理""" 35 | 36 | def __init__(self, monitered_dir, queue, bucket): 37 | self.monitered_dir = monitered_dir 38 | self.queue = queue 39 | self.bucket = bucket 40 | self.logger = logging.getLogger('app') 41 | dbpath = 'db/ossync.db' 42 | self.qm = queue_model.QueueModel(dbpath) 43 | 44 | def process_IN_CREATE(self, event): 45 | self.process_event(event, 'CREATE') 46 | 47 | def process_IN_DELETE(self, event): 48 | self.process_event(event, 'DELETE') 49 | 50 | def process_IN_MODIFY(self, event): 51 | self.process_event(event, 'MODIFY') 52 | 53 | def process_IN_MOVED_FROM(self, event): 54 | self.logger.info("Moved from file: %s " % os.path.join(event.path, event.name)) 55 | self.process_event(event, 'DELETE') 56 | 57 | def process_IN_MOVED_TO(self, event): 58 | self.logger.info("Moved to file: %s " % os.path.join(event.path, event.name)) 59 | realpath = os.path.join(event.path, event.name) 60 | if event.dir: 61 | self.queue_dir(realpath) 62 | self.process_event(event, 'CREATE') 63 | 64 | def process_event(self, event, action): 65 | if len(action) == 0: 66 | return False 67 | realpath = os.path.join(event.path, event.name) 68 | relpath = os.path.relpath(realpath, self.monitered_dir) 69 | if action == 'DELETE': 70 | if event.dir: 71 | relpath += '/' 72 | self.logger.info(action.title() + " file: %s " % realpath) 73 | #print "Modify file: %s " % os.path.join(event.path, event.name) 74 | el = self.bucket + '::' + self.monitered_dir + '::' + relpath + '::' + action[0] 75 | self.save_el(self.monitered_dir, relpath, self.bucket, action[0]) 76 | self.queue_el(el) 77 | 78 | def queue_dir(self, queue_path): 79 | files = list(helper.walk_files(queue_path, yield_folders = True)) 80 | if len(files) > 0: 81 | for path in files: 82 | relpath = os.path.relpath(path, self.monitered_dir) 83 | self.save_el(self.monitered_dir, relpath, self.bucket,'C') 84 | el = self.bucket + '::' + self.monitered_dir + '::' + relpath + '::' + 'C' 85 | self.queue_el(el) 86 | 87 | def save_el(self, root, relpath, bucket, action): 88 | hashcode = helper.calc_el_md5(root, relpath, bucket) 89 | self.qm.open() 90 | if self.is_el_existed(hashcode): 91 | self.qm.update_action(hashcode, action) 92 | self.qm.update_status(hashcode, 0) 93 | else: 94 | data={"root": root, "relpath": relpath, "bucket": bucket, "action": action, "status": 0, "retries": 0} 95 | self.qm.save(data) 96 | self.qm.close() 97 | 98 | def is_el_existed(self, hashcode): 99 | row = self.qm.get(hashcode) 100 | if row: 101 | return True 102 | return False 103 | 104 | 105 | def queue_el(self, el): 106 | '''el: element of queue , formated as "bucket::root::path::C|M|D" 107 | C means CREATE, M means MODIFY, D means DELETE 108 | ''' 109 | try: 110 | self.queue.put(el, block = True, timeout = 1) 111 | msg = 'queue element:' + el 112 | #print msg 113 | self.logger.info(msg) 114 | except Full as e: 115 | #print e 116 | self.logger.error(e.message) 117 | 118 | class InotifyThread(threading.Thread): 119 | def __init__(self, bucket, root, queue, *args, **kwargs): 120 | threading.Thread.__init__(self, *args, **kwargs) 121 | self.bucket = bucket 122 | self.queue = queue 123 | self.root = root 124 | self.logger = logging.getLogger('app') 125 | self._terminate = False 126 | 127 | def terminate(self): 128 | self._terminate = True 129 | self.notify.stop() 130 | 131 | def start_notify(self, monitered_dir): 132 | wm = pyinotify.WatchManager() 133 | mask = pyinotify.IN_DELETE | pyinotify.IN_CREATE | pyinotify.IN_MODIFY | pyinotify.IN_MOVED_FROM | pyinotify.IN_MOVED_TO 134 | self.notifier = pyinotify.Notifier(wm, EventHandler(monitered_dir, self.queue, self.bucket), timeout = 10) 135 | wm.add_watch(monitered_dir, mask, rec = True, auto_add = True) 136 | self.logger.info('now starting monitor %s'%(monitered_dir)) 137 | # self.notifier.loop() 138 | while True: 139 | if self._terminate: 140 | break 141 | self.notifier.process_events() 142 | if self.notifier.check_events(): 143 | self.notifier.read_events() 144 | 145 | def run(self): 146 | self.start_notify(self.root) 147 | return 148 | 149 | if __name__ == '__main__': 150 | queue = Queue() 151 | root = '.' 152 | bucket = 'dzdata' 153 | logger = logging.getLogger('app') 154 | logger.setLevel(logging.INFO) 155 | logger.addHandler(logging.FileHandler('logs/app.log')) 156 | inotifythd = InotifyThread(bucket, root, queue) 157 | inotifythd.start() 158 | 159 | -------------------------------------------------------------------------------- /ossync.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Copyright (c) 2012 Wu Tangsheng(lanbaba) 4 | 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | 12 | # The above copyright notice and this permission notice shall be included in 13 | # all copies or substantial portions of the Software. 14 | 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | # THE SOFTWARE. 22 | 23 | import os, threading 24 | from Queue import * 25 | from ossync.sdk.oss_api import * 26 | from queue_thread import QueueThread 27 | from sync_thread import SyncThread 28 | from inotify_thread import InotifyThread 29 | from init import * 30 | 31 | if __name__ == '__main__': 32 | try: 33 | set_sys_to_utf8() 34 | logger = get_logger() 35 | check_config(logger) 36 | queue = Queue() 37 | # check unprocessed items, if exists queue them 38 | queue_unprocessed(queue, logger) 39 | oss = OssAPI(HOST, ACCESS_ID, SECRET_ACCESS_KEY) 40 | 41 | syncthd = SyncThread(oss, queue) 42 | syncthd.start() 43 | dirs = [] 44 | for oss_mapper in oss_mappers: 45 | bucket = oss_mapper['bucket'] 46 | local_folders = oss_mapper['local_folders'] 47 | if(len(bucket) > 0 and len(local_folders) > 0): 48 | for local_folder in local_folders: 49 | inotifythd = InotifyThread(bucket, local_folder, queue) 50 | inotifythd.daemon = True 51 | dirs.append(local_folder) 52 | # inotifythd.start() 53 | queuethd = QueueThread(bucket, dirs, queue) 54 | queuethd.start() 55 | except KeyboardInterrupt as e: 56 | for thd in threading.enumerate(): 57 | if thd is main_thread: 58 | continue 59 | else: 60 | thd.terminate() 61 | logger.error(msg) 62 | #print e.message() 63 | exit(0) 64 | 65 | -------------------------------------------------------------------------------- /ossync/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lanbaba/ossync/7fe9e6ceaac62b0ccba956bc1e09c730cf75ea81/ossync/__init__.py -------------------------------------------------------------------------------- /ossync/lib/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lanbaba/ossync/7fe9e6ceaac62b0ccba956bc1e09c730cf75ea81/ossync/lib/__init__.py -------------------------------------------------------------------------------- /ossync/lib/helper.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Copyright (c) 2012 Wu Tangsheng(lanbaba) 4 | 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | 12 | # The above copyright notice and this permission notice shall be included in 13 | # all copies or substantial portions of the Software. 14 | 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | # THE SOFTWARE. 22 | 23 | import os, fnmatch 24 | import hashlib 25 | 26 | def encode(str): 27 | if(isinstance(str, unicode)): 28 | return str.encode('utf-8') 29 | else: 30 | return str 31 | 32 | def walk_files(root, patterns = '*', single_level = False, yield_folders = False): 33 | patterns = patterns.split(';') 34 | for path, subdirs, files in os.walk(root): 35 | if yield_folders: 36 | files.extend(subdirs) 37 | files.sort() 38 | for name in files: 39 | for pattern in patterns: 40 | if fnmatch.fnmatch(name, pattern): 41 | yield os.path.join(path, name) 42 | break 43 | if single_level: 44 | break 45 | 46 | def calc_file_md5(filepath): 47 | """calc files's hashcode """ 48 | with open(filepath, 'rb') as f: 49 | md5obj = hashlib.md5() 50 | md5obj.update(f.read()) 51 | hashstr = md5obj.hexdigest() 52 | return hashstr 53 | 54 | def calc_el_md5(root, relpath, bucket): 55 | """calc queue element's hashcode """ 56 | m = hashlib.md5() 57 | m.update(root + relpath + bucket) 58 | hashcode = m.hexdigest() 59 | return hashcode 60 | 61 | 62 | 63 | 64 | -------------------------------------------------------------------------------- /ossync/lib/queue_model.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Copyright (c) 2012 Wu Tangsheng(lanbaba) 4 | 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | 12 | # The above copyright notice and this permission notice shall be included in 13 | # all copies or substantial portions of the Software. 14 | 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | # THE SOFTWARE. 22 | 23 | import sqlite3, md5 24 | import itertools 25 | import hashlib 26 | 27 | class QueueModel(object): 28 | def __init__(self, dbpath): 29 | self.dbpath = dbpath 30 | 31 | def open(self): 32 | self.conn = sqlite3.connect(self.dbpath) 33 | self.conn.text_factory = lambda x: unicode(x, 'utf-8', 'ignore') 34 | self.cursor = self.conn.cursor() 35 | 36 | def close(self): 37 | self.cursor.close() 38 | self.conn.close() 39 | 40 | def save(self, data={"root": '', "relpath": '', "bucket": '', "action": '', "status": 0, "retries": 0}): 41 | if(len(data) == 0): 42 | return False 43 | m = hashlib.md5() 44 | m.update(data['root'] + data['relpath'] + data['bucket']) 45 | hashcode = m.hexdigest() 46 | self.cursor.execute('insert into queue values(?, ?, ?, ?, ?, ?, ?)', (data['root'], data['relpath'], data['bucket'], data['action'], data['status'], hashcode, data['retries'])) 47 | self.conn.commit() 48 | 49 | def get(self, hashcode): 50 | self.cursor.execute('select * from queue where hashcode=?', (hashcode, )) 51 | result = self._map_fields(self.cursor) 52 | if len(result) > 0: 53 | return result[0] 54 | return None 55 | 56 | def find_all(self, status): 57 | self.cursor.execute('select * from queue where status=?', (status, )) 58 | result = self._map_fields(self.cursor) 59 | if len(result) > 0: 60 | return result 61 | return None 62 | 63 | def update_status(self, hashcode, status): 64 | self.cursor.execute('update queue set status=? where hashcode=?', (status, hashcode)) 65 | self.conn.commit() 66 | 67 | def update_action(self, hashcode, action): 68 | self.cursor.execute('update queue set action=? where hashcode=?', (action, hashcode)) 69 | self.conn.commit() 70 | 71 | def update_retries(self, hashcode, retries): 72 | self.cursor.execute('update queue set retries=? where hashcode=?', (retries, hashcode)) 73 | self.conn.commit() 74 | 75 | def delete(self, hashcode): 76 | self.cursor.execute('delete from queue where hashcode=?', (hashcode,)) 77 | self.conn.commit() 78 | 79 | def _map_fields(self, cursor): 80 | """将结果元组映射到命名字段中""" 81 | filednames = [d[0].lower() for d in cursor.description] 82 | result = [] 83 | while True: 84 | rows = cursor.fetchmany() 85 | if not rows: 86 | break 87 | for row in rows: 88 | result.append(dict(itertools.izip(filednames, row))) 89 | return result 90 | 91 | 92 | -------------------------------------------------------------------------------- /ossync/sdk/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lanbaba/ossync/7fe9e6ceaac62b0ccba956bc1e09c730cf75ea81/ossync/sdk/__init__.py -------------------------------------------------------------------------------- /ossync/sdk/oss_api.py: -------------------------------------------------------------------------------- 1 | #coding=utf8 2 | 3 | # Copyright (c) 2011, Alibaba Cloud Computing 4 | # All rights reserved. 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a 7 | # copy of this software and associated documentation files (the 8 | # "Software"), to deal in the Software without restriction, including 9 | # without limitation the rights to use, copy, modify, merge, publish, dis- 10 | # tribute, sublicense, and/or sell copies of the Software, and to permit 11 | # persons to whom the Software is furnished to do so, subject to the fol- 12 | # lowing conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included 15 | # in all copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 18 | # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- 19 | # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 20 | # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 21 | # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 23 | # IN THE SOFTWARE. 24 | 25 | import httplib 26 | import time 27 | import base64 28 | import hmac 29 | import os 30 | import urllib 31 | import StringIO 32 | from oss_util import * 33 | from hashlib import sha1 as sha 34 | 35 | class OssAPI: 36 | ''' 37 | A simple OSS API 38 | ''' 39 | DefaultContentType = 'application/octet-stream' 40 | SendBufferSize = 8192 41 | GetBufferSize = 1024*1024*10 42 | 43 | def __init__(self, host, access_id, secret_access_key=""): 44 | self.host = host 45 | self.access_id = access_id 46 | self.secret_access_key = secret_access_key 47 | 48 | def sign_url_auth_with_expire_time(self, method, url, headers = {}, resource="/", timeout = 60): 49 | ''' 50 | Create the authorization for OSS based on the input method, url, body and headers 51 | :type method: string 52 | :param method: one of PUT, GET, DELETE, HEAD 53 | 54 | :type url: string 55 | :param:HTTP address of bucket or object, eg: http://HOST/bucket/object 56 | 57 | :type headers: dict 58 | :param: HTTP header 59 | 60 | :type resource: string 61 | :param:path of bucket or object, eg: /bucket/ or /bucket/object 62 | 63 | :type timeout: int 64 | :param 65 | 66 | Returns: 67 | signature url. 68 | ''' 69 | 70 | send_time= safe_get_element('Date', headers) 71 | if len(send_time) == 0: 72 | send_time = str(int(time.time()) + timeout) 73 | headers['Date'] = send_time 74 | auth_value = get_assign(self.secret_access_key, method, headers, resource) 75 | params = {"OSSAccessKeyId":self.access_id, "Expires":str(send_time), "Signature":auth_value} 76 | return append_param(url, params) 77 | 78 | def _create_sign_for_normal_auth(self, method, headers = {}, resource="/"): 79 | ''' 80 | NOT public API 81 | Create the authorization for OSS based on header input. 82 | it should be put into "Authorization" parameter of header. 83 | 84 | :type method: string 85 | :param:one of PUT, GET, DELETE, HEAD 86 | 87 | :type headers: dict 88 | :param: HTTP header 89 | 90 | :type resource: string 91 | :param:path of bucket or object, eg: /bucket/ or /bucket/object 92 | 93 | Returns: 94 | signature string 95 | 96 | ''' 97 | 98 | auth_value = "OSS " + self.access_id + ":" + get_assign(self.secret_access_key, method, headers, resource) 99 | return auth_value 100 | 101 | def bucket_operation(self, method, bucket, headers={}, params={}): 102 | ''' 103 | Send bucket operation request 104 | 105 | :type method: string 106 | :param method: one of PUT, GET, DELETE, HEAD 107 | 108 | :type bucket: string 109 | :param 110 | 111 | :type headers: dict 112 | :param: HTTP header 113 | 114 | :type params: dict 115 | :param: parameters that will be appeded at the end of resource 116 | 117 | Returns: 118 | HTTP Response 119 | 120 | ''' 121 | 122 | url = append_param("/" + bucket + "/", params) 123 | date = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime()) 124 | # Create REST header 125 | headers['Date'] = date 126 | headers['Host'] = self.host 127 | if params.has_key('acl'): 128 | resource = "/" + bucket + "/?acl" 129 | else: 130 | resource = "/" + bucket + "/" 131 | if "" != self.secret_access_key and "" != self.access_id: 132 | headers['Authorization'] = self._create_sign_for_normal_auth(method, headers, resource) 133 | elif "" != self.access_id: 134 | headers['Authorization'] = self.access_id 135 | 136 | conn = httplib.HTTPConnection(self.host) 137 | conn.request(method, url, "", headers) 138 | return conn.getresponse() 139 | 140 | def object_operation(self, method, bucket, object, headers = {}, data=""): 141 | ''' 142 | Send Object operation request 143 | 144 | :type method: string 145 | :param method: one of PUT, GET, DELETE, HEAD 146 | 147 | :type bucket: string 148 | :param 149 | 150 | :type object: string 151 | :param 152 | 153 | :type headers: dict 154 | :param: HTTP header 155 | 156 | :type data: string 157 | :param 158 | 159 | Returns: 160 | HTTP Response 161 | ''' 162 | 163 | if isinstance(object, unicode): 164 | object = object.encode('utf-8') 165 | resource = "/" + bucket + "/" 166 | resource = resource.encode('utf-8') + object 167 | object = urllib.quote(object) 168 | url = "/" + bucket + "/" + object 169 | date = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime()) 170 | 171 | # Create REST header 172 | headers['Date'] = date 173 | headers['Host'] = self.host 174 | if "" != self.secret_access_key and "" != self.access_id: 175 | headers['Authorization'] = self._create_sign_for_normal_auth(method, headers, resource) 176 | elif "" != self.access_id: 177 | headers['Authorization'] = self.access_id 178 | 179 | conn = httplib.HTTPConnection(self.host) 180 | conn.request(method, url, data, headers) 181 | return conn.getresponse() 182 | 183 | def get_service(self): 184 | ''' 185 | List all buckets of user 186 | ''' 187 | return self.list_all_my_buckets() 188 | 189 | def list_all_my_buckets(self): 190 | ''' 191 | List all buckets of user 192 | ''' 193 | 194 | method = "GET" 195 | url = "/" 196 | date = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime()) 197 | headers = {} 198 | 199 | headers['Date'] = date 200 | headers['Host'] = self.host 201 | resource = "/" 202 | if "" != self.secret_access_key and "" != self.access_id: 203 | headers['Authorization'] = self._create_sign_for_normal_auth(method, headers, resource) 204 | elif "" != self.access_id: 205 | headers['Authorization'] = self.access_id 206 | 207 | conn = httplib.HTTPConnection(self.host) 208 | conn.request(method, url, "", headers) 209 | return conn.getresponse() 210 | 211 | def get_bucket_acl(self, bucket): 212 | ''' 213 | Get Access Control Level of bucket 214 | 215 | :type bucket: string 216 | :param 217 | 218 | Returns: 219 | HTTP Response 220 | ''' 221 | headers = {} 222 | params = {} 223 | params['acl'] = '' 224 | return self.bucket_operation("GET", bucket, headers, params) 225 | 226 | def get_bucket(self, bucket, prefix='', marker='', delimiter='', maxkeys='', headers = {}): 227 | ''' 228 | List object that in bucket 229 | ''' 230 | return self.list_bucket(bucket, prefix, marker, delimiter, maxkeys, headers) 231 | 232 | def list_bucket(self, bucket, prefix='', marker='', delimiter='', maxkeys='', headers = {}): 233 | ''' 234 | List object that in bucket 235 | 236 | :type bucket: string 237 | :param 238 | 239 | :type prefix: string 240 | :param 241 | 242 | :type marker: string 243 | :param 244 | 245 | :type delimiter: string 246 | :param 247 | 248 | :type maxkeys: string 249 | :param 250 | 251 | :type headers: dict 252 | :param: HTTP header 253 | 254 | Returns: 255 | HTTP Response 256 | ''' 257 | 258 | params = {} 259 | params['prefix'] = prefix 260 | params['marker'] = marker 261 | params['delimiter'] = delimiter 262 | params['max-keys'] = maxkeys 263 | return self.bucket_operation("GET", bucket, headers, params) 264 | 265 | def create_bucket(self, bucket, acl='', headers = {}): 266 | ''' 267 | Create bucket 268 | ''' 269 | return self.put_bucket(bucket, acl, headers) 270 | 271 | def put_bucket(self, bucket, acl='', headers = {}): 272 | ''' 273 | Create bucket 274 | 275 | :type bucket: string 276 | :param 277 | 278 | :type acl: string 279 | :param: one of private 280 | 281 | :type headers: dict 282 | :param: HTTP header 283 | 284 | Returns: 285 | HTTP Response 286 | ''' 287 | 288 | if acl != '': 289 | headers['x-oss-acl'] = acl 290 | return self.bucket_operation("PUT", bucket, headers) 291 | 292 | def delete_bucket(self, bucket): 293 | ''' 294 | Delete bucket 295 | 296 | :type bucket: string 297 | :param 298 | 299 | Returns: 300 | HTTP Response 301 | ''' 302 | 303 | return self.bucket_operation("DELETE", bucket) 304 | 305 | def put_object_with_data(self, bucket, object, input_content, content_type=DefaultContentType, headers = {}): 306 | ''' 307 | Put object into bucket, the content of object is from input_content 308 | ''' 309 | return self.put_object_from_string(bucket, object, input_content, content_type, headers) 310 | 311 | def put_object_from_string(self, bucket, object, input_content, content_type=DefaultContentType, headers = {}): 312 | ''' 313 | Put object into bucket, the content of object is from input_content 314 | 315 | :type bucket: string 316 | :param 317 | 318 | :type object: string 319 | :param 320 | 321 | :type input_content: string 322 | :param 323 | 324 | :type content_type: string 325 | :param: the object content type that supported by HTTP 326 | 327 | :type headers: dict 328 | :param: HTTP header 329 | 330 | Returns: 331 | HTTP Response 332 | ''' 333 | 334 | headers['Content-Type'] = content_type 335 | headers['Content-Length'] = str(len(input_content)) 336 | fp = StringIO.StringIO(input_content) 337 | res = self.put_object_from_fp(bucket, object, fp, content_type, headers) 338 | fp.close() 339 | return res 340 | 341 | def _open_conn_to_put_object(self, bucket, object, filesize, content_type=DefaultContentType, headers = {}): 342 | ''' 343 | NOT public API 344 | Open a connectioon to put object 345 | 346 | :type bucket: string 347 | :param 348 | 349 | :type filesize: int 350 | :param 351 | 352 | :type object: string 353 | :param 354 | 355 | :type input_content: string 356 | :param 357 | 358 | :type content_type: string 359 | :param: the object content type that supported by HTTP 360 | 361 | :type headers: dict 362 | :param: HTTP header 363 | 364 | Returns: 365 | HTTP Response 366 | ''' 367 | 368 | method = "PUT" 369 | if isinstance(object, unicode): 370 | object = object.encode('utf-8') 371 | resource = "/" + bucket + "/" 372 | resource = resource.encode('utf-8') + object 373 | object = urllib.quote(object) 374 | url = "/" + bucket + "/" + object 375 | date = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime()) 376 | 377 | conn = httplib.HTTPConnection(self.host) 378 | conn.putrequest(method, url) 379 | headers["Content-Type"] = content_type 380 | headers["Content-Length"] = filesize 381 | headers["Date"] = date 382 | headers["Host"] = self.host 383 | headers["Expect"] = "100-Continue" 384 | for k in headers.keys(): 385 | conn.putheader(k, headers[k]) 386 | if "" != self.secret_access_key and "" != self.access_id: 387 | auth = self._create_sign_for_normal_auth(method, headers, resource) 388 | conn.putheader("Authorization", auth) 389 | conn.endheaders() 390 | return conn 391 | 392 | def put_object_from_file(self, bucket, object, filename, content_type=DefaultContentType, headers = {}): 393 | ''' 394 | put object into bucket, the content of object is read from file 395 | 396 | :type bucket: string 397 | :param 398 | 399 | :type object: string 400 | :param 401 | 402 | :type fllename: string 403 | :param: the name of the read file 404 | 405 | :type content_type: string 406 | :param: the object content type that supported by HTTP 407 | 408 | :type headers: dict 409 | :param: HTTP header 410 | 411 | Returns: 412 | HTTP Response 413 | ''' 414 | 415 | fp = open(filename, 'rb') 416 | res = self.put_object_from_fp(bucket, object, fp, content_type, headers) 417 | fp.close() 418 | return res 419 | 420 | def put_object_from_fp(self, bucket, object, fp, content_type=DefaultContentType, headers = {}): 421 | ''' 422 | Put object into bucket, the content of object is read from file pointer 423 | 424 | :type bucket: string 425 | :param 426 | 427 | :type object: string 428 | :param 429 | 430 | :type fp: file 431 | :param: the pointer of the read file 432 | 433 | :type content_type: string 434 | :param: the object content type that supported by HTTP 435 | 436 | :type headers: dict 437 | :param: HTTP header 438 | 439 | Returns: 440 | HTTP Response 441 | ''' 442 | 443 | fp.seek(os.SEEK_SET, os.SEEK_END) 444 | filesize = fp.tell() 445 | fp.seek(os.SEEK_SET) 446 | 447 | conn = self._open_conn_to_put_object(bucket, object, filesize, content_type, headers) 448 | l = fp.read(self.SendBufferSize) 449 | while len(l) > 0: 450 | conn.send(l) 451 | l = fp.read(self.SendBufferSize) 452 | return conn.getresponse() 453 | 454 | def get_object(self, bucket, object, headers = {}): 455 | ''' 456 | Get object 457 | 458 | :type bucket: string 459 | :param 460 | 461 | :type object: string 462 | :param 463 | 464 | :type headers: dict 465 | :param: HTTP header 466 | 467 | Returns: 468 | HTTP Response 469 | ''' 470 | 471 | return self.object_operation("GET", bucket, object, headers) 472 | 473 | def get_object_to_file(self, bucket, object, filename, headers = {}): 474 | ''' 475 | Get object and write the content of object into a file 476 | 477 | :type bucket: string 478 | :param 479 | 480 | :type object: string 481 | :param 482 | 483 | :type filename: string 484 | :param 485 | 486 | :type headers: dict 487 | :param: HTTP header 488 | 489 | Returns: 490 | HTTP Response 491 | ''' 492 | 493 | res = self.get_object(bucket, object, headers) 494 | f = file(filename, 'wb') 495 | data = "" 496 | while True: 497 | data = res.read(self.GetBufferSize) 498 | if len(data) != 0: 499 | f.write(data) 500 | else: 501 | break 502 | f.close() 503 | # TODO: get object with flow 504 | return res 505 | 506 | def delete_object(self, bucket, object, headers = {}): 507 | ''' 508 | Delete object 509 | 510 | :type bucket: string 511 | :param 512 | 513 | :type object: string 514 | :param 515 | 516 | :type headers: dict 517 | :param: HTTP header 518 | 519 | Returns: 520 | HTTP Response 521 | ''' 522 | 523 | return self.object_operation("DELETE", bucket, object, headers) 524 | 525 | def head_object(self, bucket, object, headers = {}): 526 | ''' 527 | Head object, to get the meta message of object without the content 528 | 529 | :type bucket: string 530 | :param 531 | 532 | :type object: string 533 | :param 534 | 535 | :type headers: dict 536 | :param: HTTP header 537 | 538 | Returns: 539 | HTTP Response 540 | ''' 541 | 542 | return self.object_operation("HEAD", bucket, object, headers) 543 | 544 | def post_object_group(self, bucket, object, object_group_msg_xml, headers = {}, params = {}): 545 | ''' 546 | Post object group, merge all objects in object_group_msg_xml into one object 547 | :type bucket: string 548 | :param 549 | 550 | :type object: string 551 | :param 552 | 553 | :type object_group_msg_xml: string 554 | :param: xml format string, like 555 | 556 | 557 | N 558 | objectN 559 | "47BCE5C74F589F4867DBD57E9CA9F808" 560 | 561 | 562 | :type headers: dict 563 | :param: HTTP header 564 | 565 | :type params: dict 566 | :param: parameters 567 | 568 | Returns: 569 | HTTP Response 570 | ''' 571 | method = "POST" 572 | url = "/" + bucket + "/" + object + "?group" 573 | date = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime()) 574 | # Create REST header 575 | headers['Date'] = date 576 | headers['Host'] = self.host 577 | headers['Content-Type'] = 'text/xml' 578 | headers['Content-Length'] = len(object_group_msg_xml) 579 | resource = "/" + bucket + "/" + object + "?group" 580 | if "" != self.secret_access_key and "" != self.access_id: 581 | headers['Authorization'] = self._create_sign_for_normal_auth(method, headers, resource) 582 | elif "" != self.access_id: 583 | headers['Authorization'] = self.access_id 584 | 585 | conn = httplib.HTTPConnection(self.host) 586 | conn.request(method, url, object_group_msg_xml, headers) 587 | return conn.getresponse() 588 | 589 | def get_object_group_index(self, bucket, object, headers = {}): 590 | ''' 591 | Get object group_index 592 | 593 | :type bucket: string 594 | :param 595 | 596 | :type object: string 597 | :param 598 | 599 | :type headers: dict 600 | :param: HTTP header 601 | 602 | Returns: 603 | HTTP Response 604 | ''' 605 | 606 | headers["x-oss-file-group"] = "" 607 | return self.object_operation("GET", bucket, object, headers) 608 | 609 | 610 | def put_object_from_file_given_pos(self, bucket, object, filename, offset, partsize, content_type=DefaultContentType, headers = {}): 611 | ''' 612 | Put object into bucket, the content of object is read from given posision of filename 613 | :type bucket: string 614 | :param 615 | 616 | :type object: string 617 | :param 618 | 619 | :type fllename: string 620 | :param: the name of the read file 621 | 622 | :type offset: int 623 | :param: the given position of file 624 | 625 | :type partsize: int 626 | :param: the size of read content 627 | 628 | :type content_type: string 629 | :param: the object content type that supported by HTTP 630 | 631 | :type headers: dict 632 | :param: HTTP header 633 | 634 | Returns: 635 | HTTP Response 636 | ''' 637 | fp = open(filename, 'rb') 638 | if offset > os.path.getsize(filename): 639 | fp.seek(os.SEEK_SET, os.SEEK_END) 640 | else: 641 | fp.seek(offset) 642 | 643 | conn = self._open_conn_to_put_object(bucket, object, partsize, content_type, headers) 644 | 645 | left_len = partsize 646 | while True: 647 | if left_len <= 0: 648 | break 649 | elif left_len < self.SendBufferSize: 650 | buffer_content = fp.read(left_len) 651 | else: 652 | buffer_content = fp.read(self.SendBufferSize) 653 | 654 | if len(buffer_content) > 0: 655 | conn.send(buffer_content) 656 | 657 | left_len = left_len - len(buffer_content) 658 | 659 | fp.close() 660 | return conn.getresponse() 661 | 662 | def upload_large_file(self, bucket, object, filename, thread_num = 10, max_part_num = 1000): 663 | ''' 664 | Upload large file, the content is read from filename. The large file is splitted into many parts. It will put the many parts into bucket and then merge all the parts into one object. 665 | 666 | :type bucket: string 667 | :param 668 | 669 | :type object: string 670 | :param 671 | 672 | :type fllename: string 673 | :param: the name of the read file 674 | 675 | ''' 676 | #split the large file into 1000 parts or many parts 677 | #get part_msg_list 678 | if isinstance(filename, unicode): 679 | filename = filename.encode('utf-8') 680 | 681 | part_msg_list = split_large_file(filename, object, max_part_num) 682 | 683 | #make sure all the parts are put into same bucket 684 | if len(part_msg_list) < thread_num and len(part_msg_list) != 0: 685 | thread_num = len(part_msg_list) 686 | 687 | step = len(part_msg_list) / thread_num 688 | threadpool = [] 689 | for i in range(0, thread_num): 690 | if i == thread_num - 1: 691 | end = len(part_msg_list) 692 | else: 693 | end = i * step + step 694 | 695 | begin = i * step 696 | oss = OssAPI(self.host, self.access_id, self.secret_access_key) 697 | current = PutObjectGroupWorker(oss, bucket, filename, part_msg_list[begin:end]) 698 | threadpool.append(current) 699 | current.start() 700 | 701 | for item in threadpool: 702 | item.join() 703 | 704 | #get xml string that contains msg of object group 705 | object_group_msg_xml = create_object_group_msg_xml(part_msg_list) 706 | 707 | return self.post_object_group(bucket, object, object_group_msg_xml) 708 | -------------------------------------------------------------------------------- /ossync/sdk/oss_cmd.py: -------------------------------------------------------------------------------- 1 | #coding=utf8 2 | from oss_api import * 3 | from oss_xml_handler import * 4 | 5 | msg_remind = " PLEASE USE # TO SEPERATE EACH INPUT PARAMETER" 6 | 7 | msg_quit = "q or quit, will quit out this console" 8 | msg_help = "h or help, will show the message again" 9 | 10 | #msg_get_service = "gs means get service , it lists all buckets that user has" 11 | msg_get_service = "gs means get service." 12 | example_get_service = "Input: gs" 13 | GET_SERVICE = "get service" 14 | 15 | #msg_put_bucket = "pb means put bucket, it creates bucket" 16 | msg_put_bucket = "pb means put bucket." 17 | example_put_bucket = "Input: pb#bucket||pb#bucket#acl||pb#bucket#acl#headers. " 18 | PUT_BUCKET = "put bucket" 19 | 20 | #msg_delete_bucket = "db means delete bucket, it deletes the bucket that user created" 21 | msg_delete_bucket = "db means delete bucket." 22 | example_delete_bucket = "Input: db#bucket" 23 | DELETE_BUCKET = "delete bucket" 24 | 25 | #msg_get_bucket = "gb means get bucket, it lists objects in the bucket" 26 | msg_get_bucket = "gb means get bucket." 27 | example_get_bucket = "Input: gb#bucket||gb#bucket#prefix||gb#bucket#prefix#marker||gb#bucket#prefix#marker#delimiter||gb#bucket#prefix#marker#delimiter#maxkeys||gb#bucket#prefix#marker#delimiter#maxkeys#headers" 28 | GET_BUCKET = "get bucket" 29 | 30 | #msg_get_bucket_acl = "gba means get bucket acl, it lists bucket acl" 31 | msg_get_bucket_acl = "gba means get bucket acl." 32 | example_get_bucket_acl = "Input: gba#bucket" 33 | GET_BUCKET_ACL = "get bucket acl" 34 | 35 | #msg_put_bucket_acl = "pba means put bucket acl, it sets bucket acl" 36 | msg_put_bucket_acl = "pba means put bucket acl." 37 | example_put_bucket_acl = "Input: pba#bucket#acl, Example: pba#mybucket#public-read" 38 | PUT_BUCKET_ACL = "put bucket acl" 39 | 40 | #msg_put_object_with_data = "powd means put object with data, it puts object, its content from string data" 41 | msg_put_object_with_data = "powd means put object with data." 42 | example_put_object_with_data = "Input: powd#bucket#object#string_data||powd#bucket#object#string_data#content_type||powd#bucket#object#string_data#content_type#headers, Example: powd#mybucket#myobjcet#abcdefghijk#txt/plain" 43 | PUT_OBJECT_WITH_DATA = "put object with data" 44 | 45 | #msg_put_object_from_file = "poff means put object from file, it puts object , its content from file" 46 | msg_put_object_from_file = "poff means put object from file." 47 | example_put_object_from_file = "Input: poff#bucket#object#filename||poff#bucket#object#filename#content_type||poff#bucket#object#filename#content_type#headers, Example: poff#mybucket#myobjcet#myfilename" 48 | PUT_OBJECT_FROM_FILE = "put object from file" 49 | 50 | #msg_get_object = "go means get object, it gets object and print its content" 51 | msg_get_object = "go means get object." 52 | example_get_object = "Input: go#bucket#object||go#bucket#object#headers, Example: go#mybucket#myobjcet" 53 | GET_OBJECT = "get object" 54 | 55 | #msg_get_object_to_file = "gotf means get object to file, it gets object and save its content to file" 56 | msg_get_object_to_file = "gotf means get object to file." 57 | example_get_object_to_file = "Input: gotf#bucket#object#filename||go#bucket#object#filename#headers, Example: goff#mybucket#myobjcet#tofilename" 58 | GET_OBJECT_TO_FILE = "get object to file" 59 | 60 | #msg_delete_object = "do means delete object, it deletes object" 61 | msg_delete_object = "do means delete object." 62 | example_delete_object = "Input: do#bucket#object||do#bucket#object#headers, Example: do#mybucket#myobjcet" 63 | DELETE_OBJECT = "delete object" 64 | 65 | #msg_head_object = "ho means head object, it gets meta info of object" 66 | msg_head_object = "ho means head object." 67 | example_head_object = "Input: ho#bucket#object||ho#bucket#object#headers, Example: ho#mybucket#myobject" 68 | HEAD_OBJECT = "head object" 69 | 70 | def usage(): 71 | width = 40 72 | print "=> 1) ", msg_quit.ljust(width) 73 | print "=> 2) ", msg_help.ljust(width) 74 | print '***************************************' 75 | print "NOTE:acl in below means access control level, acl SHOULD be public-read, private, or public-read-write" 76 | print "headers and parameters below means dic, it SHOULD be dic like a=b c=d" 77 | print "content type means object type , it is used in headers, default is application/octet-stream" 78 | print '***************************************' 79 | print 'bucket operation' 80 | print "=> 3) ", msg_get_service.ljust(width), example_get_service 81 | print "=> 4) ", msg_put_bucket.ljust(width), example_put_bucket 82 | print "=> 5) ", msg_delete_bucket.ljust(width), example_delete_bucket 83 | print "=> 6) ", msg_get_bucket.ljust(width), example_get_bucket 84 | print "=> 7) ", msg_get_bucket_acl.ljust(width), example_get_bucket_acl 85 | print "=> 8) ", msg_put_bucket_acl.ljust(width), example_put_bucket_acl 86 | print '' 87 | print '***************************************' 88 | print 'objcet operation' 89 | print "=> 9) ", msg_put_object_with_data.ljust(width), example_put_object_with_data 90 | print "=>10) ", msg_put_object_from_file.ljust(width), example_put_object_from_file 91 | print "=>11) ", msg_get_object.ljust(width), example_get_object 92 | print "=>12) ", msg_get_object_to_file.ljust(width), example_get_object_to_file 93 | print "=>13) ", msg_delete_object.ljust(width), example_delete_object 94 | print "=>14) ", msg_head_object.ljust(width), example_head_object 95 | print '' 96 | 97 | def print_result(cmd, res): 98 | if res.status / 100 == 2: 99 | print cmd, "OK" 100 | else: 101 | print "error headers", res.getheaders() 102 | print "error body", res.read() 103 | print cmd, "Failed!" 104 | 105 | def check_input(cmd_str, cmd_list, min, max): 106 | if len(cmd_list) < min or len(cmd_list) > max: 107 | print "ERROR! ", cmd_str, "needs ", min, "-", max, " paramters" 108 | if example_map.has_key(cmd_str): 109 | print example_map[cmd_str] 110 | return False 111 | else: 112 | return True 113 | 114 | def get_cmd(cmd): 115 | string = cmd 116 | tmp_list = string.split('#') 117 | cmd_list = [] 118 | for i in tmp_list: 119 | if len(i.strip()) != 0: 120 | cmd_list.append(i.strip()) 121 | return cmd_list 122 | 123 | def transfer_string_to_dic(string): 124 | list = string.split() 125 | dic = {} 126 | for entry in list: 127 | key, val = entry.split('=') 128 | dic[key.strip()] = val.strip() 129 | print dic 130 | return dic 131 | 132 | if __name__ == "__main__": 133 | import sys 134 | host = "" 135 | user_name = "" 136 | access_id = "" 137 | secret_access_key = "" 138 | remind = "the method you input is not supported! please input h or help to check method supported" 139 | 140 | if len(sys.argv) != 4: 141 | print '***************************************' 142 | print 'Please input the parameters like below' 143 | print '***************************************' 144 | print 'python ', __file__,' host access_id access_key' 145 | print 'host is the ip address with port of oss apache server . For Example: 10.249.105.22:8080' 146 | print 'access_id is public key that oss server provided. For Example: 84792jahfdsah+=' 147 | print 'access_key is private secret key that oss server provided. For Example: atdh+=flahmzhha=+' 148 | print '***************************************' 149 | print '' 150 | print '' 151 | exit() 152 | else: 153 | host = sys.argv[1] 154 | access_id = sys.argv[2] 155 | secret_access_key = sys.argv[3] 156 | 157 | oss = OssAPI(host, access_id, secret_access_key) 158 | usage() 159 | bucketname = "" 160 | 161 | example_map = {} 162 | example_map[PUT_BUCKET] = example_put_bucket 163 | example_map[PUT_BUCKET_ACL] = example_put_bucket_acl 164 | example_map[GET_SERVICE] = example_get_service 165 | example_map[GET_BUCKET] = example_get_bucket 166 | example_map[DELETE_BUCKET] = example_delete_bucket 167 | example_map[GET_BUCKET_ACL] = example_get_bucket_acl 168 | example_map[PUT_OBJECT_WITH_DATA] = example_put_object_with_data 169 | example_map[PUT_OBJECT_FROM_FILE] = example_put_object_from_file 170 | example_map[GET_OBJECT] = example_get_object 171 | example_map[GET_OBJECT_TO_FILE] = example_get_object_to_file 172 | example_map[DELETE_OBJECT] = example_delete_object 173 | example_map[HEAD_OBJECT] = example_head_object 174 | 175 | while True: 176 | cmd = raw_input(">>") 177 | cmd = cmd.strip() 178 | cmd_list = get_cmd(cmd) 179 | if ("q" == cmd.lower() or "quit" == cmd.lower()): 180 | break 181 | elif ("h" == cmd.lower() or "help" == cmd.lower()): 182 | usage() 183 | elif len(cmd_list) > 0: 184 | if cmd_list[0].lower() == "pb": 185 | cmd_str = PUT_BUCKET 186 | min = 2 187 | max = 4 188 | if not check_input(cmd_str, cmd_list, min, max): 189 | pass 190 | else: 191 | print "cmd", cmd_list 192 | bucketname = cmd_list[1] 193 | if len(cmd_list) == 2: 194 | res = oss.put_bucket(cmd_list[1]) 195 | elif len(cmd_list) == 3: 196 | res = oss.put_bucket(cmd_list[1], cmd_list[2]) 197 | elif len(cmd_list) == 4: 198 | dic = transfer_string_to_dic(cmd_list[3]) 199 | res = oss.put_bucket(cmd_list[1], cmd_list[2], dic) 200 | print_result(cmd_str, res) 201 | 202 | elif cmd_list[0] == "pba": 203 | cmd_str = PUT_BUCKET_ACL 204 | min = 3 205 | max = 3 206 | if not check_input(cmd_str, cmd_list, min, max): 207 | pass 208 | else: 209 | print "cmd", cmd_list 210 | bucketname = cmd_list[1] 211 | res = oss.put_bucket(cmd_list[1], cmd_list[2]) 212 | print_result(cmd_str, res) 213 | 214 | elif cmd_list[0].lower() == "gs": 215 | cmd_str = GET_SERVICE 216 | min = 1 217 | max = 1 218 | if not check_input(cmd_str, cmd_list, min, max): 219 | pass 220 | else: 221 | print "cmd", cmd_list 222 | if len(cmd_list) == 1: 223 | res = oss.get_service() 224 | print_result(cmd_str, res) 225 | if (res.status / 100) == 2: 226 | body = res.read() 227 | h = GetServiceXml(body) 228 | print "bucket list size is: ", len(h.list()) 229 | print "bucket list is: " 230 | for i in h.list(): 231 | print i 232 | 233 | elif cmd_list[0].lower() == "gb": 234 | cmd_str = GET_BUCKET 235 | min = 2 236 | max = 7 237 | if not check_input(cmd_str, cmd_list, min, max): 238 | pass 239 | else: 240 | print "cmd", cmd_list 241 | bucketname = cmd_list[1] 242 | #get_bucket(bucket, prefix='', marker='', delimiter='', maxkeys='', headers = {}): 243 | if len(cmd_list) == 2: 244 | res = oss.get_bucket(cmd_list[1]) 245 | elif len(cmd_list) == 3: 246 | res = oss.get_bucket(cmd_list[1], cmd_list[2]) 247 | elif len(cmd_list) == 4: 248 | res = oss.get_bucket(cmd_list[1], cmd_list[2], cmd_list[3]) 249 | elif len(cmd_list) == 5: 250 | res = oss.get_bucket(cmd_list[1], cmd_list[2], cmd_list[3], cmd_list[4]) 251 | elif len(cmd_list) == 6: 252 | res = oss.get_bucket(cmd_list[1], cmd_list[2], cmd_list[3], cmd_list[4], cmd_list[5]) 253 | elif len(cmd_list) == 7: 254 | dic = transfer_string_to_dic(cmd_list[6]) 255 | res = oss.get_bucket(cmd_list[1], cmd_list[2], cmd_list[3], cmd_list[4], cmd_list[5], dic) 256 | print_result(cmd_str, res) 257 | if (res.status / 100) == 2: 258 | body = res.read() 259 | hh = GetBucketXml(body) 260 | (fl, pl) = hh.list() 261 | print "prefix list size is: ", len(pl) 262 | print "prefix listis: " 263 | for i in pl: 264 | print i 265 | print "file list size is: ", len(fl) 266 | print "file list is: " 267 | for i in fl: 268 | print i 269 | 270 | elif cmd_list[0].lower() == "db": 271 | cmd_str = DELETE_BUCKET 272 | print cmd_str 273 | min = 2 274 | max = 2 275 | if not check_input(cmd_str, cmd_list, min, max): 276 | pass 277 | else: 278 | print "cmd", cmd_list 279 | if len(cmd_list) == 2: 280 | res = oss.delete_bucket(cmd_list[1]) 281 | print_result(cmd_str, res) 282 | 283 | elif cmd_list[0].lower() == "gba": 284 | cmd_str = GET_BUCKET_ACL 285 | print cmd_str 286 | 287 | min = 2 288 | max = 2 289 | if not check_input(cmd_str, cmd_list, min, max): 290 | pass 291 | else: 292 | print "cmd", cmd_list 293 | bucketname = cmd_list[1] 294 | if len(cmd_list) == 2: 295 | res = oss.get_bucket_acl(cmd_list[1]) 296 | print_result(cmd_str, res) 297 | 298 | if (res.status / 100) == 2: 299 | body = res.read() 300 | h = GetBucketAclXml(body) 301 | print "bucket ", bucketname, " acl is: ", h.grant 302 | 303 | elif cmd_list[0].lower() == "powd" or cmd_list[0].lower() == "poff": 304 | if cmd_list[0].lower() == "powd": 305 | cmd_str = PUT_OBJECT_WITH_DATA 306 | else: 307 | cmd_str = PUT_OBJECT_FROM_FILE 308 | min = 4 309 | max = 6 310 | if not check_input(cmd_str, cmd_list, min, max): 311 | pass 312 | else: 313 | print "cmd", cmd_list 314 | bucketname = cmd_list[1] 315 | if isinstance(cmd_list[2], str): 316 | tmp = unicode(cmd_list[2], 'utf-8') 317 | cmd_list[2] = tmp 318 | if len(cmd_list) == 4: 319 | if cmd_list[0].lower() == "powd": 320 | res = oss.put_object_with_data(cmd_list[1], cmd_list[2], cmd_list[3]) 321 | else: 322 | res = oss.put_object_from_file(cmd_list[1], cmd_list[2], cmd_list[3]) 323 | elif len(cmd_list) == 5: 324 | if cmd_list[0].lower() == "powd": 325 | res = oss.put_object_with_data(cmd_list[1], cmd_list[2], cmd_list[3], cmd_list[4]) 326 | else: 327 | res = oss.put_object_from_file(cmd_list[1], cmd_list[2], cmd_list[3], cmd_list[4]) 328 | elif len(cmd_list) == 6: 329 | if cmd_list[0].lower() == "powd": 330 | dic = transfer_string_to_dic(cmd_list[5]) 331 | res = oss.put_object_with_data(cmd_list[1], cmd_list[2], cmd_list[3], cmd_list[4], dic) 332 | else: 333 | dic = transfer_string_to_dic(cmd_list[5]) 334 | res = oss.put_object_from_file(cmd_list[1], cmd_list[2], cmd_list[3], cmd_list[4], dic) 335 | print_result(cmd_str, res) 336 | 337 | elif cmd_list[0].lower() == "go": 338 | cmd_str = GET_OBJECT 339 | min = 3 340 | max = 4 341 | if not check_input(cmd_str, cmd_list, min, max): 342 | pass 343 | else: 344 | print "cmd", cmd_list 345 | bucketname = cmd_list[1] 346 | if len(cmd_list) == 3: 347 | res = oss.get_object(cmd_list[1], cmd_list[2]) 348 | elif len(cmd_list) == 4: 349 | dic = transfer_string_to_dic(cmd_list[3]) 350 | res = oss.get_object(cmd_list[1], cmd_list[2], dic) 351 | print_result(cmd_str, res) 352 | if res.status / 100 == 2: 353 | print res.read() 354 | 355 | elif cmd_list[0].lower() == "gotf": 356 | cmd_str = GET_OBJECT_TO_FILE 357 | min = 4 358 | max = 5 359 | if not check_input(cmd_str, cmd_list, min, max): 360 | pass 361 | else: 362 | print "cmd", cmd_list 363 | bucketname = cmd_list[1] 364 | if len(cmd_list) == 4: 365 | res = oss.get_object_to_file(cmd_list[1], cmd_list[2], cmd_list[3]) 366 | elif len(cmd_list) == 5: 367 | dic = transfer_string_to_dic(cmd_list[4]) 368 | res = oss.get_object_to_file(cmd_list[1], cmd_list[2], cmd_list[3], dic) 369 | print_result(cmd_str, res) 370 | 371 | elif cmd_list[0].lower() == "do": 372 | cmd_str = DELETE_OBJECT 373 | min = 3 374 | max = 4 375 | if not check_input(cmd_str, cmd_list, min, max): 376 | pass 377 | else: 378 | print "cmd", cmd_list 379 | bucketname = cmd_list[1] 380 | if len(cmd_list) == 3: 381 | res = oss.delete_object(cmd_list[1], cmd_list[2]) 382 | elif len(cmd_list) == 4: 383 | dic = transfer_string_to_dic(cmd_list[3]) 384 | res = oss.delete_object(cmd_list[1], cmd_list[2], dic) 385 | print_result(cmd_str, res) 386 | 387 | elif cmd_list[0].lower() == "ho": 388 | cmd_str = HEAD_OBJECT 389 | min = 3 390 | max = 4 391 | if not check_input(cmd_str, cmd_list, min, max): 392 | pass 393 | else: 394 | print "cmd", cmd_list 395 | bucketname = cmd_list[1] 396 | if len(cmd_list) == 3: 397 | res = oss.head_object(cmd_list[1], cmd_list[2]) 398 | elif len(cmd_list) == 4: 399 | dic = transfer_string_to_dic(cmd_list[3]) 400 | res = oss.head_object(cmd_list[1], cmd_list[2], dic) 401 | print_result(cmd_str, res) 402 | if res.status / 100 == 2: 403 | print res.getheaders() 404 | else: 405 | print remind 406 | else: 407 | print remind 408 | -------------------------------------------------------------------------------- /ossync/sdk/oss_fs.py: -------------------------------------------------------------------------------- 1 | #coding=utf8 2 | from oss_api import * 3 | from oss_xml_handler import * 4 | 5 | ''' 6 | Mock a file system for testing purpose 7 | ''' 8 | class OssFS: 9 | def __init__(self, host, id="", key=""): 10 | self.buckets = [] 11 | self.host = host 12 | self.oss = OssAPI(host, id, key) 13 | 14 | def put_bucket(self, bucketname,acl='private'): 15 | if bucketname in self.buckets: 16 | return False 17 | res = self.oss.put_bucket(bucketname,acl) 18 | if (res.status / 100) == 2: 19 | self.buckets.append(bucketname) 20 | return True 21 | else: 22 | return False 23 | 24 | def delete_bucket(self, bucketname): 25 | res = self.oss.delete_bucket(bucketname) 26 | if (res.status / 100) == 2: 27 | while bucketname in self.buckets: 28 | del self.buckets[self.buckets.index(bucketname)] 29 | return True 30 | else: 31 | return False 32 | 33 | def list_bucket(self): 34 | res = self.oss.get_service() 35 | if (res.status / 100) == 2: 36 | body = res.read() 37 | h = GetServiceXml(body) 38 | self.buckets = h.list() 39 | return self.buckets 40 | 41 | def get_bucket_acl(self, bucket): 42 | res = self.oss.get_bucket_acl(bucket) 43 | body = res.read() 44 | h = GetBucketAclXml(body) 45 | return h.grant 46 | 47 | 48 | def put_bucket_acl(self, bucket, acl=''): 49 | res = self.oss.put_bucket_acl(bucket,acl) 50 | if (res.status / 100) == 2: 51 | return True 52 | else: 53 | return False 54 | 55 | #fileobj is django defined uploaded file structure 56 | def upload_file(self, bucket, object, filename): 57 | res = self.oss.put_object_from_file(bucket, object, filename) 58 | if (res.status / 100) == 2: 59 | return True 60 | else: 61 | return False 62 | 63 | def make_dir(self, bucket, dirname): 64 | object = dirname + '/' 65 | res = self.oss.put_object_with_data(bucket, object, "") 66 | if (res.status / 100) == 2: 67 | return True 68 | else: 69 | return False 70 | 71 | def read_file(self, bucket, filename): 72 | res = self.oss.get_object(bucket, filename) 73 | return res.read() 74 | 75 | #return a list of file 76 | def list_file(self, bucket, prefix="", delim="", marker="", maxkeys=5): 77 | fl = [] 78 | pl = [] 79 | res = self.oss.get_bucket(bucket, prefix, delimiter=delim, marker=marker, maxkeys=maxkeys) 80 | if (res.status / 100) == 2: 81 | body = res.read() 82 | h = GetBucketXml(body) 83 | (fl, pl) = h.list() 84 | return (fl, pl) 85 | 86 | def delete_file(self, bucket, filename): 87 | res = self.oss.delete_object(bucket, filename) 88 | if (res.status / 100) == 2: 89 | return True 90 | else: 91 | return False 92 | 93 | def open_file_for_write(self, bucket, filename, filesize): 94 | conn = self.oss._open_conn_to_put_object(bucket, filename, filesize) 95 | return WriteFileObject(conn) 96 | 97 | def open_file_for_read(self, bucket, filename): 98 | res = self.oss.get_object(bucket, filename) 99 | return ReadFileObject(res) 100 | 101 | class ReadFileObject: 102 | def __init__(self, res): 103 | self.res = res 104 | 105 | def read(self, buffer_size): 106 | if (self.res.status / 100) == 2: 107 | return self.res.read(buffer_size) 108 | else: 109 | return "" 110 | 111 | def close(self): 112 | if (self.res.status / 100) == 2: 113 | return True 114 | else: 115 | return False 116 | 117 | class WriteFileObject: 118 | def __init__(self, conn): 119 | self.conn = conn 120 | 121 | def write(self, data): 122 | self.conn.send(data) 123 | 124 | def close(self): 125 | res = self.conn.getresponse() 126 | if (res.status / 100) == 2: 127 | return True 128 | else: 129 | return False 130 | 131 | 132 | def test_open_file_for_write(bucket, object, filename): 133 | fp = file(filename, 'r') 134 | fp.seek(os.SEEK_SET, os.SEEK_END) 135 | filesize = fp.tell() 136 | fp.seek(os.SEEK_SET) 137 | wo = fs.open_file_for_write(bucket, object, filesize) 138 | l = fp.read(BufferSize) 139 | while len(l) > 0: 140 | print len(l) 141 | wo.write(l) 142 | l = fp.read(BufferSize) 143 | print wo.close() 144 | 145 | def test_open_file_for_read(bucket, object, filename): 146 | fp = file(filename, 'w') 147 | ro = fs.open_file_for_read(bucket, object) 148 | buf = ro.read(1024) 149 | while len(buf) > 0: 150 | print len(buf) 151 | fp.write(buf) 152 | buf = ro.read(1024) 153 | print ro.close() 154 | 155 | if __name__ == "__main__": 156 | host = '127.0.0.1:8080' 157 | id = "" 158 | key = "" 159 | BufferSize = 8000 160 | fs = OssFS(host, id, key) 161 | test_open_file_for_write('6uu', "a/b/c/d", "put.jpg") 162 | test_open_file_for_read('6uu', "a/b/c/d", "put.jpg") 163 | print fs.upload_file('6uu', "a/b/c", 'put.jpg') 164 | print fs.upload_file('6uu', "b/b/c", 'put.jpg') 165 | print fs.upload_file('6uu', "b/c", 'put.jpg') 166 | print fs.upload_file('6uu', "c", 'put.jpg') 167 | (fl, pl) = fs.list_file('bb1','',delim='',marker='') 168 | print fl 169 | print pl 170 | (fl, pl) = fs.list_file('bb1','',delim='',marker='') 171 | print fl 172 | print pl 173 | -------------------------------------------------------------------------------- /ossync/sdk/oss_sample.py: -------------------------------------------------------------------------------- 1 | #coding=utf8 2 | import time 3 | from oss_xml_handler import * 4 | from oss_api import * 5 | 6 | HOST="storage.aliyun.com" 7 | ACCESS_ID = "" 8 | SECRET_ACCESS_KEY = "" 9 | #ACCESS_ID and SECRET_ACCESS_KEY should not be empty, please input correct one. 10 | 11 | if __name__ == "__main__": 12 | #init oss, get instance of oss 13 | if len(ACCESS_ID) == 0 or len(SECRET_ACCESS_KEY) == 0: 14 | print "Please make sure ACCESS_ID and SECRET_ACCESS_KEY are correct in ", __file__ , ", init are empty!" 15 | exit(0) 16 | oss = OssAPI(HOST, ACCESS_ID, SECRET_ACCESS_KEY) 17 | sep = "==============================" 18 | 19 | #sign_url_auth_with_expire_time(self, method, url, headers = {}, resource="/", timeout = 60): 20 | method = "GET" 21 | bucket = "test" + time.strftime("%Y-%b-%d%H-%M-%S").lower() 22 | object = "test_object" 23 | url = "http://" + HOST + "/oss/" + bucket + "/" + object 24 | headers = {} 25 | resource = "/" + bucket + "/" + object 26 | 27 | timeout = 60 28 | url_with_auth = oss.sign_url_auth_with_expire_time(method, url, headers, resource, timeout) 29 | print "after signature url is: ", url_with_auth 30 | print sep 31 | #put_bucket(self, bucket, acl='', headers = {}): 32 | acl = 'private' 33 | headers = {} 34 | res = oss.put_bucket(bucket, acl, headers) 35 | if (res.status / 100) == 2: 36 | print "put bucket ", bucket, "OK" 37 | else: 38 | print "put bucket ", bucket, "ERROR" 39 | print sep 40 | 41 | #get_service(self): 42 | res = oss.get_service() 43 | if (res.status / 100) == 2: 44 | body = res.read() 45 | h = GetServiceXml(body) 46 | print "bucket list size is: ", len(h.list()) 47 | print "bucket list is: " 48 | for i in h.list(): 49 | print i 50 | else: 51 | print res.status 52 | print sep 53 | 54 | #put_object_from_string(self, bucket, object, input_content, content_type=DefaultContentType, headers = {}): 55 | object = "object_test" 56 | input_content = "hello, OSS" 57 | content_type = "text/HTML" 58 | headers = {} 59 | res = oss.put_object_from_string(bucket, object, input_content, content_type, headers) 60 | if (res.status / 100) == 2: 61 | print "put_object_from_string OK" 62 | else: 63 | print "put_object_from_string ERROR" 64 | print sep 65 | 66 | #put_object_from_file(self, bucket, object, filename, content_type=DefaultContentType, headers = {}): 67 | object = "object_test" 68 | filename = __file__ 69 | content_type = "text/HTML" 70 | headers = {} 71 | res = oss.put_object_from_file(bucket, object, filename, content_type, headers) 72 | if (res.status / 100) == 2: 73 | print "put_object_from_file OK" 74 | else: 75 | print "put_object_from_file ERROR" 76 | print sep 77 | 78 | #put_object_from_fp(self, bucket, object, fp, content_type=DefaultContentType, headers = {}): 79 | object = "object_test" 80 | filename = __file__ 81 | content_type = "text/HTML" 82 | headers = {} 83 | 84 | fp = open(filename, 'rb') 85 | res = oss.put_object_from_fp(bucket, object, fp, content_type, headers) 86 | fp.close() 87 | if (res.status / 100) == 2: 88 | print "put_object_from_fp OK" 89 | else: 90 | print "put_object_from_fp ERROR" 91 | print sep 92 | 93 | #get_object(self, bucket, object, headers = {}): 94 | object = "object_test" 95 | headers = {} 96 | 97 | res = oss.get_object(bucket, object, headers) 98 | if (res.status / 100) == 2: 99 | print "get_object OK" 100 | else: 101 | print "get_object ERROR" 102 | print sep 103 | 104 | #get_object_to_file(self, bucket, object, filename, headers = {}): 105 | object = "object_test" 106 | headers = {} 107 | filename = "get_object_test_file" 108 | 109 | res = oss.get_object_to_file(bucket, object, filename, headers) 110 | if (res.status / 100) == 2: 111 | print "get_object_to_file OK" 112 | else: 113 | print "get_object_to_file ERROR" 114 | print sep 115 | 116 | #head_object(self, bucket, object, headers = {}): 117 | object = "object_test" 118 | headers = {} 119 | res = oss.head_object(bucket, object, headers) 120 | if (res.status / 100) == 2: 121 | print "head_object OK" 122 | header_map = convert_header2map(res.getheaders()) 123 | content_len = safe_get_element("content-length", header_map) 124 | etag = safe_get_element("etag", header_map).upper() 125 | print "content length is:", content_len 126 | print "ETag is: ", etag 127 | 128 | else: 129 | print "head_object ERROR" 130 | print sep 131 | 132 | #get_bucket_acl(self, bucket): 133 | res = oss.get_bucket_acl(bucket) 134 | if (res.status / 100) == 2: 135 | body = res.read() 136 | h = GetBucketAclXml(body) 137 | print "bucket acl is:", h.grant 138 | else: 139 | print "get bucket acl ERROR" 140 | print sep 141 | 142 | #get_bucket(self, bucket, prefix='', marker='', delimiter='', maxkeys='', headers = {}): 143 | prefix = "" 144 | marker = "" 145 | delimiter = "/" 146 | maxkeys = "100" 147 | headers = {} 148 | res = oss.get_bucket(bucket, prefix, marker, delimiter, maxkeys, headers) 149 | if (res.status / 100) == 2: 150 | body = res.read() 151 | h = GetBucketXml(body) 152 | (file_list, common_list) = h.list() 153 | print "object list is:" 154 | for i in file_list: 155 | print i 156 | print "common list is:" 157 | for i in common_list: 158 | print i 159 | print sep 160 | 161 | #upload_large_file(self, bucket, object, filename, thread_num = 10, max_part_num = 1000): 162 | res = oss.upload_large_file(bucket, object, __file__) 163 | if (res.status / 100) == 2: 164 | print "upload_large_file OK" 165 | else: 166 | print "upload_large_file ERROR" 167 | 168 | print sep 169 | 170 | #get_object_group_index(self, bucket, object, headers = {}) 171 | res = oss.get_object_group_index(bucket, object) 172 | if (res.status / 100) == 2: 173 | print "get_object_group_index OK" 174 | body = res.read() 175 | h = GetObjectGroupIndexXml(body) 176 | for i in h.list(): 177 | print "object group part msg:", i 178 | else: 179 | print "get_object_group_index ERROR" 180 | 181 | res = oss.get_object_group_index(bucket, object) 182 | if res.status == 200: 183 | body = res.read() 184 | h = GetObjectGroupIndexXml(body) 185 | object_group_index = h.list() 186 | for i in object_group_index: 187 | if len(i) == 4 and len(i[1]) > 0: 188 | part_name = i[1].strip() 189 | res = oss.delete_object(bucket, part_name) 190 | if res.status != 204: 191 | print "delete part ", part_name, " in bucket:", bucket, " failed!" 192 | else: 193 | print "delete part ", part_name, " in bucket:", bucket, " ok" 194 | print sep 195 | 196 | #delete_object(self, bucket, object, headers = {}): 197 | object = "object_test" 198 | headers = {} 199 | res = oss.delete_object(bucket, object, headers) 200 | if (res.status / 100) == 2: 201 | print "delete_object OK" 202 | else: 203 | print "delete_object ERROR" 204 | print sep 205 | 206 | #delete_bucket(self, bucket): 207 | res = oss.delete_bucket(bucket) 208 | if (res.status / 100) == 2: 209 | print "delete bucket ", bucket, "OK" 210 | else: 211 | print "delete bucket ", bucket, "ERROR" 212 | 213 | print sep 214 | 215 | 216 | -------------------------------------------------------------------------------- /ossync/sdk/oss_util.py: -------------------------------------------------------------------------------- 1 | #coding=utf8 2 | import urllib 3 | import base64 4 | import hmac 5 | import time 6 | from hashlib import sha1 as sha 7 | import os 8 | import md5 9 | import StringIO 10 | from oss_xml_handler import * 11 | from threading import Thread 12 | 13 | DEBUG = False 14 | 15 | self_define_header_prefix = "x-oss-" 16 | 17 | class User: 18 | def __init__(self, user_name = "", access_id = "", secret_access_key = ""): 19 | self.user_name = user_name 20 | self.access_id = access_id 21 | self.secret_access_key = secret_access_key 22 | 23 | def show(self): 24 | print (('user_name:(%s) access_id:(%s) secret_access_key(%s)')) % (self.user_name, 25 | self.access_id, self.secret_access_key) 26 | 27 | 28 | def _format_header(headers = {}): 29 | ''' 30 | format the headers that self define 31 | convert the self define headers to lower. 32 | ''' 33 | tmp_headers = {} 34 | for k in headers.keys(): 35 | if k.lower().startswith(self_define_header_prefix): 36 | k_lower = k.lower() 37 | tmp_headers[k_lower] = headers[k] 38 | else: 39 | tmp_headers[k] = headers[k] 40 | return tmp_headers 41 | 42 | def get_assign(secret_access_key, method, headers = {}, resource="/", result = []): 43 | ''' 44 | Create the authorization for OSS based on header input. 45 | You should put it into "Authorization" parameter of header. 46 | ''' 47 | content_md5 = "" 48 | content_type = "" 49 | date = "" 50 | canonicalized_oss_headers = "" 51 | if DEBUG: 52 | print "secret_access_key", secret_access_key 53 | content_md5 = safe_get_element('Content-Md5', headers) 54 | content_type = safe_get_element('Content-Type', headers) 55 | date = safe_get_element('Date', headers) 56 | canonicalized_resource = resource 57 | tmp_headers = _format_header(headers) 58 | if len(tmp_headers) > 0: 59 | x_header_list = tmp_headers.keys() 60 | x_header_list.sort() 61 | for k in x_header_list: 62 | if k.startswith(self_define_header_prefix): 63 | canonicalized_oss_headers += k + ":" + tmp_headers[k] + "\n" 64 | 65 | string_to_sign = method + "\n" + content_md5 + "\n" + content_type + "\n" + date + "\n" + canonicalized_oss_headers + canonicalized_resource; 66 | result.append(string_to_sign) 67 | if DEBUG: 68 | print "string_to_sign", string_to_sign, "string_to_sign_size", len(string_to_sign) 69 | 70 | h = hmac.new(secret_access_key, string_to_sign, sha) 71 | 72 | return base64.encodestring(h.digest()).strip() 73 | 74 | def convert_header2map(header_list): 75 | header_map = {} 76 | for (a, b) in header_list: 77 | header_map[a] = b 78 | return header_map 79 | 80 | def safe_get_element(name, container): 81 | if name in container: 82 | return container[name] 83 | else: 84 | return "" 85 | 86 | def append_param(url, params): 87 | l = [] 88 | for k,v in params.items(): 89 | k = k.replace('_', '-') 90 | if k == 'maxkeys': 91 | k = 'max-keys' 92 | if isinstance(v, unicode): 93 | v = v.encode('utf-8') 94 | if v is not None and v != '': 95 | l.append('%s=%s' % (urllib.quote(k), urllib.quote(str(v)))) 96 | elif k == 'acl': 97 | l.append('%s' % (urllib.quote(k))) 98 | if len(l): 99 | url = url + '?' + '&'.join(l) 100 | return url 101 | 102 | def create_object_group_msg_xml(part_msg_list = []): 103 | xml_string = r'' 104 | for part in part_msg_list: 105 | if len(part) >= 3: 106 | if isinstance(part[1], unicode): 107 | file_path = part[1].encode('utf-8') 108 | else: 109 | file_path = part[1] 110 | xml_string += r'' 111 | xml_string += r'' + str(part[0]) + r'' 112 | xml_string += r'' + str(file_path) + r'' 113 | xml_string += r'"' + str(part[2]).upper() + r'"' 114 | xml_string += r'' 115 | else: 116 | print "the ", part, " in part_msg_list is not as expected!" 117 | return "" 118 | xml_string += r'' 119 | 120 | return xml_string 121 | 122 | def delete_all_parts_of_object_group(oss, bucket, object_group_name): 123 | res = oss.get_object_group_index(bucket, object_group_name) 124 | if res.status == 200: 125 | body = res.read() 126 | h = GetObjectGroupIndexXml(body) 127 | object_group_index = h.list() 128 | for i in object_group_index: 129 | if len(i) == 4 and len(i[1]) > 0: 130 | part_name = i[1].strip() 131 | res = oss.delete_object(bucket, part_name) 132 | if res.status != 204: 133 | print "delete part ", part_name, " in bucket:", bucket, " failed!" 134 | return False 135 | else: 136 | print "delete part ", part_name, " in bucket:", bucket, " ok" 137 | else: 138 | return False 139 | 140 | return True; 141 | 142 | def split_large_file(file_path, object_prefix = "", max_part_num = 1000, part_size = 10 * 1024 * 1024, buffer_size = 10 * 1024): 143 | parts_list = [] 144 | 145 | if os.path.isfile(file_path): 146 | file_size = os.path.getsize(file_path) 147 | 148 | if file_size > part_size * max_part_num: 149 | part_size = (file_size + max_part_num - file_size % max_part_num) / max_part_num 150 | 151 | part_order = 1 152 | fp = open(file_path, 'rb') 153 | fp.seek(os.SEEK_SET) 154 | 155 | total_split_len = 0 156 | part_num = file_size / part_size 157 | if file_size % part_size != 0: 158 | part_num += 1 159 | 160 | for i in range(0, part_num): 161 | left_len = part_size 162 | real_part_size = 0 163 | m = md5.new() 164 | offset = part_size * i 165 | while True: 166 | read_size = 0 167 | if left_len <= 0: 168 | break 169 | elif left_len < buffer_size: 170 | read_size = left_len 171 | else: 172 | read_size = buffer_size 173 | 174 | buffer_content = fp.read(read_size) 175 | m.update(buffer_content) 176 | real_part_size += len(buffer_content) 177 | 178 | left_len = left_len - read_size 179 | 180 | md5sum = m.hexdigest() 181 | 182 | temp_file_name = os.path.basename(file_path) + "_" + str(part_order) 183 | if len(object_prefix) == 0: 184 | file_name = sum_string(temp_file_name) + "_" + temp_file_name 185 | else: 186 | file_name = object_prefix + "/" + sum_string(temp_file_name) + "_" + temp_file_name 187 | part_msg = (part_order, file_name, md5sum, real_part_size, offset) 188 | total_split_len += real_part_size 189 | parts_list.append(part_msg) 190 | part_order += 1 191 | 192 | fp.close() 193 | else: 194 | print "ERROR! No file: ", file_path, ", please check." 195 | 196 | return parts_list 197 | 198 | def sumfile(fobj): 199 | '''Returns an md5 hash for an object with read() method.''' 200 | m = md5.new() 201 | while True: 202 | d = fobj.read(8096) 203 | if not d: 204 | break 205 | m.update(d) 206 | return m.hexdigest() 207 | 208 | def md5sum(fname): 209 | '''Returns an md5 hash for file fname, or stdin if fname is "-".''' 210 | if fname == '-': 211 | ret = sumfile(sys.stdin) 212 | else: 213 | try: 214 | f = file(fname, 'rb') 215 | except: 216 | return 'Failed to open file' 217 | ret = sumfile(f) 218 | f.close() 219 | return ret 220 | 221 | def md5sum2(filename, offset = 0, partsize = 0): 222 | m = md5.new() 223 | fp = open(filename, 'rb') 224 | if offset > os.path.getsize(filename): 225 | fp.seek(os.SEEK_SET, os.SEEK_END) 226 | else: 227 | fp.seek(offset) 228 | 229 | left_len = partsize 230 | BufferSize = 8 * 1024 231 | while True: 232 | if left_len <= 0: 233 | break 234 | elif left_len < BufferSize: 235 | buffer_content = fp.read(left_len) 236 | else: 237 | buffer_content = fp.read(BufferSize) 238 | m.update(buffer_content) 239 | 240 | left_len = left_len - len(buffer_content) 241 | md5sum = m.hexdigest() 242 | return md5sum 243 | 244 | def sum_string(content): 245 | f = StringIO.StringIO(content) 246 | md5sum = sumfile(f) 247 | f.close() 248 | return md5sum 249 | 250 | class PutObjectGroupWorker(Thread): 251 | def __init__(self, oss, bucket, file_path, part_msg_list): 252 | Thread.__init__(self) 253 | self.oss = oss 254 | self.bucket = bucket 255 | self.part_msg_list = part_msg_list 256 | self.file_path = file_path 257 | 258 | def run(self): 259 | for part in self.part_msg_list: 260 | if len(part) == 5: 261 | bucket = self.bucket 262 | file_name = part[1] 263 | object_name = file_name 264 | res = self.oss.head_object(bucket, object_name) 265 | if res.status == 200: 266 | header_map = convert_header2map(res.getheaders()) 267 | etag = safe_get_element("etag", header_map) 268 | md5 = part[2] 269 | if etag.replace('"', "").upper() == md5.upper(): 270 | continue 271 | 272 | partsize = part[3] 273 | offset = part[4] 274 | res = self.oss.put_object_from_file_given_pos(bucket, object_name, self.file_path, offset, partsize) 275 | if res.status != 200: 276 | print "upload ", file_name, "failed!"," ret is:", res.status 277 | print "headers", res.getheaders() 278 | else: 279 | print "ERROR! part", part , " is not as expected!" 280 | 281 | class GetAllObjects: 282 | def __init__(self): 283 | self.object_list = [] 284 | 285 | def get_object_in_bucket(self, oss, bucket="", marker="", prefix=""): 286 | object_list = [] 287 | res = oss.get_bucket(bucket, prefix, marker) 288 | body = res.read() 289 | hh = GetBucketXml(body) 290 | (fl, pl) = hh.list() 291 | if len(fl) != 0: 292 | for i in fl: 293 | if isinstance(i[0], unicode): 294 | object = i[0].encode('utf-8') 295 | object_list.append(object) 296 | 297 | if hh.is_truncated: 298 | marker = hh.nextmarker 299 | return (object_list, marker) 300 | 301 | 302 | def get_all_object_in_bucket(self, oss, bucket="", marker="", prefix=""): 303 | marker2 = "" 304 | while True: 305 | (object_list, marker) = self.get_object_in_bucket(oss, bucket, marker2, prefix) 306 | marker2 = marker 307 | if len(object_list) != 0: 308 | self.object_list.extend(object_list) 309 | 310 | if len(marker) == 0: 311 | break 312 | 313 | def clear_all_objects_in_bucket(oss_instance, bucket): 314 | ''' 315 | it will clean all objects in bucket, after that, it will delete this bucket. 316 | 317 | example: 318 | from oss_api import * 319 | host = "" 320 | id = "" 321 | key = "" 322 | oss_instance = OssAPI(host, id, key) 323 | bucket = "leopublicreadprivatewrite" 324 | if clear_all_objects_in_bucket(oss_instance, bucket): 325 | print "clean OK" 326 | else: 327 | print "clean Fail" 328 | ''' 329 | b = GetAllObjects() 330 | b.get_all_object_in_bucket(oss_instance, bucket) 331 | for i in b.object_list: 332 | res = oss_instance.delete_object(bucket, i) 333 | if (res.status / 100 != 2): 334 | print "clear_all_objects_in_bucket: delete object fail, ret is:", res.status, "object is: ", i 335 | return False 336 | else: 337 | print "clear_all_objects_in_bucket: delete object ok", "object is: ", i 338 | res = oss_instance.delete_bucket(bucket) 339 | if (res.status / 100 != 2 and res.status != 404): 340 | print "clear_all_objects_in_bucket: delete bucket fail, ret is:", res.status 341 | return False 342 | return True 343 | 344 | if __name__ == '__main__': 345 | pass 346 | -------------------------------------------------------------------------------- /ossync/sdk/oss_xml_handler.py: -------------------------------------------------------------------------------- 1 | #coding=utf8 2 | from xml.dom import minidom 3 | 4 | def get_tag_text(element, tag): 5 | nodes = element.getElementsByTagName(tag) 6 | if len(nodes) == 0: 7 | return "" 8 | else: 9 | node = nodes[0] 10 | rc = "" 11 | for node in node.childNodes: 12 | if node.nodeType in ( node.TEXT_NODE, node.CDATA_SECTION_NODE): 13 | rc = rc + node.data 14 | return rc 15 | 16 | class ErrorXml: 17 | def __init__(self, xml_string): 18 | self.xml = minidom.parseString(xml_string) 19 | self.code = get_tag_text(self.xml, 'Code') 20 | self.msg = get_tag_text(self.xml, 'Message') 21 | self.resource = get_tag_text(self.xml, 'Resource') 22 | self.request_id = get_tag_text(self.xml, 'RequestId') 23 | self.host_id = get_tag_text(self.xml, 'HostId') 24 | 25 | def show(self): 26 | print "Code: %s\nMessage: %s\nResource: %s\nRequestId: %s \nHostId: %s" % (self.code, self.msg, self.resource, self.request_id, self.host_id) 27 | 28 | class Owner: 29 | def __init__(self, xml_element): 30 | self.element = xml_element 31 | self.id = get_tag_text(self.element, "ID") 32 | self.display_name = get_tag_text(self.element, "DisplayName") 33 | 34 | def show(self): 35 | print "ID: %s\nDisplayName: %s" % (self.id, self.display_name) 36 | 37 | class Bucket: 38 | def __init__(self, xml_element): 39 | self.element = xml_element 40 | self.name = get_tag_text(self.element, "Name") 41 | self.creation_date = get_tag_text(self.element, "CreationDate") 42 | 43 | def show(self): 44 | print "Name: %s\nCreationDate: %s" % (self.name, self.creation_date) 45 | 46 | class GetServiceXml: 47 | def __init__(self, xml_string): 48 | self.xml = minidom.parseString(xml_string) 49 | self.owner = Owner(self.xml.getElementsByTagName('Owner')[0]) 50 | self.buckets = self.xml.getElementsByTagName('Bucket') 51 | self.bucket_list = [] 52 | for b in self.buckets: 53 | self.bucket_list.append(Bucket(b)) 54 | 55 | def show(self): 56 | print "Owner:" 57 | self.owner.show() 58 | print "\nBucket list:" 59 | for b in self.bucket_list: 60 | b.show() 61 | print "" 62 | 63 | def list(self): 64 | bl = [] 65 | for b in self.bucket_list: 66 | bl.append((b.name, b.creation_date)) 67 | return bl 68 | 69 | class Content: 70 | def __init__(self, xml_element): 71 | self.element = xml_element 72 | self.key = get_tag_text(self.element, "Key") 73 | self.last_modified = get_tag_text(self.element, "LastModified") 74 | self.etag = get_tag_text(self.element, "ETag") 75 | self.size = get_tag_text(self.element, "Size") 76 | self.owner = Owner(self.element.getElementsByTagName('Owner')[0]) 77 | self.storage_class = get_tag_text(self.element, "StorageClass") 78 | 79 | def show(self): 80 | print "Key: %s\nLastModified: %s\nETag: %s\nSize: %s\nStorageClass: %s" % (self.key, self.last_modified, self.etag, self.size, self.storage_class) 81 | self.owner.show() 82 | 83 | class Part: 84 | def __init__(self, xml_element): 85 | self.element = xml_element 86 | self.part_num = get_tag_text(self.element, "PartNumber") 87 | self.object_name = get_tag_text(self.element, "PartName") 88 | self.object_size = get_tag_text(self.element, "PartSize") 89 | self.etag = get_tag_text(self.element, "ETag") 90 | 91 | def show(self): 92 | print "PartNumber: %s\nPartName: %s\nPartSize: %s\nETag: %s\n" % (self.part_num, self.object_name, self.object_size, self.etag) 93 | 94 | class PostObjectGroupXml: 95 | def __init__(self, xml_string): 96 | self.xml = minidom.parseString(xml_string) 97 | self.bucket = get_tag_text(self.xml, 'Bucket') 98 | self.key = get_tag_text(self.xml, 'Key') 99 | self.size = get_tag_text(self.xml, 'Size') 100 | self.etag = get_tag_text(self.xml, "ETag") 101 | 102 | def show(self): 103 | print "Post Object Group, Bucket: %s\nKey: %s\nSize: %s\nETag: %s" % (self.bucket, self.key, self.size, self.etag) 104 | 105 | class GetObjectGroupIndexXml: 106 | def __init__(self, xml_string): 107 | self.xml = minidom.parseString(xml_string) 108 | self.bucket = get_tag_text(self.xml, 'Bucket') 109 | self.key = get_tag_text(self.xml, 'Key') 110 | self.etag = get_tag_text(self.xml, 'Etag') 111 | self.file_length = get_tag_text(self.xml, 'FileLength') 112 | self.index_list = [] 113 | index_lists = self.xml.getElementsByTagName('Part') 114 | for i in index_lists: 115 | self.index_list.append(Part(i)) 116 | 117 | def list(self): 118 | index_list = [] 119 | for i in self.index_list: 120 | index_list.append((i.part_num, i.object_name, i.object_size, i.etag)) 121 | return index_list 122 | 123 | def show(self): 124 | print "Bucket: %s\nObject: %s\nEtag: %s\nObjectSize: %s" % (self.bucket, self.key, self.etag, self.file_length) 125 | print "\nPart list:" 126 | for p in self.index_list: 127 | p.show() 128 | 129 | class GetBucketXml: 130 | def __init__(self, xml_string): 131 | self.xml = minidom.parseString(xml_string) 132 | self.name = get_tag_text(self.xml, 'Name') 133 | self.prefix = get_tag_text(self.xml, 'Prefix') 134 | self.marker = get_tag_text(self.xml, 'Marker') 135 | self.nextmarker = get_tag_text(self.xml, 'NextMarker') 136 | self.maxkeys = get_tag_text(self.xml, 'MaxKeys') 137 | self.delimiter = get_tag_text(self.xml, 'Delimiter') 138 | self.is_truncated = get_tag_text(self.xml, 'IsTruncated') 139 | 140 | self.prefix_list = [] 141 | prefixes = self.xml.getElementsByTagName('CommonPrefixes') 142 | for p in prefixes: 143 | tag_txt = get_tag_text(p, "Prefix") 144 | self.prefix_list.append(tag_txt) 145 | 146 | self.content_list = [] 147 | contents = self.xml.getElementsByTagName('Contents') 148 | for c in contents: 149 | self.content_list.append(Content(c)) 150 | 151 | def show(self): 152 | print "Name: %s\nPrefix: %s\nMarker: %s\nNextMarker: %s\nMaxKeys: %s\nDelimiter: %s\nIsTruncated: %s" % (self.name, self.prefix, self.marker, self.nextmarker, self.maxkeys, self.delimiter, self.is_truncated) 153 | print "\nPrefix list:" 154 | for p in self.prefix_list: 155 | print p 156 | print "\nContent list:" 157 | for c in self.content_list: 158 | c.show() 159 | print "" 160 | 161 | def list(self): 162 | cl = [] 163 | pl = [] 164 | for c in self.content_list: 165 | cl.append((c.key, c.last_modified, c.etag, c.size, c.owner.id, c.owner.display_name, c.storage_class)) 166 | for p in self.prefix_list: 167 | pl.append(p) 168 | 169 | return (cl, pl) 170 | 171 | class GetBucketAclXml: 172 | def __init__(self, xml_string): 173 | self.xml = minidom.parseString(xml_string) 174 | if len(self.xml.getElementsByTagName('Owner')) != 0: 175 | self.owner = Owner(self.xml.getElementsByTagName('Owner')[0]) 176 | else: 177 | self.owner = "" 178 | self.grant = get_tag_text(self.xml, 'Grant') 179 | 180 | def show(self): 181 | print "Owner Name: %s\nOwner ID: %s\nGrant: %s" % (self.owner.id, self.owner.display_name, self.grant) 182 | 183 | def test_get_bucket_xml(): 184 | body = "sweet-memoryIMGIMG_01000falseIMG_2744.JPG2011-03-04T06:20:37.000Z"a56047f218618a43a9b1c2dca2d8c592"2207782cfd76976de6f4c6f4e05fcd02680c4ca619428123681589efcb203f29dce924sanbo_ustcSTANDARDIMG_2745.JPG2011-03-04T06:20:39.000Z"511c0b52911bcd667338103c385741af"2446122cfd76976de6f4c6f4e05fcd02680c4ca619428123681589efcb203f29dce924sanbo_ustcSTANDARDIMG_3344.JPG2011-03-04T06:20:48.000Z"4ea11d796ecc742b216864dcf5dfd193"2292112cfd76976de6f4c6f4e05fcd02680c4ca619428123681589efcb203f29dce924sanbo_ustcSTANDARDIMG_3387.JPG2011-03-04T06:20:53.000Z"c32b5568ae4fb0a3421f0daba25ecfd4"4600622cfd76976de6f4c6f4e05fcd02680c4ca619428123681589efcb203f29dce924sanbo_ustcSTANDARDIMG_3420.JPG2011-03-04T06:20:25.000Z"edf010d2a8a4877ce0362b245fcc963b"1749732cfd76976de6f4c6f4e05fcd02680c4ca619428123681589efcb203f29dce924sanbo_ustcSTANDARD中文.case2011-03-04T06:20:26.000Z"7fd64eec21799ef048ed827cf6098f06"2081342cfd76976de6f4c6f4e05fcd02680c4ca619428123681589efcb203f29dce924sanbo_ustcSTANDARD" 185 | h = GetBucketXml(body) 186 | h.show() 187 | (fl, pl) = h.list() 188 | print "\nfile_list: ", fl 189 | print "prefix list: ", pl 190 | 191 | def test_get_service_xml(): 192 | body = "2cfd76976de6f4c6f4e05fcd02680c4ca619428123681589efcb203f29dce924sanbo_ustc360buy2011-03-04T09:25:37.000Zaliyun-test-test2011-04-11T12:24:06.000Zirecoffee2011-03-04T06:14:56.000Zsweet-memory2011-04-12T11:56:04.000Z" 193 | h = GetServiceXml(body) 194 | h.show() 195 | print "\nbucket list: ", h.list() 196 | 197 | def test_get_bucket_acl_xml(): 198 | body = '61155b1e39dbca1d0d0f3c7faa32d9e8e9a90a9cd86edbd27d8eed5d0ad8ce82megjianpublic-read-write' 199 | h = GetBucketAclXml(body) 200 | h.show() 201 | 202 | def test_get_object_group_xml(): 203 | body = 'ut_test_post_object_group ut_test_post_object_group "91E8503F4DA1324E28434AA6B6E20D15"1073741824 1 4d37380c7149508bedf78dc7c5c68f55_test_post_object_group.txt_110485760"A957A9F1EF44ED7D40CD5C738D113509"27aa26b8da263589e875d179b87642691_test_post_object_group.txt_210485760"A957A9F1EF44ED7D40CD5C738D113509"328b0c8a9bd69469f76cd102d6e1b0f03_test_post_object_group.txt_310485760"A957A9F1EF44ED7D40CD5C738D113509"' 204 | h = GetObjectGroupIndexXml(body) 205 | h.show() 206 | 207 | if __name__ == "__main__": 208 | test_get_bucket_xml() 209 | test_get_service_xml() 210 | test_get_bucket_acl_xml() 211 | test_get_object_group_xml() 212 | -------------------------------------------------------------------------------- /ossync/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | -------------------------------------------------------------------------------- /ossync/tests/api_test.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import sys, time, md5 4 | from xml.dom import minidom 5 | from ..sdk import oss_api 6 | from ..sdk import oss_xml_handler 7 | import hashlib 8 | import unittest 9 | 10 | class TestOssApi(unittest.TestCase): 11 | 12 | def setUp(self): 13 | HOST = "storage.aliyun.com" 14 | ACCESS_ID = "ACSAu8DDXiZ4e23d" 15 | SECRET_ACCESS_KEY = "xyZ9mBgXtG" 16 | self.GetBufferSize = 1024*1024*10 17 | self.oss = oss_api.OssAPI(HOST, ACCESS_ID, SECRET_ACCESS_KEY) 18 | if len(ACCESS_ID) == 0 or len(SECRET_ACCESS_KEY) == 0: 19 | print "Please set ACCESS_ID and SECRET_ACCESS_KEY" 20 | exit(0) 21 | 22 | def tearDown(self): 23 | pass 24 | 25 | def testGetService(self): 26 | res = self.oss.get_service() 27 | if(res.status / 100 == 2): 28 | body = res.read() 29 | h = oss_xml_handler.GetServiceXml(body) 30 | print "buckect list size is: ", len(h.list()) 31 | print "buckect list is:" 32 | for i in h.list(): 33 | print i[0] 34 | else: 35 | print res.status 36 | self.assertEqual(res.status / 100, 2) 37 | 38 | def getObjectToFile(self): 39 | filename = 'favicon.ico' 40 | res = self.oss.get_object_to_file(bucket = 'dzdata', object = 'forum/favicon.ico', filename = filename, headers = {}) 41 | '''if(res.status / 100 == 2): 42 | print 'get object success' 43 | data = res.read(self.GetBufferSize) 44 | print data 45 | else: 46 | print 'get object failure' 47 | ''' 48 | self.assertEqual(res.status / 100, 2) 49 | 50 | def testGetObject(self): 51 | res = self.oss.get_object(bucket = 'dzdata', object = 'docs/mmmmm/favicon.ico', headers = {}) 52 | '''if(res.status / 100 == 2): 53 | print res.getheader('Etag') 54 | print res.read() 55 | else: 56 | print 'head object failure' 57 | ''' 58 | self.assertEqual(res.status / 100, 2) 59 | 60 | def testHeadObject(self): 61 | res = self.oss.head_object(bucket = 'dzdata', object = 'docs/mmmmm/favicon.ico', headers = {}) 62 | '''if(res.status / 100 == 2): 63 | print res.getheader('Etag') 64 | print res.msg 65 | print res.reason 66 | else: 67 | print 'head object failure' 68 | ''' 69 | self.assertEqual(res.status / 100, 2) 70 | 71 | ''' 72 | def testPutObjectFromString(self): 73 | somestr = "" 74 | # print self._calcStrMd5(somestr).upper() 75 | res = self.oss.put_object_from_string(bucket = 'dzdata', object = 'forum/xmun/', input_content = somestr, content_type = "text/plain", headers = {}) 76 | # print res.getheader('Etag') 77 | self.assertEqual(res.status / 100, 2) 78 | ''' 79 | 80 | def testDeleteObject(self): 81 | res = self.oss.delete_object('dzdata', 'docs/mmmmm/', {}) 82 | '''if(res.status == 200): 83 | print 'delete object success' 84 | else: 85 | print 'delete object failure' 86 | print res.status 87 | ''' 88 | print res.status 89 | self.assertEqual(res.status / 100, 2) 90 | 91 | def testPutObjectFromFile(self): 92 | filepath = '/Users/wuts/work/ossync/ossync/tests/api_test.py' 93 | hashstr = self._calcFileMd5(filepath).upper() 94 | # print hashstr 95 | res = self.oss.put_object_from_file(bucket = 'dzdata', object = 'ossync/tests/api_test.py', filename = filepath, content_type = "text/plain", headers = {}) 96 | # print res.getheader('Etag') 97 | '''if(res.status == 200): 98 | print res.getheader('Etag') 99 | print 'put object from string success' 100 | else: 101 | print 'put object from string failure' 102 | print res.status 103 | ''' 104 | self.assertEqual(res.status / 100, 2) 105 | 106 | 107 | def testGetBucket(self): 108 | result = [] 109 | bucket = 'privdata' 110 | delimiter = '/' 111 | prefix = 'images/' 112 | marker = '' 113 | headers = {} 114 | maxkeys = '' 115 | self._walk_bucket( bucket, prefix, marker, delimiter, maxkeys, headers, result) 116 | print result 117 | self.assertEqual(len(result) > 0, True) 118 | 119 | def testCreateBucket(self): 120 | bucket = 'myosdata' 121 | headers = {} 122 | acl = '' 123 | res = self.oss.create_bucket(bucket, acl, headers) 124 | self.assertEqual(res.status / 100, 2) 125 | 126 | def _walk_bucket(self, bucket, prefix, marker, delimiter, maxkeys, headers, result = []): 127 | res = self.oss.get_bucket(bucket, prefix, marker, delimiter, maxkeys, headers) 128 | if (res.status / 100) == 2: 129 | body = res.read() 130 | h = oss_xml_handler.GetBucketXml(body) 131 | (file_list, common_list) = h.list() 132 | if len(file_list) > 0: 133 | for item in file_list: 134 | result.append(item[0]) 135 | if len(common_list) > 0: 136 | for path in common_list: 137 | result.append(path) 138 | self._walk_bucket(bucket, path, marker, delimiter, maxkeys, headers, result) 139 | 140 | 141 | def _calcStrMd5(self, somestr): 142 | md5obj = hashlib.md5() 143 | md5obj.update(somestr) 144 | hashstr = md5obj.hexdigest() 145 | return hashstr 146 | 147 | def _calcFileMd5(self, filepath): 148 | with open(filepath, 'rb') as f: 149 | md5obj = hashlib.md5() 150 | md5obj.update(f.read()) 151 | hashstr = md5obj.hexdigest() 152 | return hashstr 153 | 154 | -------------------------------------------------------------------------------- /ossync/tests/filehash.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | import hashlib 4 | 5 | def calcMd5(filepath): 6 | with open(filepath, 'rb') as f: 7 | md5obj = hashlib.md5() 8 | md5obj.update(f.read()) 9 | hashstr = md5obj.hexdigest() 10 | return hashstr 11 | 12 | if __name__ == '__main__': 13 | filepath = '/Users/wuts/work/ossync/ossync/tests/api_test.py' 14 | print calcMd5(filepath) 15 | -------------------------------------------------------------------------------- /ossync/tests/helper_test.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import os, sys 4 | import unittest 5 | from ..lib import helper 6 | 7 | class TestHelper(unittest.TestCase): 8 | 9 | def setUp(self): 10 | pass 11 | 12 | def tearDown(self): 13 | pass 14 | 15 | def testWalkFiles(self): 16 | root = '/Volumes/Macintosh HD 2/software' 17 | files = list(helper.walk_files(root, yield_folders = True)) 18 | self.assertEqual(len(files) > 0, True) 19 | 20 | def testCalcFileMd5(self): 21 | filepath = '/Users/wuts/work/ossync/ossync/tests/api_test.py' 22 | hashstr = helper.calc_file_md5(filepath) 23 | print hashstr 24 | self.assertEqual(hashstr.upper(), 'ACC853492102F896787C2B9AAB3C766C') 25 | 26 | 27 | -------------------------------------------------------------------------------- /ossync/tests/path_test.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import os, sys 4 | def cur_dir(): 5 | path = sys.path[0] 6 | print path 7 | if os.path.isdir(path): 8 | return path 9 | elif os.path.isfile(path): 10 | return os.path.dirname(path) 11 | 12 | if __name__ == '__main__': 13 | print cur_dir() 14 | -------------------------------------------------------------------------------- /ossync/tests/pyinotify_test.py: -------------------------------------------------------------------------------- 1 | # coding:utf-8 2 | 3 | import os 4 | import pyinotify 5 | 6 | class EventHandler(pyinotify.ProcessEvent): 7 | """事件处理""" 8 | def process_IN_CREATE(self, event): 9 | print "Create file: %s " % os.path.join(event.path,event.name) 10 | 11 | def process_IN_DELETE(self, event): 12 | print "Delete file: %s " % os.path.join(event.path,event.name) 13 | 14 | def process_IN_MODIFY(self, event): 15 | print "Modify file: %s " % os.path.join(event.path,event.name) 16 | 17 | def FSMonitor(path = '.'): 18 | wm = pyinotify.WatchManager() 19 | mask = pyinotify.IN_DELETE | pyinotify.IN_CREATE | pyinotify.IN_MODIFY 20 | notifier = pyinotify.Notifier(wm, EventHandler()) 21 | wm.add_watch(path, mask, rec = True, auto_add = True) 22 | print 'now starting monitor %s'%(path) 23 | while True: 24 | try: 25 | notifier.process_events() 26 | if notifier.check_events(): 27 | notifier.read_events() 28 | except KeyboardInterrupt: 29 | notifier.stop() 30 | break 31 | 32 | if __name__ == "__main__": 33 | FSMonitor() -------------------------------------------------------------------------------- /ossync/tests/queue_model_test.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import sys 4 | from ..lib import queue_model 5 | import hashlib 6 | import unittest 7 | 8 | class TestQueueModel(unittest.TestCase): 9 | 10 | def setUp(self): 11 | dbpath = 'db/ossync.db' 12 | self.queue_model = queue_model.QueueModel(dbpath) 13 | self.queue_model.open() 14 | 15 | def tearDown(self): 16 | self.queue_model.close() 17 | 18 | def testSave(self): 19 | data = { 20 | "root": '/Users/wuts/newwork/ossync', 21 | "relpath": 'docs/OSS_API.pdf', 22 | "bucket": 'dzdata', 23 | "action": 'C', 24 | "status": 1, 25 | "retries": 0, 26 | } 27 | m = hashlib.md5() 28 | m.update(data['root'] + data['relpath'] + data['bucket']) 29 | hashcode = m.hexdigest() 30 | self.queue_model.save(data) 31 | res = self.queue_model.get(hashcode) 32 | self.assertNotEqual(res, None) 33 | 34 | def testGet(self): 35 | res = self.queue_model.get('e94bdba1f90e768442f2fa6d036428db') 36 | # print res['root'] 37 | self.assertNotEqual(res, None) 38 | 39 | def testFindAll(self): 40 | res = self.queue_model.find_all(status = 1) 41 | # print res 42 | self.assertNotEqual(res, None) 43 | 44 | def testUpdateStatus(self): 45 | self.queue_model.update_status('e94bdba1f90e768442f2fa6d036428db', 1) 46 | res = self.queue_model.get('e94bdba1f90e768442f2fa6d036428db') 47 | self.assertEqual(res['status'], 1) 48 | 49 | def testUpdateAction(self): 50 | self.queue_model.update_action('e94bdba1f90e768442f2fa6d036428db', 'M') 51 | res = self.queue_model.get('e94bdba1f90e768442f2fa6d036428db') 52 | self.assertEqual(res['action'], 'M') 53 | 54 | def testUpdateRetries(self): 55 | self.queue_model.update_retries('e94bdba1f90e768442f2fa6d036428db', 2) 56 | res = self.queue_model.get('e94bdba1f90e768442f2fa6d036428db') 57 | self.assertEqual(res['retries'], 2) 58 | 59 | """ 60 | def testDelete(self): 61 | self.queue_model.delete('b875fa0086d8af47a3a0126c91d6133f') 62 | res = self.queue_model.get('b875fa0086d8af47a3a0126c91d6133f') 63 | self.assertEqual(res, None) 64 | """ 65 | 66 | 67 | -------------------------------------------------------------------------------- /ossync/tests/walkdir_test.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import os, sys 4 | from os.path import * 5 | from Queue import * 6 | sys.path.append("/Users/wuts/src/python/ossync/lib") 7 | from walkdir import walk_files 8 | files = list(walk_files(root = "/Volumes/Macintosh HD 2/software", yield_folders = True)) 9 | queue = Queue(100) 10 | print queue.qsize() 11 | for path in files: 12 | try: 13 | queue.put(path, block = True, timeout = 1) 14 | except Full as e: 15 | print e 16 | break 17 | print "get queue:" 18 | while not queue.empty(): 19 | item = queue.get(block = True, timeout = 1) 20 | print item -------------------------------------------------------------------------------- /queue_thread.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Copyright (c) 2012 Wu Tangsheng(lanbaba) 4 | 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | 12 | # The above copyright notice and this permission notice shall be included in 13 | # all copies or substantial portions of the Software. 14 | 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | # THE SOFTWARE. 22 | 23 | import os, threading, logging 24 | import os.path 25 | from Queue import * 26 | import hashlib 27 | from ossync.lib import helper 28 | from ossync.lib import queue_model 29 | 30 | class QueueThread(threading.Thread): 31 | 32 | """ 此线程的作用是将bucket,root, path压入要上传的队列，队列元素格式： 33 | "bucket::root::relpath::action::life" 34 | 其中action表示文件是新建还是修改还是删除;life表示重入次数 35 | """ 36 | def __init__(self, bucket, dirs, queue, *args, **kwargs): 37 | threading.Thread.__init__(self, *args, **kwargs) 38 | self.bucket = bucket 39 | self.queue = queue 40 | self.dirs = dirs 41 | 42 | self._terminate = False 43 | self.logger = logging.getLogger('app') 44 | dbpath = 'db/ossync.db' 45 | self.qm = queue_model.QueueModel(dbpath) 46 | 47 | def terminate(self): 48 | self._terminate = True 49 | 50 | def is_el_queued(self, hashcode): 51 | row = self.qm.get(hashcode) 52 | if row: 53 | return True 54 | return False 55 | 56 | def run(self): 57 | files = {} 58 | for d in self.dirs: 59 | files[d] = list(helper.walk_files(os.path.normpath(d), yield_folders = True)) 60 | if len(files) > 0: 61 | self.qm.open() 62 | self.logger.info('Queue path ...') 63 | for i in files: 64 | if len(files[i]) > 0: 65 | for path in files[i]: 66 | relpath = os.path.relpath(path, i) # 相对于root的相对路径 67 | el = self.bucket + '::' + i+ '::' + relpath + '::C' 68 | hashcode = helper.calc_el_md5(i, relpath, self.bucket) 69 | if not self.is_el_queued(hashcode): 70 | data={"root": i, "relpath": relpath, "bucket": self.bucket, "action": 'C', "status": 0, "retries" : 0} 71 | self.qm.save(data) 72 | '''queue el, el: element of queue , formated as "bucket::root::path"''' 73 | try: 74 | self.queue.put(el, block = True, timeout = 1) 75 | msg = 'queue element:' + el 76 | #print msg 77 | self.logger.info(msg) 78 | except Full as e: 79 | self.queue.put(None) 80 | self.logger.error(e.message) 81 | self.qm.close() 82 | self.queue.put(None) 83 | #self.queue.join() 84 | return 85 | 86 | 87 | 88 | 89 | -------------------------------------------------------------------------------- /runtest.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import unittest 4 | from ossync.tests.api_test import * 5 | #from ossync.tests.helper_test import * 6 | # from ossync.tests.queue_model_test import * 7 | #from ossync.tests.oss_object_test import * 8 | 9 | if __name__ == '__main__': 10 | unittest.main() -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Copyright (c) 2012 Wu Tangsheng(lanbaba) 4 | 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | 12 | # The above copyright notice and this permission notice shall be included in 13 | # all copies or substantial portions of the Software. 14 | 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | # THE SOFTWARE. 22 | 23 | import os 24 | import sys 25 | 26 | # check if python version >= 2.6 and < 3.0 27 | if sys.version_info < (2, 6): 28 | sys.stderr.write("Sorry, OSSync requires at least Python 2.6\n") 29 | sys.exit(0) 30 | if sys.version_info >= (3, 0): 31 | sys.stderr.write("Sorry, Python 3.0+ is unsupported at present。\n") 32 | sys.exit(0) 33 | 34 | # check if linux kernel supports inotify 35 | if not os.path.exists("/proc/sys/fs/inotify"): 36 | sys.stderr.write("Sorry, your linux kernel doesn't support inotify。\n") 37 | sys.exit(0) 38 | 39 | print "Start to install necessary modules ..." 40 | # check if pip has been installed 41 | excode = os.system("pip --version") 42 | if excode > 0: 43 | # try to install pip 44 | os.system("sudo curl http://python-distribute.org/distribute_setup.py | python") 45 | os.system("curl https://raw.github.com/pypa/pip/master/contrib/get-pip.py | python") 46 | # clean temp files 47 | os.system("rm -f distribute*.tar.gz") 48 | 49 | # try to install pyinotify 50 | os.system("sudo pip install pyinotify") 51 | 52 | # check if pyinotify has been installed 53 | try: 54 | import pyinotify 55 | print "Installation complete successfully!" 56 | except ImportError as e: 57 | sys.stderr.write("Sorry, Installation pyinotify module failure! Please try to install it manually。\n") 58 | sys.exit(0) 59 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /sync_thread.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Copyright (c) 2012 Wu Tangsheng(lanbaba) 4 | 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | 12 | # The above copyright notice and this permission notice shall be included in 13 | # all copies or substantial portions of the Software. 14 | 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | # THE SOFTWARE. 22 | 23 | import os, sys, threading 24 | import logging 25 | import hashlib 26 | from Queue import * 27 | from ossync.lib import queue_model 28 | from ossync.lib import helper 29 | from ossync.sdk.oss_api import * 30 | from ossync.sdk.oss_xml_handler import * 31 | from config.setting import * 32 | 33 | class SyncThread(threading.Thread): 34 | def __init__(self, oss, queue, *args, **kwargs): 35 | threading.Thread.__init__(self, *args, **kwargs) 36 | self.queue = queue 37 | self.oss = oss 38 | self._terminate = False 39 | self.logger = logging.getLogger('app') 40 | dbpath = 'db/ossync.db' 41 | self.qm = queue_model.QueueModel(dbpath) 42 | 43 | def terminate(self): 44 | self._terminate = True 45 | 46 | def upload(self, bucket, oss_obj_name, filename): 47 | if not os.path.lexists(filename): 48 | return None 49 | success = False 50 | if(os.path.isdir(filename)): 51 | oss_obj_name += '/' 52 | res = self.oss.put_object_with_data(bucket = bucket, object = oss_obj_name, input_content = '') 53 | if (res.status / 100) == 2: 54 | success = True 55 | else: 56 | file_size = os.path.getsize(filename) 57 | if file_size > 2000000: 58 | res = self.oss.upload_large_file(bucket = bucket, object = oss_obj_name, filename = filename) 59 | else: 60 | res = self.oss.put_object_from_file(bucket = bucket, object = oss_obj_name, filename = filename) 61 | filehash = helper.calc_file_md5(filename) 62 | header_map = convert_header2map(res.getheaders()) 63 | etag = safe_get_element("etag", header_map).upper().replace('"', '') 64 | if (res.status / 100) == 2 and filehash.upper() == etag: 65 | success = True 66 | return success 67 | 68 | def exists_oss_object(self, bucket, oss_obj_name): 69 | headers = {} 70 | res = self.oss.head_object(bucket, oss_obj_name, headers) 71 | if (res.status / 100) == 2: 72 | return True 73 | else: 74 | return False 75 | 76 | def walk_bucket(self, bucket, prefix, marker, delimiter, maxkeys, headers, result = []): 77 | res = self.oss.get_bucket(bucket, prefix, marker, delimiter, maxkeys, headers) 78 | if (res.status / 100) == 2: 79 | body = res.read() 80 | h = GetBucketXml(body) 81 | (file_list, common_list) = h.list() 82 | if len(file_list) > 0: 83 | for item in file_list: 84 | result.append(item[0]) 85 | if len(common_list) > 0: 86 | for path in common_list: 87 | result.append(path) 88 | self.walk_bucket(bucket, path, marker, delimiter, maxkeys, headers, result) 89 | 90 | def delete_oss_object(self, bucket, oss_obj_name): 91 | headers = {} 92 | res = self.oss.delete_object(bucket, oss_obj_name, headers) 93 | if (res.status / 100) == 2: 94 | return True 95 | else: 96 | return False 97 | 98 | def delete_oss_objects(self, bucket, oss_obj_name): 99 | headers = {} 100 | result = [] 101 | marker = '' 102 | delimiter = '/' 103 | maxkeys = 100 104 | self.walk_bucket(bucket, oss_obj_name, marker, delimiter, maxkeys, headers, result) 105 | if len(result) > 0: 106 | for item in result: 107 | self.oss.delete_object(bucket, item, headers) 108 | else: 109 | self.oss.delete_object(bucket, oss_obj_name, headers) 110 | return True 111 | 112 | def queue_el(self, el): 113 | '''el: element of queue , formated as "bucket::root::path"''' 114 | try: 115 | self.queue.put(el, block = True, timeout = 1) 116 | msg = 'requeue element:' + el 117 | self.logger.info(msg) 118 | except Full as e: 119 | self.logger.error(e.message) 120 | print e 121 | 122 | def is_el_processed(self, hashcode): 123 | row = self.qm.get(hashcode) 124 | if row and str(row['status']) == '1': 125 | return True 126 | return False 127 | 128 | def run(self): 129 | self.logger.info('Now starting sync thread ...') 130 | self.qm.open() 131 | while True: 132 | if self._terminate: 133 | break 134 | item = self.queue.get() 135 | if item is None: 136 | break 137 | (bucket, root, relpath, action) = item.split('::') 138 | if len(bucket) > 0 and len(root) > 0 and len(relpath) > 0 and len(action) > 0: 139 | hashcode = helper.calc_el_md5(root, relpath, bucket) 140 | if not self.is_el_processed(hashcode): 141 | oss_obj_name = os.path.join(os.path.basename(root), relpath) 142 | if len(oss_obj_name) > 0: 143 | if(action == 'M' or action == 'C'): 144 | success = self.upload(bucket, oss_obj_name, os.path.join(root, relpath)) 145 | msg = 'put object ' + oss_obj_name + ' to bucket ' + bucket 146 | if(action == 'D'): 147 | success = self.delete_oss_objects(bucket, oss_obj_name) 148 | msg = 'delete object ' + oss_obj_name + ' of bucket ' + bucket 149 | if success: 150 | msg += ' success' 151 | self.logger.info(msg) 152 | self.qm.update_status(hashcode, 1) 153 | else: 154 | if success == False: 155 | msg += ' failure' 156 | self.logger.error(msg) 157 | """requeue losing element""" 158 | row = self.qm.get(hashcode) 159 | if row: 160 | retries = int(row['retries'] ) 161 | if retries < MAX_RETRIES: 162 | self.queue_el(item) 163 | self.qm.update_retries(hashcode, retries + 1) 164 | else: 165 | self.logger.critical(msg + ' exceed max retries') 166 | else: 167 | self.logger.critical(msg + ' failure, resource may not exists.') 168 | pass 169 | self.qm.close() 170 | self.queue.task_done() 171 | return 172 | --------------------------------------------------------------------------------