├── oss ├── __init__.py ├── oss_fs.py ├── oss_sample.py ├── oss_xml_handler.py ├── oss_util.py ├── oss_cmd.py └── oss_api.py ├── osscmdlib ├── __init__.py ├── pkginfo.py ├── ossuri.py ├── utils.py └── Config.py ├── setup.cfg ├── INSTALL ├── setup.py ├── README └── osscmd /oss/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /osscmdlib/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [sdist] 2 | formats = gztar,zip 3 | -------------------------------------------------------------------------------- /INSTALL: -------------------------------------------------------------------------------- 1 | Just run: 2 | python setup.py install 3 | (If you do not have a root authority, you may need to run: sudo python setup.py install) 4 | 5 | -------------------------------------------------------------------------------- /osscmdlib/pkginfo.py: -------------------------------------------------------------------------------- 1 | package = "osscmd" 2 | version = "0.1.1" 3 | license = "GPL version 2" 4 | short_description = "Command line tool for managing Aliyun OSS" 5 | long_description = """ 6 | osscmd lets you copy files from/to Alibaba OSS 7 | (Open Storage Service) using a simple to use 8 | command line client. Supports rsync-like backup, 9 | GPG encryption, and more. 10 | """ 11 | 12 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | ## ossbrowser 4 | ## Author: melory 5 | ## Email:imsrch@melory.me 6 | ## License: GPL Version 2 7 | 8 | from distutils.core import setup 9 | import sys 10 | import os 11 | 12 | import osscmdlib.pkginfo as pkginfo 13 | 14 | if float("%d.%d" % sys.version_info[:2]) < 2.4: 15 | sys.stderr.write("Your Python version %d.%d.%d is not supported.\n" % sys.version_info[:3]) 16 | sys.stderr.write("osscmd requires Python 2.4 or newer.\n") 17 | sys.exit(1) 18 | 19 | man_path = "share/man" 20 | doc_path = "share/doc/packages" 21 | data_files = [ 22 | (doc_path+"/osscmd", [ "README", "INSTALL" ]), 23 | ] 24 | 25 | ## Main distutils info 26 | setup( 27 | ## Content description 28 | name = pkginfo.package, 29 | version = pkginfo.version, 30 | packages = ['oss', 'osscmdlib'], 31 | scripts = ['osscmd'], 32 | data_files = data_files, 33 | 34 | ## Packaging details 35 | author = "linjiudui", 36 | author_email = "linjd828917@gmail.com", 37 | license = pkginfo.license, 38 | description = pkginfo.short_description, 39 | long_description = pkginfo.long_description 40 | ) 41 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | OSSCMD 2 | ====== 3 | 4 | OSScmd is a command line tool for uploading, retrieving and managing data in Aliyun OSS. 5 | 6 | (Chinese README) 7 | 查看INSTALL文件查看具体的安装方法 8 | (运行 python setup.py install, Linux下可能需要管理员权限，如在ubuntu下， 9 | cd 到setup.py所在目录，然后输入:sudo python setup.py install，这样就可以像使用Linux 10 | 自带的其他命令一样使用osscmd命令) 11 | 12 | osscmd支持Windows和Linux平台,在Linux平台下可以安装为系统的一条命令。 13 | 很多Linux的用户更习惯命令行的操作,osscmd也更适合在linux下使用 14 | 15 | osscmd提供了访问和OSS的命令行接口,提供了包括 16 | 1: 一些基本的命令如ls,la,sign(用secret key对任意字符串签名) 17 | 2: 对bucket的基本操作,包括create,delete,setacl,getacl,head 18 | 3: 上传文件(object),文件夹到OSS 19 | 4: 下载文件(object),文件夹到本地 20 | 5: 删除OSS上object或者某个子目录 21 | 6: 提供同步功能,同步文件或文件夹到OSS（OSS同步到本地暂不支持） 22 | 7: OSS端的拷贝(cp)和移动(mv)操作等等 23 | 8: 丰富的命令行选项 24 | 9: 具体可以通过运行命令osscmd --help看到具体的帮助信息 25 | 命令 26 | 27 | 28 | osscmd命令举例 29 | 30 | 1: 首次运行osscmd命令需要进行配置,主要配置oss的secret_key和access_id保存到配置文件 31 | 命令:osscmd --configure 32 | 33 | 2: 列举bucket或文件夹ls 34 | osscmd ls:显示所有的bucket 35 | osscmd ls oss://bucket-name/[prefix]: 列出指定目录下当前层的object 36 | osscmd ls -r oss://bucket-name/[prefix]: 递归的显示此目录下所有的object 37 | osscmd la:显示所有bucket下所有的objcet,每个bucket每次至多显示1000个object 38 | 39 | 3:创建bucket 40 | osscmd mb oss://bucket-name 41 | osscmd mb oss://bucket-name --acl-public:创建一个public的bucket 42 | osscmd mb oss://bucket-name --acl-private:创建一个private的bucket 43 | 44 | 3:删除bucket 45 | osscmd rb oss://bucket-name 46 | 如果bucket不为空需要指定-r --recursive选项,否则删除失败 47 | 48 | 4:查看和修改某个bucket的acl 49 | osscmd getacl oss://bucket-name/[prefix]:获取指定bucket的acl,prefix会被忽略 50 | osscmd setacl oss://bucket-name/ --acl-public:设置指定bucket的acl,目前不提供 51 | 设置public-read-write权限 52 | 53 | 5:上传文件或目录qianhui@cs.zju.edu.cn 54 | osscmd put localfile oss://bucket-name/[prefix] 上传一个文件到OSS 55 | osscmd put -r localdir oss://bucket-name/[prefix] 上传一个目录到OSS 56 | 上传目录是如果加了--dry-run(-n)那么命令行会显示这要上传的文件列表, 57 | 但是不会进行实际的上传操作,改选项同样适用于get,del,sync等操作 58 | 59 | 6:下载文件或目录 60 | osscmd get oss://bucket-name/file localdir/file2:将oss上的file文件下载到 61 | 本地保存为file2,如果不指定第二个参数,则保存到当前目录,如果第二个参数 62 | 是一个目录,则将文件下载到指定目录下,文件名认为file 63 | osscmd get -r oss://bucket-name/[prefix] localdir:将oss上的指定bucket下某个 64 | 子目录(前缀)下载到本地目录,localdir必须是一个目录 65 | osscmd get -r --dry-run oss://bucket-name/[prefix] localdir:--dry-run的作用参见4 66 | 67 | 7:删除文件或目录 68 | osscmd del oss://bucket-name/file:删除指定bucket下的某个file文件 69 | osscmd del -r oss://bucket-name/[prefix]:如果删除是一个子目录(不止一个文件),必须 70 | 指定-r(--recursive)选项 71 | 72 | 8:获取某个bucket或object的信息 73 | osscmd info oss://bucket-name/[file] 74 | 75 | 9:在OSS端执行copy和move操作 76 | osscmd cp oss://bucket1/[prefix] oss://bucket2/[prefix]:将bucket1下某个文件或子目录 77 | 拷贝到bucket2的某个子目录下,如果拷贝不是一个文件,必须指定-r(--recursive选项) 78 | osscmd cp oss://bucket1/[prefix] oss://bucket2/[prefix]:功能通cp相同,只是copy好后,将 79 | 源文件或目录删除 80 | 可以指定-n(--dry-run)选项先查看此次操作的会移动的文件 81 | 82 | 10:同步功能 83 | osscmd sync localdir oss://bucket-name/[prefix]:同步到oss端,通过比对文件的大小和md5值 84 | 确定是否需要上传文件,如果大小和md5一直则不上传,同样可以使用-n(--dry-run)选项查看上传的 85 | 文件和不执行实际的操作 86 | osscmd sync oss://bucket-name/[prefix] localdir:将OSS端的文件数据同步到本地，仍然是通过比较 87 | 两端两个文件（如果存在）的size和md5值,如果指定--no-check-md5选项，则不检查md5值 88 | 89 | 11:用secret key对任意字符串签名 90 | osscmd sign string 91 | 92 | 12:输出info信息和debug信息 93 | 每条命令指定-v(--verbose)和-d(--debug)选项 94 | 95 | 13:更多的选项和命令请使用osscmd --help查看 96 | -------------------------------------------------------------------------------- /osscmdlib/ossuri.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | ## ossbrowser 4 | ## Author: melory 5 | ## Email:imsrch@melory.me 6 | ## License: GPL Version 2 7 | 8 | import os 9 | import re 10 | import sys 11 | from utils import unicodise 12 | 13 | class OSSUri(object): 14 | type = None 15 | _subclasses = None 16 | 17 | def __new__(self, string): 18 | if not self._subclasses: 19 | ## Generate a list of all subclasses of OSSUri 20 | self._subclasses = [] 21 | dict = sys.modules[__name__].__dict__ 22 | for something in dict: 23 | if type(dict[something]) is not type(self): 24 | continue 25 | if issubclass(dict[something], self) and dict[something] != self: 26 | self._subclasses.append(dict[something]) 27 | for subclass in self._subclasses: 28 | try: 29 | instance = object.__new__(subclass) 30 | instance.__init__(string) 31 | return instance 32 | except ValueError, e: 33 | continue 34 | raise ValueError("%s: not a recognized URI" % string) 35 | 36 | def __str__(self): 37 | return self.uri() 38 | 39 | def __unicode__(self): 40 | return self.uri() 41 | 42 | def public_url(self): 43 | raise ValueError("This OSS URI does not have Anonymous URL representation") 44 | 45 | def basename(self): 46 | return self.__unicode__().split(os.path.sep)[-1] 47 | 48 | class OSSUriOSS(OSSUri): 49 | type = "oss" 50 | _re = re.compile("^oss://([^/]+)/?(.*)", re.IGNORECASE) 51 | def __init__(self, string): 52 | match = self._re.match(string) 53 | if not match: 54 | raise ValueError("%s: not a OSS URI" % string) 55 | groups = match.groups() 56 | self._bucket = groups[0] 57 | self._object = unicodise(groups[1]) 58 | 59 | def bucket(self): 60 | return self._bucket 61 | 62 | def object(self): 63 | return self._object 64 | 65 | def has_bucket(self): 66 | return bool(self._bucket) 67 | 68 | def has_object(self): 69 | return bool(self._object) 70 | 71 | def uri(self): 72 | return "/".join(["oss:/", self._bucket, self._object]) 73 | 74 | def public_url(self): 75 | return "http://storage.aliyun.com/%s/%s" % (self._bucket, self._object) 76 | 77 | def host_name(self): 78 | return "http://storage.aliyun.com/%s/" % (self._bucket) 79 | 80 | @staticmethod 81 | def compose_uri(bucket, object = ""): 82 | return "oss://%s/%s" % (bucket, object) 83 | 84 | class OSSUriFile(OSSUri): 85 | type = "file" 86 | _re = re.compile("^(\w+://)?(.*)") 87 | def __init__(self, string): 88 | match = self._re.match(string) 89 | groups = match.groups() 90 | if groups[0] not in (None, "file://"): 91 | raise ValueError("%s: not a file:// URI" % string) 92 | self._path = unicodise(groups[1]).split(os.path.sep) 93 | 94 | def path(self): 95 | return os.path.sep.join(self._path) 96 | 97 | def uri(self): 98 | return os.path.sep.join([self.path()]) 99 | 100 | def isdir(self): 101 | return os.path.isdir(self.path()) 102 | 103 | def dirname(self): 104 | return os.path.dirname(self.path()) 105 | 106 | if __name__ == "__main__": 107 | uri = OSSUri("oss://bucket/object") 108 | print "type() =", type(uri) 109 | print "uri =", uri 110 | print "uri.type=", uri.type 111 | print "bucket =", uri.bucket() 112 | print "object =", uri.object() 113 | print 114 | 115 | uri = OSSUri("oss://bucket") 116 | print "type() =", type(uri) 117 | print "uri =", uri 118 | print "uri.type=", uri.type 119 | print "bucket =", uri.bucket() 120 | print 121 | 122 | uri = OSSUri("/path/to/local/file.txt") 123 | print "type() =", type(uri) 124 | print "uri =", uri 125 | print "uri.type=", uri.type 126 | print "path =", uri.path() 127 | print 128 | 129 | -------------------------------------------------------------------------------- /osscmdlib/utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | ## ossbrowser 4 | ## Author: melory 5 | ## Email:imsrch@melory.me 6 | ## License: GPL Version 2 7 | 8 | import os 9 | import sys 10 | import time 11 | import re 12 | import hmac 13 | import base64 14 | 15 | from logging import debug, info, warning, error 16 | 17 | import Config 18 | 19 | # hashlib backported to python 2.4 / 2.5 is not compatible with hmac! 20 | if sys.version_info[0] == 2 and sys.version_info[1] < 6: 21 | from md5 import md5 22 | import sha as sha1 23 | else: 24 | from hashlib import md5, sha1 25 | 26 | 27 | 28 | __all__ = [] 29 | 30 | def formatSize(size, human_readable = False, floating_point = False): 31 | size = floating_point and float(size) or int(size) 32 | if human_readable: 33 | coeffs = ['k', 'M', 'G', 'T'] 34 | coeff = "" 35 | while size > 2048: 36 | size /= 1024 37 | coeff = coeffs.pop(0) 38 | return (size, coeff) 39 | else: 40 | return (size, "") 41 | __all__.append("formatSize") 42 | 43 | def dateOSStoPython(date): 44 | date = re.compile("(\.\d*)?Z").sub(".000Z", date) 45 | return time.strptime(date, "%Y-%m-%dT%H:%M:%S.000Z") 46 | __all__.append("dateOSStoPython") 47 | 48 | def formatDateTime(osstimestamp): 49 | return time.strftime("%Y-%m-%d %H:%M", dateOSStoPython(osstimestamp)) 50 | __all__.append("formatDateTime") 51 | 52 | def hash_file_md5(filename): 53 | h = md5() 54 | f = open(filename, "rb") 55 | while True: 56 | # Hash 32kB chunks 57 | data = f.read(32*1024) 58 | if not data: 59 | break 60 | h.update(data) 61 | f.close() 62 | return h.hexdigest() 63 | __all__.append("hash_file_md5") 64 | 65 | 66 | 67 | def unicodise(string, encoding = None, errors = "replace"): 68 | """ 69 | Convert 'string' to Unicode or raise an exception. 70 | """ 71 | if not encoding: 72 | encoding = Config.Config().encoding 73 | 74 | if type(string) == unicode: 75 | return string 76 | debug("Unicodising %r using %s" % (string, encoding)) 77 | try: 78 | return string.decode(encoding, errors) 79 | except UnicodeDecodeError: 80 | raise UnicodeDecodeError("Conversion to unicode failed: %r" % string) 81 | __all__.append("unicodise") 82 | 83 | def deunicodise(string, encoding = None, errors = "replace"): 84 | """ 85 | Convert unicode 'string' to , by default replacing 86 | all invalid characters with '?' or raise an exception. 87 | """ 88 | 89 | if not encoding: 90 | encoding = Config.Config().encoding 91 | 92 | if type(string) != unicode: 93 | return str(string) 94 | debug("DeUnicodising %r using %s" % (string, encoding)) 95 | try: 96 | return string.encode(encoding, errors) 97 | except UnicodeEncodeError: 98 | raise UnicodeEncodeError("Conversion from unicode failed: %r" % string) 99 | __all__.append("deunicodise") 100 | 101 | def unicodise_safe(string, encoding = None): 102 | """ 103 | Convert 'string' to Unicode according to current encoding 104 | and replace all invalid characters with '?' 105 | """ 106 | 107 | return unicodise(deunicodise(string, encoding), encoding).replace(u'\ufffd', '?') 108 | __all__.append("unicodise_safe") 109 | 110 | def replace_nonprintables(string): 111 | """ 112 | replace_nonprintables(string) 113 | 114 | Replaces all non-printable characters 'ch' in 'string' 115 | where ord(ch) <= 26 with ^@, ^A, ... ^Z 116 | """ 117 | new_string = "" 118 | modified = 0 119 | for c in string: 120 | o = ord(c) 121 | if (o <= 31): 122 | new_string += "^" + chr(ord('@') + o) 123 | modified += 1 124 | elif (o == 127): 125 | new_string += "^?" 126 | modified += 1 127 | else: 128 | new_string += c 129 | if modified and Config.Config().urlencoding_mode != "fixbucket": 130 | warning("%d non-printable characters replaced in: %s" % (modified, new_string)) 131 | return new_string 132 | __all__.append("replace_nonprintables") 133 | 134 | def sign_string(string_to_sign): 135 | #debug("string_to_sign: %s" % string_to_sign) 136 | signature = base64.encodestring(hmac.new(Config.Config().secret_access_key, string_to_sign, sha1).digest()).strip() 137 | #debug("signature: %s" % signature) 138 | return signature 139 | __all__.append("sign_string") 140 | 141 | if __name__ == '__main__': 142 | pass 143 | -------------------------------------------------------------------------------- /osscmdlib/Config.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from logging import debug, warning, error 3 | import re 4 | 5 | class Config(object): 6 | _instance = None 7 | _parsed_files = [] 8 | access_id = "" 9 | secret_access_key = "" 10 | host_base = "storage.aliyun.com" 11 | verbosity = logging.WARNING 12 | list_md5 = False 13 | human_readable_sizes = False 14 | force = False 15 | # get_continue = False 16 | skip_existing = False 17 | recursive = False 18 | acl_public = None 19 | dry_run = False 20 | delete_removed = False 21 | # List of checks to be performed for 'sync' 22 | sync_checks = ['size', 'md5'] 23 | encoding = "utf-8" 24 | urlencoding_mode = "normal" 25 | 26 | ## Creating a singleton 27 | def __new__(self, configfile = None): 28 | if self._instance is None: 29 | self._instance = object.__new__(self) 30 | return self._instance 31 | 32 | def __init__(self, configfile = None): 33 | if configfile: 34 | self.read_config_file(configfile) 35 | 36 | def option_list(self): 37 | retval = [] 38 | for option in dir(self): 39 | ## Skip attributes that start with underscore or are not string, int or bool 40 | option_type = type(getattr(Config, option)) 41 | if option.startswith("_") or \ 42 | not (option_type in ( 43 | type("string"), # str 44 | type(42), # int 45 | type(True))): # bool 46 | continue 47 | retval.append(option) 48 | return retval 49 | 50 | def read_config_file(self, configfile): 51 | cp = ConfigParser(configfile) 52 | for option in self.option_list(): 53 | self.update_option(option, cp.get(option)) 54 | self._parsed_files.append(configfile) 55 | 56 | def dump_config(self, stream): 57 | ConfigDumper(stream).dump("default", self) 58 | 59 | def update_option(self, option, value): 60 | if value is None: 61 | return 62 | #### Special treatment of some options 63 | ## verbosity must be known to "logging" module 64 | if option == "verbosity": 65 | try: 66 | setattr(Config, "verbosity", logging._levelNames[value]) 67 | except KeyError: 68 | error("Config: verbosity level '%s' is not valid" % value) 69 | ## allow yes/no, true/false, on/off and 1/0 for boolean options 70 | elif type(getattr(Config, option)) is type(True): # bool 71 | if str(value).lower() in ("true", "yes", "on", "1"): 72 | setattr(Config, option, True) 73 | elif str(value).lower() in ("false", "no", "off", "0"): 74 | setattr(Config, option, False) 75 | else: 76 | error("Config: value of option '%s' must be Yes or No, not '%s'" % (option, value)) 77 | elif type(getattr(Config, option)) is type(42): # int 78 | try: 79 | setattr(Config, option, int(value)) 80 | except ValueError, e: 81 | error("Config: value of option '%s' must be an integer, not '%s'" % (option, value)) 82 | else: # string 83 | setattr(Config, option, value) 84 | 85 | class ConfigParser(object): 86 | def __init__(self, file, sections = []): 87 | self.cfg = {} 88 | self.parse_file(file, sections) 89 | 90 | def parse_file(self, file, sections = []): 91 | debug("ConfigParser: Reading file '%s'" % file) 92 | if type(sections) != type([]): 93 | sections = [sections] 94 | in_our_section = True 95 | f = open(file, "r") 96 | r_comment = re.compile("^\s*#.*") 97 | r_empty = re.compile("^\s*$") 98 | r_section = re.compile("^\[([^\]]+)\]") 99 | r_data = re.compile("^\s*(?P\w+)\s*=\s*(?P.*)") 100 | r_quotes = re.compile("^\"(.*)\"\s*$") 101 | for line in f: 102 | if r_comment.match(line) or r_empty.match(line): 103 | continue 104 | is_section = r_section.match(line) 105 | if is_section: 106 | section = is_section.groups()[0] 107 | in_our_section = (section in sections) or (len(sections) == 0) 108 | continue 109 | is_data = r_data.match(line) 110 | if is_data and in_our_section: 111 | data = is_data.groupdict() 112 | if r_quotes.match(data["value"]): 113 | data["value"] = data["value"][1:-1] 114 | self.__setitem__(data["key"], data["value"]) 115 | if data["key"] in ("access_key", "secret_key", "gpg_passphrase"): 116 | print_value = (data["value"][:2]+"...%d_chars..."+data["value"][-1:]) % (len(data["value"]) - 3) 117 | else: 118 | print_value = data["value"] 119 | debug("ConfigParser: %s->%s" % (data["key"], print_value)) 120 | continue 121 | warning("Ignoring invalid line in '%s': %s" % (file, line)) 122 | 123 | def __getitem__(self, name): 124 | return self.cfg[name] 125 | 126 | def __setitem__(self, name, value): 127 | self.cfg[name] = value 128 | 129 | def get(self, name, default = None): 130 | if self.cfg.has_key(name): 131 | return self.cfg[name] 132 | return default 133 | 134 | class ConfigDumper(object): 135 | def __init__(self, stream): 136 | self.stream = stream 137 | 138 | def dump(self, section, config): 139 | self.stream.write("[%s]\n" % section) 140 | for option in config.option_list(): 141 | self.stream.write("%s = %s\n" % (option, getattr(config, option))) 142 | 143 | -------------------------------------------------------------------------------- /oss/oss_fs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | ## ossbrowser 4 | ## Author: melory 5 | ## Email:imsrch@melory.me 6 | ## License: GPL Version 2 7 | 8 | from oss_api import * 9 | from oss_xml_handler import * 10 | 11 | ''' 12 | Mock a file system for testing purpose 13 | ''' 14 | class OssFS: 15 | def __init__(self, host, id="", key=""): 16 | self.buckets = [] 17 | self.host = host 18 | self.oss = OssAPI(host, id, key) 19 | 20 | def put_bucket(self, bucketname,acl='private'): 21 | if bucketname in self.buckets: 22 | return False 23 | res = self.oss.put_bucket(bucketname,acl) 24 | if (res.status / 100) == 2: 25 | self.buckets.append(bucketname) 26 | return True 27 | else: 28 | return False 29 | 30 | def delete_bucket(self, bucketname): 31 | res = self.oss.delete_bucket(bucketname) 32 | if (res.status / 100) == 2: 33 | while bucketname in self.buckets: 34 | del self.buckets[self.buckets.index(bucketname)] 35 | return True 36 | else: 37 | return False 38 | 39 | def list_bucket(self): 40 | res = self.oss.get_service() 41 | if (res.status / 100) == 2: 42 | body = res.read() 43 | h = GetServiceXml(body) 44 | self.buckets = h.list() 45 | return self.buckets 46 | 47 | def get_bucket_acl(self, bucket): 48 | res = self.oss.get_bucket_acl(bucket) 49 | body = res.read() 50 | h = GetBucketAclXml(body) 51 | return h.grant 52 | 53 | 54 | def put_bucket_acl(self, bucket, acl=''): 55 | res = self.oss.put_bucket_acl(bucket,acl) 56 | if (res.status / 100) == 2: 57 | return True 58 | else: 59 | return False 60 | 61 | #fileobj is django defined uploaded file structure 62 | def upload_file(self, bucket, object, filename): 63 | res = self.oss.put_object_from_file(bucket, object, filename) 64 | if (res.status / 100) == 2: 65 | return True 66 | else: 67 | return False 68 | 69 | def make_dir(self, bucket, dirname): 70 | object = dirname + '/' 71 | res = self.oss.put_object_with_data(bucket, object, "") 72 | if (res.status / 100) == 2: 73 | return True 74 | else: 75 | return False 76 | 77 | def read_file(self, bucket, filename): 78 | res = self.oss.get_object(bucket, filename) 79 | return res.read() 80 | 81 | #return a list of file 82 | def list_file(self, bucket, prefix="", delim="", marker="", maxkeys=5): 83 | fl = [] 84 | pl = [] 85 | res = self.oss.get_bucket(bucket, prefix, delimiter=delim, marker=marker, maxkeys=maxkeys) 86 | if (res.status / 100) == 2: 87 | body = res.read() 88 | h = GetBucketXml(body) 89 | (fl, pl) = h.list() 90 | return (fl, pl) 91 | 92 | def delete_file(self, bucket, filename): 93 | res = self.oss.delete_object(bucket, filename) 94 | if (res.status / 100) == 2: 95 | return True 96 | else: 97 | return False 98 | 99 | def open_file_for_write(self, bucket, filename, filesize): 100 | conn = self.oss._open_conn_to_put_object(bucket, filename, filesize) 101 | return WriteFileObject(conn) 102 | 103 | def open_file_for_read(self, bucket, filename): 104 | res = self.oss.get_object(bucket, filename) 105 | return ReadFileObject(res) 106 | 107 | class ReadFileObject: 108 | def __init__(self, res): 109 | self.res = res 110 | 111 | def read(self, buffer_size): 112 | if (self.res.status / 100) == 2: 113 | return self.res.read(buffer_size) 114 | else: 115 | return "" 116 | 117 | def close(self): 118 | if (self.res.status / 100) == 2: 119 | return True 120 | else: 121 | return False 122 | 123 | class WriteFileObject: 124 | def __init__(self, conn): 125 | self.conn = conn 126 | 127 | def write(self, data): 128 | self.conn.send(data) 129 | 130 | def close(self): 131 | res = self.conn.getresponse() 132 | if (res.status / 100) == 2: 133 | return True 134 | else: 135 | return False 136 | 137 | 138 | def test_open_file_for_write(bucket, object, filename): 139 | fp = file(filename, 'r') 140 | fp.seek(os.SEEK_SET, os.SEEK_END) 141 | filesize = fp.tell() 142 | fp.seek(os.SEEK_SET) 143 | wo = fs.open_file_for_write(bucket, object, filesize) 144 | l = fp.read(BufferSize) 145 | while len(l) > 0: 146 | print len(l) 147 | wo.write(l) 148 | l = fp.read(BufferSize) 149 | print wo.close() 150 | 151 | def test_open_file_for_read(bucket, object, filename): 152 | fp = file(filename, 'w') 153 | ro = fs.open_file_for_read(bucket, object) 154 | buf = ro.read(1024) 155 | while len(buf) > 0: 156 | print len(buf) 157 | fp.write(buf) 158 | buf = ro.read(1024) 159 | print ro.close() 160 | 161 | if __name__ == "__main__": 162 | host = '127.0.0.1:8080' 163 | id = "" 164 | key = "" 165 | BufferSize = 8000 166 | fs = OssFS(host, id, key) 167 | test_open_file_for_write('6uu', "a/b/c/d", "put.jpg") 168 | test_open_file_for_read('6uu', "a/b/c/d", "put.jpg") 169 | print fs.upload_file('6uu', "a/b/c", 'put.jpg') 170 | print fs.upload_file('6uu', "b/b/c", 'put.jpg') 171 | print fs.upload_file('6uu', "b/c", 'put.jpg') 172 | print fs.upload_file('6uu', "c", 'put.jpg') 173 | (fl, pl) = fs.list_file('bb1','',delim='',marker='') 174 | print fl 175 | print pl 176 | (fl, pl) = fs.list_file('bb1','',delim='',marker='') 177 | print fl 178 | print pl 179 | -------------------------------------------------------------------------------- /oss/oss_sample.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | ## ossbrowser 4 | ## Author: melory 5 | ## Email:imsrch@melory.me 6 | ## License: GPL Version 2 7 | 8 | import time 9 | from oss_xml_handler import * 10 | from oss_api import * 11 | 12 | HOST="storage.aliyun.com" 13 | ACCESS_ID = "YOUR ACCESS ID" 14 | SECRET_ACCESS_KEY = "YOUR SECRET ACCESS KEY" 15 | #ACCESS_ID and SECRET_ACCESS_KEY should not be empty, please input correct one. 16 | 17 | if __name__ == "__main__": 18 | #init oss, get instance of oss 19 | if len(ACCESS_ID) == 0 or len(SECRET_ACCESS_KEY) == 0: 20 | print "Please make sure ACCESS_ID and SECRET_ACCESS_KEY are correct in ", __file__ , ", init are empty!" 21 | exit(0) 22 | oss = OssAPI(HOST, ACCESS_ID, SECRET_ACCESS_KEY) 23 | sep = "==============================" 24 | 25 | #sign_url_auth_with_expire_time(self, method, url, headers = {}, resource="/", timeout = 60): 26 | method = "GET" 27 | bucket = "test" + time.strftime("%Y-%b-%d%H-%M-%S").lower() 28 | object = "test_object" 29 | url = "http://" + HOST + "/oss/" + bucket + "/" + object 30 | headers = {} 31 | resource = "/" + bucket + "/" + object 32 | 33 | timeout = 60 34 | url_with_auth = oss.sign_url_auth_with_expire_time(method, url, headers, resource, timeout) 35 | print "after signature url is: ", url_with_auth 36 | print sep 37 | #put_bucket(self, bucket, acl='', headers = {}): 38 | acl = 'private' 39 | headers = {} 40 | res = oss.put_bucket(bucket, acl, headers) 41 | if (res.status / 100) == 2: 42 | print "put bucket ", bucket, "OK" 43 | else: 44 | print "put bucket ", bucket, "ERROR" 45 | print sep 46 | 47 | #get_service(self): 48 | res = oss.get_service() 49 | if (res.status / 100) == 2: 50 | body = res.read() 51 | h = GetServiceXml(body) 52 | print "bucket list size is: ", len(h.list()) 53 | print "bucket list is: " 54 | for i in h.list(): 55 | print i 56 | else: 57 | print res.status 58 | print sep 59 | 60 | #put_object_from_string(self, bucket, object, input_content, content_type=DefaultContentType, headers = {}): 61 | object = "object_test" 62 | input_content = "hello, OSS" 63 | content_type = "text/HTML" 64 | headers = {} 65 | res = oss.put_object_from_string(bucket, object, input_content, content_type, headers) 66 | if (res.status / 100) == 2: 67 | print "put_object_from_string OK" 68 | else: 69 | print "put_object_from_string ERROR" 70 | print sep 71 | 72 | #put_object_from_file(self, bucket, object, filename, content_type=DefaultContentType, headers = {}): 73 | object = "object_test" 74 | filename = __file__ 75 | content_type = "text/HTML" 76 | headers = {} 77 | res = oss.put_object_from_file(bucket, object, filename, content_type, headers) 78 | if (res.status / 100) == 2: 79 | print "put_object_from_file OK" 80 | else: 81 | print "put_object_from_file ERROR" 82 | print sep 83 | 84 | #put_object_from_fp(self, bucket, object, fp, content_type=DefaultContentType, headers = {}): 85 | object = "object_test" 86 | filename = __file__ 87 | content_type = "text/HTML" 88 | headers = {} 89 | 90 | fp = open(filename, 'rb') 91 | res = oss.put_object_from_fp(bucket, object, fp, content_type, headers) 92 | fp.close() 93 | if (res.status / 100) == 2: 94 | print "put_object_from_fp OK" 95 | else: 96 | print "put_object_from_fp ERROR" 97 | print sep 98 | 99 | #get_object(self, bucket, object, headers = {}): 100 | object = "object_test" 101 | headers = {} 102 | 103 | res = oss.get_object(bucket, object, headers) 104 | if (res.status / 100) == 2: 105 | print "get_object OK" 106 | else: 107 | print "get_object ERROR" 108 | print sep 109 | 110 | #get_object_to_file(self, bucket, object, filename, headers = {}): 111 | object = "object_test" 112 | headers = {} 113 | filename = "get_object_test_file" 114 | 115 | res = oss.get_object_to_file(bucket, object, filename, headers) 116 | if (res.status / 100) == 2: 117 | print "get_object_to_file OK" 118 | else: 119 | print "get_object_to_file ERROR" 120 | print sep 121 | 122 | #head_object(self, bucket, object, headers = {}): 123 | object = "object_test" 124 | headers = {} 125 | res = oss.head_object(bucket, object, headers) 126 | if (res.status / 100) == 2: 127 | print "head_object OK" 128 | header_map = convert_header2map(res.getheaders()) 129 | content_len = safe_get_element("content-length", header_map) 130 | etag = safe_get_element("etag", header_map).upper() 131 | print "content length is:", content_len 132 | print "ETag is: ", etag 133 | 134 | else: 135 | print "head_object ERROR" 136 | print sep 137 | 138 | #get_bucket_acl(self, bucket): 139 | res = oss.get_bucket_acl(bucket) 140 | if (res.status / 100) == 2: 141 | body = res.read() 142 | h = GetBucketAclXml(body) 143 | print "bucket acl is:", h.grant 144 | else: 145 | print "get bucket acl ERROR" 146 | print sep 147 | 148 | #get_bucket(self, bucket, prefix='', marker='', delimiter='', maxkeys='', headers = {}): 149 | prefix = "" 150 | marker = "" 151 | delimiter = "/" 152 | maxkeys = "100" 153 | headers = {} 154 | res = oss.get_bucket(bucket, prefix, marker, delimiter, maxkeys, headers) 155 | if (res.status / 100) == 2: 156 | body = res.read() 157 | h = GetBucketXml(body) 158 | (file_list, common_list) = h.list() 159 | print "object list is:" 160 | for i in file_list: 161 | print i 162 | print "common list is:" 163 | for i in common_list: 164 | print i 165 | print sep 166 | 167 | #upload_large_file(self, bucket, object, filename, thread_num = 10, max_part_num = 1000): 168 | res = oss.upload_large_file(bucket, object, __file__) 169 | if (res.status / 100) == 2: 170 | print "upload_large_file OK" 171 | else: 172 | print "upload_large_file ERROR" 173 | 174 | print sep 175 | 176 | #get_object_group_index(self, bucket, object, headers = {}) 177 | res = oss.get_object_group_index(bucket, object) 178 | if (res.status / 100) == 2: 179 | print "get_object_group_index OK" 180 | body = res.read() 181 | h = GetObjectGroupIndexXml(body) 182 | for i in h.list(): 183 | print "object group part msg:", i 184 | else: 185 | print "get_object_group_index ERROR" 186 | 187 | res = oss.get_object_group_index(bucket, object) 188 | if res.status == 200: 189 | body = res.read() 190 | h = GetObjectGroupIndexXml(body) 191 | object_group_index = h.list() 192 | for i in object_group_index: 193 | if len(i) == 4 and len(i[1]) > 0: 194 | part_name = i[1].strip() 195 | res = oss.delete_object(bucket, part_name) 196 | if res.status != 204: 197 | print "delete part ", part_name, " in bucket:", bucket, " failed!" 198 | else: 199 | print "delete part ", part_name, " in bucket:", bucket, " ok" 200 | print sep 201 | 202 | #delete_object(self, bucket, object, headers = {}): 203 | object = "object_test" 204 | headers = {} 205 | res = oss.delete_object(bucket, object, headers) 206 | if (res.status / 100) == 2: 207 | print "delete_object OK" 208 | else: 209 | print "delete_object ERROR" 210 | print sep 211 | 212 | #delete_bucket(self, bucket): 213 | res = oss.delete_bucket(bucket) 214 | if (res.status / 100) == 2: 215 | print "delete bucket ", bucket, "OK" 216 | else: 217 | print "delete bucket ", bucket, "ERROR" 218 | 219 | print sep 220 | 221 | 222 | -------------------------------------------------------------------------------- /oss/oss_xml_handler.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | ## ossbrowser 4 | ## Author: melory 5 | ## Email:imsrch@melory.me 6 | ## License: GPL Version 2 7 | 8 | from xml.dom import minidom 9 | 10 | def get_tag_text(element, tag): 11 | nodes = element.getElementsByTagName(tag) 12 | if len(nodes) == 0: 13 | return "" 14 | else: 15 | node = nodes[0] 16 | rc = "" 17 | for node in node.childNodes: 18 | if node.nodeType in ( node.TEXT_NODE, node.CDATA_SECTION_NODE): 19 | rc = rc + node.data 20 | return rc 21 | 22 | class ErrorXml: 23 | def __init__(self, xml_string): 24 | self.xml = minidom.parseString(xml_string) 25 | self.code = get_tag_text(self.xml, 'Code') 26 | self.msg = get_tag_text(self.xml, 'Message') 27 | self.resource = get_tag_text(self.xml, 'Resource') 28 | self.request_id = get_tag_text(self.xml, 'RequestId') 29 | self.host_id = get_tag_text(self.xml, 'HostId') 30 | 31 | def show(self): 32 | print "Code: %s\nMessage: %s\nResource: %s\nRequestId: %s \nHostId: %s" % (self.code, self.msg, self.resource, self.request_id, self.host_id) 33 | 34 | class Owner: 35 | def __init__(self, xml_element): 36 | self.element = xml_element 37 | self.id = get_tag_text(self.element, "ID") 38 | self.display_name = get_tag_text(self.element, "DisplayName") 39 | 40 | def show(self): 41 | print "ID: %s\nDisplayName: %s" % (self.id, self.display_name) 42 | 43 | class Bucket: 44 | def __init__(self, xml_element): 45 | self.element = xml_element 46 | self.name = get_tag_text(self.element, "Name") 47 | self.creation_date = get_tag_text(self.element, "CreationDate") 48 | 49 | def show(self): 50 | print "Name: %s\nCreationDate: %s" % (self.name, self.creation_date) 51 | 52 | class GetServiceXml: 53 | def __init__(self, xml_string): 54 | self.xml = minidom.parseString(xml_string) 55 | self.owner = Owner(self.xml.getElementsByTagName('Owner')[0]) 56 | self.buckets = self.xml.getElementsByTagName('Bucket') 57 | self.bucket_list = [] 58 | for b in self.buckets: 59 | self.bucket_list.append(Bucket(b)) 60 | 61 | def show(self): 62 | print "Owner:" 63 | self.owner.show() 64 | print "\nBucket list:" 65 | for b in self.bucket_list: 66 | b.show() 67 | print "" 68 | 69 | def list(self): 70 | bl = [] 71 | for b in self.bucket_list: 72 | bl.append((b.name, b.creation_date)) 73 | return bl 74 | 75 | class Content: 76 | def __init__(self, xml_element): 77 | self.element = xml_element 78 | self.key = get_tag_text(self.element, "Key") 79 | self.last_modified = get_tag_text(self.element, "LastModified") 80 | self.etag = get_tag_text(self.element, "ETag") 81 | self.size = get_tag_text(self.element, "Size") 82 | self.owner = Owner(self.element.getElementsByTagName('Owner')[0]) 83 | self.storage_class = get_tag_text(self.element, "StorageClass") 84 | 85 | def show(self): 86 | print "Key: %s\nLastModified: %s\nETag: %s\nSize: %s\nStorageClass: %s" % (self.key, self.last_modified, self.etag, self.size, self.storage_class) 87 | self.owner.show() 88 | 89 | class Part: 90 | def __init__(self, xml_element): 91 | self.element = xml_element 92 | self.part_num = get_tag_text(self.element, "PartNumber") 93 | self.object_name = get_tag_text(self.element, "PartName") 94 | self.object_size = get_tag_text(self.element, "PartSize") 95 | self.etag = get_tag_text(self.element, "ETag") 96 | 97 | def show(self): 98 | print "PartNumber: %s\nPartName: %s\nPartSize: %s\nETag: %s\n" % (self.part_num, self.object_name, self.object_size, self.etag) 99 | 100 | class PostObjectGroupXml: 101 | def __init__(self, xml_string): 102 | self.xml = minidom.parseString(xml_string) 103 | self.bucket = get_tag_text(self.xml, 'Bucket') 104 | self.key = get_tag_text(self.xml, 'Key') 105 | self.size = get_tag_text(self.xml, 'Size') 106 | self.etag = get_tag_text(self.xml, "ETag") 107 | 108 | def show(self): 109 | print "Post Object Group, Bucket: %s\nKey: %s\nSize: %s\nETag: %s" % (self.bucket, self.key, self.size, self.etag) 110 | 111 | class GetObjectGroupIndexXml: 112 | def __init__(self, xml_string): 113 | self.xml = minidom.parseString(xml_string) 114 | self.bucket = get_tag_text(self.xml, 'Bucket') 115 | self.key = get_tag_text(self.xml, 'Key') 116 | self.etag = get_tag_text(self.xml, 'Etag') 117 | self.file_length = get_tag_text(self.xml, 'FileLength') 118 | self.index_list = [] 119 | index_lists = self.xml.getElementsByTagName('Part') 120 | for i in index_lists: 121 | self.index_list.append(Part(i)) 122 | 123 | def list(self): 124 | index_list = [] 125 | for i in self.index_list: 126 | index_list.append((i.part_num, i.object_name, i.object_size, i.etag)) 127 | return index_list 128 | 129 | def show(self): 130 | print "Bucket: %s\nObject: %s\nEtag: %s\nObjectSize: %s" % (self.bucket, self.key, self.etag, self.file_length) 131 | print "\nPart list:" 132 | for p in self.index_list: 133 | p.show() 134 | 135 | class GetBucketXml: 136 | def __init__(self, xml_string): 137 | self.xml = minidom.parseString(xml_string) 138 | self.name = get_tag_text(self.xml, 'Name') 139 | self.prefix = get_tag_text(self.xml, 'Prefix') 140 | self.marker = get_tag_text(self.xml, 'Marker') 141 | self.nextmarker = get_tag_text(self.xml, 'NextMarker') 142 | self.maxkeys = get_tag_text(self.xml, 'MaxKeys') 143 | self.delimiter = get_tag_text(self.xml, 'Delimiter') 144 | self.is_truncated = get_tag_text(self.xml, 'IsTruncated') 145 | 146 | self.prefix_list = [] 147 | prefixes = self.xml.getElementsByTagName('CommonPrefixes') 148 | for p in prefixes: 149 | tag_txt = get_tag_text(p, "Prefix") 150 | self.prefix_list.append(tag_txt) 151 | 152 | self.content_list = [] 153 | contents = self.xml.getElementsByTagName('Contents') 154 | for c in contents: 155 | self.content_list.append(Content(c)) 156 | 157 | def show(self): 158 | print "Name: %s\nPrefix: %s\nMarker: %s\nNextMarker: %s\nMaxKeys: %s\nDelimiter: %s\nIsTruncated: %s" % (self.name, self.prefix, self.marker, self.nextmarker, self.maxkeys, self.delimiter, self.is_truncated) 159 | print "\nPrefix list:" 160 | for p in self.prefix_list: 161 | print p 162 | print "\nContent list:" 163 | for c in self.content_list: 164 | c.show() 165 | print "" 166 | 167 | def list(self): 168 | cl = [] 169 | pl = [] 170 | for c in self.content_list: 171 | cl.append((c.key, c.last_modified, c.etag, c.size, c.owner.id, c.owner.display_name, c.storage_class)) 172 | for p in self.prefix_list: 173 | pl.append(p) 174 | 175 | return (cl, pl) 176 | 177 | class GetBucketAclXml: 178 | def __init__(self, xml_string): 179 | self.xml = minidom.parseString(xml_string) 180 | if len(self.xml.getElementsByTagName('Owner')) != 0: 181 | self.owner = Owner(self.xml.getElementsByTagName('Owner')[0]) 182 | else: 183 | self.owner = "" 184 | self.grant = get_tag_text(self.xml, 'Grant') 185 | 186 | def show(self): 187 | print "Owner Name: %s\nOwner ID: %s\nGrant: %s" % (self.owner.id, self.owner.display_name, self.grant) 188 | 189 | def test_get_bucket_xml(): 190 | body = "sweet-memoryIMGIMG_01000falseIMG_2744.JPG2011-03-04T06:20:37.000Z"a56047f218618a43a9b1c2dca2d8c592"2207782cfd76976de6f4c6f4e05fcd02680c4ca619428123681589efcb203f29dce924sanbo_ustcSTANDARDIMG_2745.JPG2011-03-04T06:20:39.000Z"511c0b52911bcd667338103c385741af"2446122cfd76976de6f4c6f4e05fcd02680c4ca619428123681589efcb203f29dce924sanbo_ustcSTANDARDIMG_3344.JPG2011-03-04T06:20:48.000Z"4ea11d796ecc742b216864dcf5dfd193"2292112cfd76976de6f4c6f4e05fcd02680c4ca619428123681589efcb203f29dce924sanbo_ustcSTANDARDIMG_3387.JPG2011-03-04T06:20:53.000Z"c32b5568ae4fb0a3421f0daba25ecfd4"4600622cfd76976de6f4c6f4e05fcd02680c4ca619428123681589efcb203f29dce924sanbo_ustcSTANDARDIMG_3420.JPG2011-03-04T06:20:25.000Z"edf010d2a8a4877ce0362b245fcc963b"1749732cfd76976de6f4c6f4e05fcd02680c4ca619428123681589efcb203f29dce924sanbo_ustcSTANDARD中文.case2011-03-04T06:20:26.000Z"7fd64eec21799ef048ed827cf6098f06"2081342cfd76976de6f4c6f4e05fcd02680c4ca619428123681589efcb203f29dce924sanbo_ustcSTANDARD" 191 | h = GetBucketXml(body) 192 | h.show() 193 | (fl, pl) = h.list() 194 | print "\nfile_list: ", fl 195 | print "prefix list: ", pl 196 | 197 | def test_get_service_xml(): 198 | body = "2cfd76976de6f4c6f4e05fcd02680c4ca619428123681589efcb203f29dce924sanbo_ustc360buy2011-03-04T09:25:37.000Zaliyun-test-test2011-04-11T12:24:06.000Zirecoffee2011-03-04T06:14:56.000Zsweet-memory2011-04-12T11:56:04.000Z" 199 | h = GetServiceXml(body) 200 | h.show() 201 | print "\nbucket list: ", h.list() 202 | 203 | def test_get_bucket_acl_xml(): 204 | body = '61155b1e39dbca1d0d0f3c7faa32d9e8e9a90a9cd86edbd27d8eed5d0ad8ce82megjianpublic-read-write' 205 | h = GetBucketAclXml(body) 206 | h.show() 207 | 208 | def test_get_object_group_xml(): 209 | body = 'ut_test_post_object_group ut_test_post_object_group "91E8503F4DA1324E28434AA6B6E20D15"1073741824 1 4d37380c7149508bedf78dc7c5c68f55_test_post_object_group.txt_110485760"A957A9F1EF44ED7D40CD5C738D113509"27aa26b8da263589e875d179b87642691_test_post_object_group.txt_210485760"A957A9F1EF44ED7D40CD5C738D113509"328b0c8a9bd69469f76cd102d6e1b0f03_test_post_object_group.txt_310485760"A957A9F1EF44ED7D40CD5C738D113509"' 210 | h = GetObjectGroupIndexXml(body) 211 | h.show() 212 | 213 | if __name__ == "__main__": 214 | test_get_bucket_xml() 215 | test_get_service_xml() 216 | test_get_bucket_acl_xml() 217 | test_get_object_group_xml() 218 | -------------------------------------------------------------------------------- /oss/oss_util.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | ## ossbrowser 4 | ## Author: melory 5 | ## Email:imsrch@melory.me 6 | ## License: GPL Version 2 7 | 8 | import urllib 9 | import base64 10 | import hmac 11 | import time 12 | from hashlib import sha1 as sha 13 | import os 14 | import md5 15 | import StringIO 16 | from oss_xml_handler import * 17 | from threading import Thread 18 | 19 | DEBUG = False 20 | 21 | self_define_header_prefix = "x-oss-" 22 | 23 | class User: 24 | def __init__(self, user_name = "", access_id = "", secret_access_key = ""): 25 | self.user_name = user_name 26 | self.access_id = access_id 27 | self.secret_access_key = secret_access_key 28 | 29 | def show(self): 30 | print (('user_name:(%s) access_id:(%s) secret_access_key(%s)')) % (self.user_name, 31 | self.access_id, self.secret_access_key) 32 | 33 | 34 | def _format_header(headers = {}): 35 | ''' 36 | format the headers that self define 37 | convert the self define headers to lower. 38 | ''' 39 | tmp_headers = {} 40 | for k in headers.keys(): 41 | if k.lower().startswith(self_define_header_prefix): 42 | k_lower = k.lower() 43 | tmp_headers[k_lower] = headers[k] 44 | else: 45 | tmp_headers[k] = headers[k] 46 | return tmp_headers 47 | 48 | def get_assign(secret_access_key, method, headers = {}, resource="/", result = []): 49 | ''' 50 | Create the authorization for OSS based on header input. 51 | You should put it into "Authorization" parameter of header. 52 | ''' 53 | content_md5 = "" 54 | content_type = "" 55 | date = "" 56 | canonicalized_oss_headers = "" 57 | if DEBUG: 58 | print "secret_access_key", secret_access_key 59 | content_md5 = safe_get_element('Content-Md5', headers) 60 | content_type = safe_get_element('Content-Type', headers) 61 | date = safe_get_element('Date', headers) 62 | canonicalized_resource = resource 63 | tmp_headers = _format_header(headers) 64 | if len(tmp_headers) > 0: 65 | x_header_list = tmp_headers.keys() 66 | x_header_list.sort() 67 | for k in x_header_list: 68 | if k.startswith(self_define_header_prefix): 69 | canonicalized_oss_headers += k + ":" + tmp_headers[k] + "\n" 70 | 71 | string_to_sign = method + "\n" + content_md5 + "\n" + content_type + "\n" + date + "\n" + canonicalized_oss_headers + canonicalized_resource; 72 | result.append(string_to_sign) 73 | if DEBUG: 74 | print "string_to_sign", string_to_sign, "string_to_sign_size", len(string_to_sign) 75 | 76 | h = hmac.new(secret_access_key, string_to_sign, sha) 77 | 78 | return base64.encodestring(h.digest()).strip() 79 | 80 | def convert_header2map(header_list): 81 | header_map = {} 82 | for (a, b) in header_list: 83 | header_map[a] = b 84 | return header_map 85 | 86 | def safe_get_element(name, container): 87 | if name in container: 88 | return container[name] 89 | else: 90 | return "" 91 | 92 | def append_param(url, params): 93 | l = [] 94 | for k,v in params.items(): 95 | k = k.replace('_', '-') 96 | if k == 'maxkeys': 97 | k = 'max-keys' 98 | if isinstance(v, unicode): 99 | v = v.encode('utf-8') 100 | if v is not None and v != '': 101 | l.append('%s=%s' % (urllib.quote(k), urllib.quote(str(v)))) 102 | elif k == 'acl': 103 | l.append('%s' % (urllib.quote(k))) 104 | if len(l): 105 | url = url + '?' + '&'.join(l) 106 | return url 107 | 108 | def create_object_group_msg_xml(part_msg_list = []): 109 | xml_string = r'' 110 | for part in part_msg_list: 111 | if len(part) >= 3: 112 | if isinstance(part[1], unicode): 113 | file_path = part[1].encode('utf-8') 114 | else: 115 | file_path = part[1] 116 | xml_string += r'' 117 | xml_string += r'' + str(part[0]) + r'' 118 | xml_string += r'' + str(file_path) + r'' 119 | xml_string += r'"' + str(part[2]).upper() + r'"' 120 | xml_string += r'' 121 | else: 122 | print "the ", part, " in part_msg_list is not as expected!" 123 | return "" 124 | xml_string += r'' 125 | 126 | return xml_string 127 | 128 | def delete_all_parts_of_object_group(oss, bucket, object_group_name): 129 | res = oss.get_object_group_index(bucket, object_group_name) 130 | if res.status == 200: 131 | body = res.read() 132 | h = GetObjectGroupIndexXml(body) 133 | object_group_index = h.list() 134 | for i in object_group_index: 135 | if len(i) == 4 and len(i[1]) > 0: 136 | part_name = i[1].strip() 137 | res = oss.delete_object(bucket, part_name) 138 | if res.status != 204: 139 | print "delete part ", part_name, " in bucket:", bucket, " failed!" 140 | return False 141 | else: 142 | print "delete part ", part_name, " in bucket:", bucket, " ok" 143 | else: 144 | return False 145 | 146 | return True; 147 | 148 | def split_large_file(file_path, object_prefix = "", max_part_num = 1000, part_size = 10 * 1024 * 1024, buffer_size = 10 * 1024): 149 | parts_list = [] 150 | 151 | if os.path.isfile(file_path): 152 | file_size = os.path.getsize(file_path) 153 | 154 | if file_size > part_size * max_part_num: 155 | part_size = (file_size + max_part_num - file_size % max_part_num) / max_part_num 156 | 157 | part_order = 1 158 | fp = open(file_path, 'rb') 159 | fp.seek(os.SEEK_SET) 160 | 161 | total_split_len = 0 162 | part_num = file_size / part_size 163 | if file_size % part_size != 0: 164 | part_num += 1 165 | 166 | for i in range(0, part_num): 167 | left_len = part_size 168 | real_part_size = 0 169 | m = md5.new() 170 | offset = part_size * i 171 | while True: 172 | read_size = 0 173 | if left_len <= 0: 174 | break 175 | elif left_len < buffer_size: 176 | read_size = left_len 177 | else: 178 | read_size = buffer_size 179 | 180 | buffer_content = fp.read(read_size) 181 | m.update(buffer_content) 182 | real_part_size += len(buffer_content) 183 | 184 | left_len = left_len - read_size 185 | 186 | md5sum = m.hexdigest() 187 | 188 | temp_file_name = os.path.basename(file_path) + "_" + str(part_order) 189 | if len(object_prefix) == 0: 190 | file_name = sum_string(temp_file_name) + "_" + temp_file_name 191 | else: 192 | file_name = object_prefix + "/" + sum_string(temp_file_name) + "_" + temp_file_name 193 | part_msg = (part_order, file_name, md5sum, real_part_size, offset) 194 | total_split_len += real_part_size 195 | parts_list.append(part_msg) 196 | part_order += 1 197 | 198 | fp.close() 199 | else: 200 | print "ERROR! No file: ", file_path, ", please check." 201 | 202 | return parts_list 203 | 204 | def sumfile(fobj): 205 | '''Returns an md5 hash for an object with read() method.''' 206 | m = md5.new() 207 | while True: 208 | d = fobj.read(8096) 209 | if not d: 210 | break 211 | m.update(d) 212 | return m.hexdigest() 213 | 214 | def md5sum(fname): 215 | '''Returns an md5 hash for file fname, or stdin if fname is "-".''' 216 | if fname == '-': 217 | ret = sumfile(sys.stdin) 218 | else: 219 | try: 220 | f = file(fname, 'rb') 221 | except: 222 | return 'Failed to open file' 223 | ret = sumfile(f) 224 | f.close() 225 | return ret 226 | 227 | def md5sum2(filename, offset = 0, partsize = 0): 228 | m = md5.new() 229 | fp = open(filename, 'rb') 230 | if offset > os.path.getsize(filename): 231 | fp.seek(os.SEEK_SET, os.SEEK_END) 232 | else: 233 | fp.seek(offset) 234 | 235 | left_len = partsize 236 | BufferSize = 8 * 1024 237 | while True: 238 | if left_len <= 0: 239 | break 240 | elif left_len < BufferSize: 241 | buffer_content = fp.read(left_len) 242 | else: 243 | buffer_content = fp.read(BufferSize) 244 | m.update(buffer_content) 245 | 246 | left_len = left_len - len(buffer_content) 247 | md5sum = m.hexdigest() 248 | return md5sum 249 | 250 | def sum_string(content): 251 | f = StringIO.StringIO(content) 252 | md5sum = sumfile(f) 253 | f.close() 254 | return md5sum 255 | 256 | class PutObjectGroupWorker(Thread): 257 | def __init__(self, oss, bucket, file_path, part_msg_list): 258 | Thread.__init__(self) 259 | self.oss = oss 260 | self.bucket = bucket 261 | self.part_msg_list = part_msg_list 262 | self.file_path = file_path 263 | 264 | def run(self): 265 | for part in self.part_msg_list: 266 | if len(part) == 5: 267 | bucket = self.bucket 268 | file_name = part[1] 269 | object_name = file_name 270 | res = self.oss.head_object(bucket, object_name) 271 | if res.status == 200: 272 | header_map = convert_header2map(res.getheaders()) 273 | etag = safe_get_element("etag", header_map) 274 | md5 = part[2] 275 | if etag.replace('"', "").upper() == md5.upper(): 276 | continue 277 | 278 | partsize = part[3] 279 | offset = part[4] 280 | res = self.oss.put_object_from_file_given_pos(bucket, object_name, self.file_path, offset, partsize) 281 | if res.status != 200: 282 | print "upload ", file_name, "failed!"," ret is:", res.status 283 | print "headers", res.getheaders() 284 | else: 285 | print "ERROR! part", part , " is not as expected!" 286 | 287 | class GetAllObjects: 288 | def __init__(self): 289 | self.object_list = [] 290 | 291 | def get_object_in_bucket(self, oss, bucket="", marker="", prefix=""): 292 | object_list = [] 293 | res = oss.get_bucket(bucket, prefix, marker) 294 | body = res.read() 295 | hh = GetBucketXml(body) 296 | (fl, pl) = hh.list() 297 | if len(fl) != 0: 298 | for i in fl: 299 | if isinstance(i[0], unicode): 300 | object = i[0].encode('utf-8') 301 | object_list.append(object) 302 | 303 | if hh.is_truncated: 304 | marker = hh.nextmarker 305 | return (object_list, marker) 306 | 307 | 308 | def get_all_object_in_bucket(self, oss, bucket="", marker="", prefix=""): 309 | marker2 = "" 310 | while True: 311 | (object_list, marker) = self.get_object_in_bucket(oss, bucket, marker2, prefix) 312 | marker2 = marker 313 | if len(object_list) != 0: 314 | self.object_list.extend(object_list) 315 | 316 | if len(marker) == 0: 317 | break 318 | 319 | def clear_all_objects_in_bucket(oss_instance, bucket): 320 | ''' 321 | it will clean all objects in bucket, after that, it will delete this bucket. 322 | 323 | example: 324 | from oss_api import * 325 | host = "" 326 | id = "" 327 | key = "" 328 | oss_instance = OssAPI(host, id, key) 329 | bucket = "leopublicreadprivatewrite" 330 | if clear_all_objects_in_bucket(oss_instance, bucket): 331 | print "clean OK" 332 | else: 333 | print "clean Fail" 334 | ''' 335 | b = GetAllObjects() 336 | b.get_all_object_in_bucket(oss_instance, bucket) 337 | for i in b.object_list: 338 | res = oss_instance.delete_object(bucket, i) 339 | if (res.status / 100 != 2): 340 | print "clear_all_objects_in_bucket: delete object fail, ret is:", res.status, "object is: ", i 341 | return False 342 | else: 343 | print "clear_all_objects_in_bucket: delete object ok", "object is: ", i 344 | res = oss_instance.delete_bucket(bucket) 345 | if (res.status / 100 != 2 and res.status != 404): 346 | print "clear_all_objects_in_bucket: delete bucket fail, ret is:", res.status 347 | return False 348 | return True 349 | 350 | if __name__ == '__main__': 351 | pass 352 | -------------------------------------------------------------------------------- /oss/oss_cmd.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | ## ossbrowser 4 | ## Author: melory 5 | ## Email:imsrch@melory.me 6 | ## License: GPL Version 2 7 | 8 | from oss_api import * 9 | from oss_xml_handler import * 10 | 11 | msg_remind = " PLEASE USE # TO SEPERATE EACH INPUT PARAMETER" 12 | 13 | msg_quit = "q or quit, will quit out this console" 14 | msg_help = "h or help, will show the message again" 15 | 16 | #msg_get_service = "gs means get service , it lists all buckets that user has" 17 | msg_get_service = "gs means get service." 18 | example_get_service = "Input: gs" 19 | GET_SERVICE = "get service" 20 | 21 | #msg_put_bucket = "pb means put bucket, it creates bucket" 22 | msg_put_bucket = "pb means put bucket." 23 | example_put_bucket = "Input: pb#bucket||pb#bucket#acl||pb#bucket#acl#headers. " 24 | PUT_BUCKET = "put bucket" 25 | 26 | #msg_delete_bucket = "db means delete bucket, it deletes the bucket that user created" 27 | msg_delete_bucket = "db means delete bucket." 28 | example_delete_bucket = "Input: db#bucket" 29 | DELETE_BUCKET = "delete bucket" 30 | 31 | #msg_get_bucket = "gb means get bucket, it lists objects in the bucket" 32 | msg_get_bucket = "gb means get bucket." 33 | example_get_bucket = "Input: gb#bucket||gb#bucket#prefix||gb#bucket#prefix#marker||gb#bucket#prefix#marker#delimiter||gb#bucket#prefix#marker#delimiter#maxkeys||gb#bucket#prefix#marker#delimiter#maxkeys#headers" 34 | GET_BUCKET = "get bucket" 35 | 36 | #msg_get_bucket_acl = "gba means get bucket acl, it lists bucket acl" 37 | msg_get_bucket_acl = "gba means get bucket acl." 38 | example_get_bucket_acl = "Input: gba#bucket" 39 | GET_BUCKET_ACL = "get bucket acl" 40 | 41 | #msg_put_bucket_acl = "pba means put bucket acl, it sets bucket acl" 42 | msg_put_bucket_acl = "pba means put bucket acl." 43 | example_put_bucket_acl = "Input: pba#bucket#acl, Example: pba#mybucket#public-read" 44 | PUT_BUCKET_ACL = "put bucket acl" 45 | 46 | #msg_put_object_with_data = "powd means put object with data, it puts object, its content from string data" 47 | msg_put_object_with_data = "powd means put object with data." 48 | example_put_object_with_data = "Input: powd#bucket#object#string_data||powd#bucket#object#string_data#content_type||powd#bucket#object#string_data#content_type#headers, Example: powd#mybucket#myobjcet#abcdefghijk#txt/plain" 49 | PUT_OBJECT_WITH_DATA = "put object with data" 50 | 51 | #msg_put_object_from_file = "poff means put object from file, it puts object , its content from file" 52 | msg_put_object_from_file = "poff means put object from file." 53 | example_put_object_from_file = "Input: poff#bucket#object#filename||poff#bucket#object#filename#content_type||poff#bucket#object#filename#content_type#headers, Example: poff#mybucket#myobjcet#myfilename" 54 | PUT_OBJECT_FROM_FILE = "put object from file" 55 | 56 | #msg_get_object = "go means get object, it gets object and print its content" 57 | msg_get_object = "go means get object." 58 | example_get_object = "Input: go#bucket#object||go#bucket#object#headers, Example: go#mybucket#myobjcet" 59 | GET_OBJECT = "get object" 60 | 61 | #msg_get_object_to_file = "gotf means get object to file, it gets object and save its content to file" 62 | msg_get_object_to_file = "gotf means get object to file." 63 | example_get_object_to_file = "Input: gotf#bucket#object#filename||go#bucket#object#filename#headers, Example: goff#mybucket#myobjcet#tofilename" 64 | GET_OBJECT_TO_FILE = "get object to file" 65 | 66 | #msg_delete_object = "do means delete object, it deletes object" 67 | msg_delete_object = "do means delete object." 68 | example_delete_object = "Input: do#bucket#object||do#bucket#object#headers, Example: do#mybucket#myobjcet" 69 | DELETE_OBJECT = "delete object" 70 | 71 | #msg_head_object = "ho means head object, it gets meta info of object" 72 | msg_head_object = "ho means head object." 73 | example_head_object = "Input: ho#bucket#object||ho#bucket#object#headers, Example: ho#mybucket#myobject" 74 | HEAD_OBJECT = "head object" 75 | 76 | def usage(): 77 | width = 40 78 | print "=> 1) ", msg_quit.ljust(width) 79 | print "=> 2) ", msg_help.ljust(width) 80 | print '***************************************' 81 | print "NOTE:acl in below means access control level, acl SHOULD be public-read, private, or public-read-write" 82 | print "headers and parameters below means dic, it SHOULD be dic like a=b c=d" 83 | print "content type means object type , it is used in headers, default is application/octet-stream" 84 | print '***************************************' 85 | print 'bucket operation' 86 | print "=> 3) ", msg_get_service.ljust(width), example_get_service 87 | print "=> 4) ", msg_put_bucket.ljust(width), example_put_bucket 88 | print "=> 5) ", msg_delete_bucket.ljust(width), example_delete_bucket 89 | print "=> 6) ", msg_get_bucket.ljust(width), example_get_bucket 90 | print "=> 7) ", msg_get_bucket_acl.ljust(width), example_get_bucket_acl 91 | print "=> 8) ", msg_put_bucket_acl.ljust(width), example_put_bucket_acl 92 | print '' 93 | print '***************************************' 94 | print 'objcet operation' 95 | print "=> 9) ", msg_put_object_with_data.ljust(width), example_put_object_with_data 96 | print "=>10) ", msg_put_object_from_file.ljust(width), example_put_object_from_file 97 | print "=>11) ", msg_get_object.ljust(width), example_get_object 98 | print "=>12) ", msg_get_object_to_file.ljust(width), example_get_object_to_file 99 | print "=>13) ", msg_delete_object.ljust(width), example_delete_object 100 | print "=>14) ", msg_head_object.ljust(width), example_head_object 101 | print '' 102 | 103 | def print_result(cmd, res): 104 | if res.status / 100 == 2: 105 | print cmd, "OK" 106 | else: 107 | print "error headers", res.getheaders() 108 | print "error body", res.read() 109 | print cmd, "Failed!" 110 | 111 | def check_input(cmd_str, cmd_list, min, max): 112 | if len(cmd_list) < min or len(cmd_list) > max: 113 | print "ERROR! ", cmd_str, "needs ", min, "-", max, " paramters" 114 | if example_map.has_key(cmd_str): 115 | print example_map[cmd_str] 116 | return False 117 | else: 118 | return True 119 | 120 | def get_cmd(cmd): 121 | string = cmd 122 | tmp_list = string.split('#') 123 | cmd_list = [] 124 | for i in tmp_list: 125 | if len(i.strip()) != 0: 126 | cmd_list.append(i.strip()) 127 | return cmd_list 128 | 129 | def transfer_string_to_dic(string): 130 | list = string.split() 131 | dic = {} 132 | for entry in list: 133 | key, val = entry.split('=') 134 | dic[key.strip()] = val.strip() 135 | print dic 136 | return dic 137 | 138 | if __name__ == "__main__": 139 | import sys 140 | host = "" 141 | user_name = "" 142 | access_id = "" 143 | secret_access_key = "" 144 | remind = "the method you input is not supported! please input h or help to check method supported" 145 | 146 | if len(sys.argv) != 4: 147 | print '***************************************' 148 | print 'Please input the parameters like below' 149 | print '***************************************' 150 | print 'python ', __file__,' host access_id access_key' 151 | print 'host is the ip address with port of oss apache server . For Example: 10.249.105.22:8080' 152 | print 'access_id is public key that oss server provided. For Example: 84792jahfdsah+=' 153 | print 'access_key is private secret key that oss server provided. For Example: atdh+=flahmzhha=+' 154 | print '***************************************' 155 | print '' 156 | print '' 157 | exit() 158 | else: 159 | host = sys.argv[1] 160 | access_id = sys.argv[2] 161 | secret_access_key = sys.argv[3] 162 | 163 | oss = OssAPI(host, access_id, secret_access_key) 164 | usage() 165 | bucketname = "" 166 | 167 | example_map = {} 168 | example_map[PUT_BUCKET] = example_put_bucket 169 | example_map[PUT_BUCKET_ACL] = example_put_bucket_acl 170 | example_map[GET_SERVICE] = example_get_service 171 | example_map[GET_BUCKET] = example_get_bucket 172 | example_map[DELETE_BUCKET] = example_delete_bucket 173 | example_map[GET_BUCKET_ACL] = example_get_bucket_acl 174 | example_map[PUT_OBJECT_WITH_DATA] = example_put_object_with_data 175 | example_map[PUT_OBJECT_FROM_FILE] = example_put_object_from_file 176 | example_map[GET_OBJECT] = example_get_object 177 | example_map[GET_OBJECT_TO_FILE] = example_get_object_to_file 178 | example_map[DELETE_OBJECT] = example_delete_object 179 | example_map[HEAD_OBJECT] = example_head_object 180 | 181 | while True: 182 | cmd = raw_input(">>") 183 | cmd = cmd.strip() 184 | cmd_list = get_cmd(cmd) 185 | if ("q" == cmd.lower() or "quit" == cmd.lower()): 186 | break 187 | elif ("h" == cmd.lower() or "help" == cmd.lower()): 188 | usage() 189 | elif len(cmd_list) > 0: 190 | if cmd_list[0].lower() == "pb": 191 | cmd_str = PUT_BUCKET 192 | min = 2 193 | max = 4 194 | if not check_input(cmd_str, cmd_list, min, max): 195 | pass 196 | else: 197 | print "cmd", cmd_list 198 | bucketname = cmd_list[1] 199 | if len(cmd_list) == 2: 200 | res = oss.put_bucket(cmd_list[1]) 201 | elif len(cmd_list) == 3: 202 | res = oss.put_bucket(cmd_list[1], cmd_list[2]) 203 | elif len(cmd_list) == 4: 204 | dic = transfer_string_to_dic(cmd_list[3]) 205 | res = oss.put_bucket(cmd_list[1], cmd_list[2], dic) 206 | print_result(cmd_str, res) 207 | 208 | elif cmd_list[0] == "pba": 209 | cmd_str = PUT_BUCKET_ACL 210 | min = 3 211 | max = 3 212 | if not check_input(cmd_str, cmd_list, min, max): 213 | pass 214 | else: 215 | print "cmd", cmd_list 216 | bucketname = cmd_list[1] 217 | res = oss.put_bucket(cmd_list[1], cmd_list[2]) 218 | print_result(cmd_str, res) 219 | 220 | elif cmd_list[0].lower() == "gs": 221 | cmd_str = GET_SERVICE 222 | min = 1 223 | max = 1 224 | if not check_input(cmd_str, cmd_list, min, max): 225 | pass 226 | else: 227 | print "cmd", cmd_list 228 | if len(cmd_list) == 1: 229 | res = oss.get_service() 230 | print_result(cmd_str, res) 231 | if (res.status / 100) == 2: 232 | body = res.read() 233 | h = GetServiceXml(body) 234 | print "bucket list size is: ", len(h.list()) 235 | print "bucket list is: " 236 | for i in h.list(): 237 | print i 238 | 239 | elif cmd_list[0].lower() == "gb": 240 | cmd_str = GET_BUCKET 241 | min = 2 242 | max = 7 243 | if not check_input(cmd_str, cmd_list, min, max): 244 | pass 245 | else: 246 | print "cmd", cmd_list 247 | bucketname = cmd_list[1] 248 | #get_bucket(bucket, prefix='', marker='', delimiter='', maxkeys='', headers = {}): 249 | if len(cmd_list) == 2: 250 | res = oss.get_bucket(cmd_list[1]) 251 | elif len(cmd_list) == 3: 252 | res = oss.get_bucket(cmd_list[1], cmd_list[2]) 253 | elif len(cmd_list) == 4: 254 | res = oss.get_bucket(cmd_list[1], cmd_list[2], cmd_list[3]) 255 | elif len(cmd_list) == 5: 256 | res = oss.get_bucket(cmd_list[1], cmd_list[2], cmd_list[3], cmd_list[4]) 257 | elif len(cmd_list) == 6: 258 | res = oss.get_bucket(cmd_list[1], cmd_list[2], cmd_list[3], cmd_list[4], cmd_list[5]) 259 | elif len(cmd_list) == 7: 260 | dic = transfer_string_to_dic(cmd_list[6]) 261 | res = oss.get_bucket(cmd_list[1], cmd_list[2], cmd_list[3], cmd_list[4], cmd_list[5], dic) 262 | print_result(cmd_str, res) 263 | if (res.status / 100) == 2: 264 | body = res.read() 265 | hh = GetBucketXml(body) 266 | (fl, pl) = hh.list() 267 | print "prefix list size is: ", len(pl) 268 | print "prefix listis: " 269 | for i in pl: 270 | print i 271 | print "file list size is: ", len(fl) 272 | print "file list is: " 273 | for i in fl: 274 | print i 275 | 276 | elif cmd_list[0].lower() == "db": 277 | cmd_str = DELETE_BUCKET 278 | print cmd_str 279 | min = 2 280 | max = 2 281 | if not check_input(cmd_str, cmd_list, min, max): 282 | pass 283 | else: 284 | print "cmd", cmd_list 285 | if len(cmd_list) == 2: 286 | res = oss.delete_bucket(cmd_list[1]) 287 | print_result(cmd_str, res) 288 | 289 | elif cmd_list[0].lower() == "gba": 290 | cmd_str = GET_BUCKET_ACL 291 | print cmd_str 292 | 293 | min = 2 294 | max = 2 295 | if not check_input(cmd_str, cmd_list, min, max): 296 | pass 297 | else: 298 | print "cmd", cmd_list 299 | bucketname = cmd_list[1] 300 | if len(cmd_list) == 2: 301 | res = oss.get_bucket_acl(cmd_list[1]) 302 | print_result(cmd_str, res) 303 | 304 | if (res.status / 100) == 2: 305 | body = res.read() 306 | h = GetBucketAclXml(body) 307 | print "bucket ", bucketname, " acl is: ", h.grant 308 | 309 | elif cmd_list[0].lower() == "powd" or cmd_list[0].lower() == "poff": 310 | if cmd_list[0].lower() == "powd": 311 | cmd_str = PUT_OBJECT_WITH_DATA 312 | else: 313 | cmd_str = PUT_OBJECT_FROM_FILE 314 | min = 4 315 | max = 6 316 | if not check_input(cmd_str, cmd_list, min, max): 317 | pass 318 | else: 319 | print "cmd", cmd_list 320 | bucketname = cmd_list[1] 321 | if isinstance(cmd_list[2], str): 322 | tmp = unicode(cmd_list[2], 'utf-8') 323 | cmd_list[2] = tmp 324 | if len(cmd_list) == 4: 325 | if cmd_list[0].lower() == "powd": 326 | res = oss.put_object_with_data(cmd_list[1], cmd_list[2], cmd_list[3]) 327 | else: 328 | res = oss.put_object_from_file(cmd_list[1], cmd_list[2], cmd_list[3]) 329 | elif len(cmd_list) == 5: 330 | if cmd_list[0].lower() == "powd": 331 | res = oss.put_object_with_data(cmd_list[1], cmd_list[2], cmd_list[3], cmd_list[4]) 332 | else: 333 | res = oss.put_object_from_file(cmd_list[1], cmd_list[2], cmd_list[3], cmd_list[4]) 334 | elif len(cmd_list) == 6: 335 | if cmd_list[0].lower() == "powd": 336 | dic = transfer_string_to_dic(cmd_list[5]) 337 | res = oss.put_object_with_data(cmd_list[1], cmd_list[2], cmd_list[3], cmd_list[4], dic) 338 | else: 339 | dic = transfer_string_to_dic(cmd_list[5]) 340 | res = oss.put_object_from_file(cmd_list[1], cmd_list[2], cmd_list[3], cmd_list[4], dic) 341 | print_result(cmd_str, res) 342 | 343 | elif cmd_list[0].lower() == "go": 344 | cmd_str = GET_OBJECT 345 | min = 3 346 | max = 4 347 | if not check_input(cmd_str, cmd_list, min, max): 348 | pass 349 | else: 350 | print "cmd", cmd_list 351 | bucketname = cmd_list[1] 352 | if len(cmd_list) == 3: 353 | res = oss.get_object(cmd_list[1], cmd_list[2]) 354 | elif len(cmd_list) == 4: 355 | dic = transfer_string_to_dic(cmd_list[3]) 356 | res = oss.get_object(cmd_list[1], cmd_list[2], dic) 357 | print_result(cmd_str, res) 358 | if res.status / 100 == 2: 359 | print res.read() 360 | 361 | elif cmd_list[0].lower() == "gotf": 362 | cmd_str = GET_OBJECT_TO_FILE 363 | min = 4 364 | max = 5 365 | if not check_input(cmd_str, cmd_list, min, max): 366 | pass 367 | else: 368 | print "cmd", cmd_list 369 | bucketname = cmd_list[1] 370 | if len(cmd_list) == 4: 371 | res = oss.get_object_to_file(cmd_list[1], cmd_list[2], cmd_list[3]) 372 | elif len(cmd_list) == 5: 373 | dic = transfer_string_to_dic(cmd_list[4]) 374 | res = oss.get_object_to_file(cmd_list[1], cmd_list[2], cmd_list[3], dic) 375 | print_result(cmd_str, res) 376 | 377 | elif cmd_list[0].lower() == "do": 378 | cmd_str = DELETE_OBJECT 379 | min = 3 380 | max = 4 381 | if not check_input(cmd_str, cmd_list, min, max): 382 | pass 383 | else: 384 | print "cmd", cmd_list 385 | bucketname = cmd_list[1] 386 | if len(cmd_list) == 3: 387 | res = oss.delete_object(cmd_list[1], cmd_list[2]) 388 | elif len(cmd_list) == 4: 389 | dic = transfer_string_to_dic(cmd_list[3]) 390 | res = oss.delete_object(cmd_list[1], cmd_list[2], dic) 391 | print_result(cmd_str, res) 392 | 393 | elif cmd_list[0].lower() == "ho": 394 | cmd_str = HEAD_OBJECT 395 | min = 3 396 | max = 4 397 | if not check_input(cmd_str, cmd_list, min, max): 398 | pass 399 | else: 400 | print "cmd", cmd_list 401 | bucketname = cmd_list[1] 402 | if len(cmd_list) == 3: 403 | res = oss.head_object(cmd_list[1], cmd_list[2]) 404 | elif len(cmd_list) == 4: 405 | dic = transfer_string_to_dic(cmd_list[3]) 406 | res = oss.head_object(cmd_list[1], cmd_list[2], dic) 407 | print_result(cmd_str, res) 408 | if res.status / 100 == 2: 409 | print res.getheaders() 410 | else: 411 | print remind 412 | else: 413 | print remind 414 | -------------------------------------------------------------------------------- /oss/oss_api.py: -------------------------------------------------------------------------------- 1 | #coding=utf8 2 | 3 | # Copyright (c) 2011, Alibaba Cloud Computing 4 | # All rights reserved. 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a 7 | # copy of this software and associated documentation files (the 8 | # "Software"), to deal in the Software without restriction, including 9 | # without limitation the rights to use, copy, modify, merge, publish, dis- 10 | # tribute, sublicense, and/or sell copies of the Software, and to permit 11 | # persons to whom the Software is furnished to do so, subject to the fol- 12 | # lowing conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included 15 | # in all copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 18 | # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- 19 | # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 20 | # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 21 | # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 23 | # IN THE SOFTWARE. 24 | 25 | import httplib 26 | import time 27 | import base64 28 | import hmac 29 | import os 30 | import urllib 31 | import StringIO 32 | from oss_util import * 33 | from hashlib import sha1 as sha 34 | 35 | class OssAPI: 36 | ''' 37 | A simple OSS API 38 | ''' 39 | DefaultContentType = 'application/octet-stream' 40 | SendBufferSize = 8192 41 | GetBufferSize = 1024*1024*10 42 | 43 | def __init__(self, host, access_id, secret_access_key=""): 44 | self.host = host 45 | self.access_id = access_id 46 | self.secret_access_key = secret_access_key 47 | 48 | def sign_url_auth_with_expire_time(self, method, url, headers = {}, resource="/", timeout = 60): 49 | ''' 50 | Create the authorization for OSS based on the input method, url, body and headers 51 | :type method: string 52 | :param method: one of PUT, GET, DELETE, HEAD 53 | 54 | :type url: string 55 | :param:HTTP address of bucket or object, eg: http://HOST/bucket/object 56 | 57 | :type headers: dict 58 | :param: HTTP header 59 | 60 | :type resource: string 61 | :param:path of bucket or object, eg: /bucket/ or /bucket/object 62 | 63 | :type timeout: int 64 | :param 65 | 66 | Returns: 67 | signature url. 68 | ''' 69 | 70 | send_time= safe_get_element('Date', headers) 71 | if len(send_time) == 0: 72 | send_time = str(int(time.time()) + timeout) 73 | headers['Date'] = send_time 74 | auth_value = get_assign(self.secret_access_key, method, headers, resource) 75 | params = {"OSSAccessKeyId":self.access_id, "Expires":str(send_time), "Signature":auth_value} 76 | return append_param(url, params) 77 | 78 | def _create_sign_for_normal_auth(self, method, headers = {}, resource="/"): 79 | ''' 80 | NOT public API 81 | Create the authorization for OSS based on header input. 82 | it should be put into "Authorization" parameter of header. 83 | 84 | :type method: string 85 | :param:one of PUT, GET, DELETE, HEAD 86 | 87 | :type headers: dict 88 | :param: HTTP header 89 | 90 | :type resource: string 91 | :param:path of bucket or object, eg: /bucket/ or /bucket/object 92 | 93 | Returns: 94 | signature string 95 | 96 | ''' 97 | 98 | auth_value = "OSS " + self.access_id + ":" + get_assign(self.secret_access_key, method, headers, resource) 99 | return auth_value 100 | 101 | def bucket_operation(self, method, bucket, headers={}, params={}): 102 | ''' 103 | Send bucket operation request 104 | 105 | :type method: string 106 | :param method: one of PUT, GET, DELETE, HEAD 107 | 108 | :type bucket: string 109 | :param 110 | 111 | :type headers: dict 112 | :param: HTTP header 113 | 114 | :type params: dict 115 | :param: parameters that will be appeded at the end of resource 116 | 117 | Returns: 118 | HTTP Response 119 | 120 | ''' 121 | 122 | url = append_param("/" + bucket + "/", params) 123 | date = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime()) 124 | # Create REST header 125 | headers['Date'] = date 126 | headers['Host'] = self.host 127 | if params.has_key('acl'): 128 | resource = "/" + bucket + "/?acl" 129 | else: 130 | resource = "/" + bucket + "/" 131 | if "" != self.secret_access_key and "" != self.access_id: 132 | headers['Authorization'] = self._create_sign_for_normal_auth(method, headers, resource) 133 | elif "" != self.access_id: 134 | headers['Authorization'] = self.access_id 135 | 136 | conn = httplib.HTTPConnection(self.host) 137 | conn.request(method, url, "", headers) 138 | return conn.getresponse() 139 | 140 | def object_operation(self, method, bucket, object, headers = {}, data=""): 141 | ''' 142 | Send Object operation request 143 | 144 | :type method: string 145 | :param method: one of PUT, GET, DELETE, HEAD 146 | 147 | :type bucket: string 148 | :param 149 | 150 | :type object: string 151 | :param 152 | 153 | :type headers: dict 154 | :param: HTTP header 155 | 156 | :type data: string 157 | :param 158 | 159 | Returns: 160 | HTTP Response 161 | ''' 162 | 163 | if isinstance(object, unicode): 164 | object = object.encode('utf-8') 165 | resource = "/" + bucket + "/" 166 | resource = resource.encode('utf-8') + object 167 | object = urllib.quote(object) 168 | url = "/" + bucket + "/" + object 169 | date = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime()) 170 | 171 | # Create REST header 172 | headers['Date'] = date 173 | headers['Host'] = self.host 174 | if "" != self.secret_access_key and "" != self.access_id: 175 | headers['Authorization'] = self._create_sign_for_normal_auth(method, headers, resource) 176 | elif "" != self.access_id: 177 | headers['Authorization'] = self.access_id 178 | 179 | conn = httplib.HTTPConnection(self.host) 180 | conn.request(method, url, data, headers) 181 | return conn.getresponse() 182 | 183 | def get_service(self): 184 | ''' 185 | List all buckets of user 186 | ''' 187 | return self.list_all_my_buckets() 188 | 189 | def list_all_my_buckets(self): 190 | ''' 191 | List all buckets of user 192 | ''' 193 | 194 | method = "GET" 195 | url = "/" 196 | date = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime()) 197 | headers = {} 198 | 199 | headers['Date'] = date 200 | headers['Host'] = self.host 201 | resource = "/" 202 | if "" != self.secret_access_key and "" != self.access_id: 203 | headers['Authorization'] = self._create_sign_for_normal_auth(method, headers, resource) 204 | elif "" != self.access_id: 205 | headers['Authorization'] = self.access_id 206 | 207 | conn = httplib.HTTPConnection(self.host) 208 | conn.request(method, url, "", headers) 209 | return conn.getresponse() 210 | 211 | def get_bucket_acl(self, bucket): 212 | ''' 213 | Get Access Control Level of bucket 214 | 215 | :type bucket: string 216 | :param 217 | 218 | Returns: 219 | HTTP Response 220 | ''' 221 | headers = {} 222 | params = {} 223 | params['acl'] = '' 224 | return self.bucket_operation("GET", bucket, headers, params) 225 | 226 | def get_bucket(self, bucket, prefix='', marker='', delimiter='', maxkeys='', headers = {}): 227 | ''' 228 | List object that in bucket 229 | ''' 230 | return self.list_bucket(bucket, prefix, marker, delimiter, maxkeys, headers) 231 | 232 | def list_bucket(self, bucket, prefix='', marker='', delimiter='', maxkeys='', headers = {}): 233 | ''' 234 | List object that in bucket 235 | 236 | :type bucket: string 237 | :param 238 | 239 | :type prefix: string 240 | :param 241 | 242 | :type marker: string 243 | :param 244 | 245 | :type delimiter: string 246 | :param 247 | 248 | :type maxkeys: string 249 | :param 250 | 251 | :type headers: dict 252 | :param: HTTP header 253 | 254 | Returns: 255 | HTTP Response 256 | ''' 257 | 258 | params = {} 259 | params['prefix'] = prefix 260 | params['marker'] = marker 261 | params['delimiter'] = delimiter 262 | params['max-keys'] = maxkeys 263 | return self.bucket_operation("GET", bucket, headers, params) 264 | 265 | def create_bucket(self, bucket, acl='', headers = {}): 266 | ''' 267 | Create bucket 268 | ''' 269 | return self.put_bucket(bucket, acl, headers) 270 | 271 | def put_bucket(self, bucket, acl='', headers = {}): 272 | ''' 273 | Create bucket 274 | 275 | :type bucket: string 276 | :param 277 | 278 | :type acl: string 279 | :param: one of private 280 | 281 | :type headers: dict 282 | :param: HTTP header 283 | 284 | Returns: 285 | HTTP Response 286 | ''' 287 | 288 | if acl != '': 289 | headers['x-oss-acl'] = acl 290 | return self.bucket_operation("PUT", bucket, headers) 291 | 292 | def delete_bucket(self, bucket): 293 | ''' 294 | Delete bucket 295 | 296 | :type bucket: string 297 | :param 298 | 299 | Returns: 300 | HTTP Response 301 | ''' 302 | 303 | return self.bucket_operation("DELETE", bucket) 304 | 305 | def put_object_with_data(self, bucket, object, input_content, content_type=DefaultContentType, headers = {}): 306 | ''' 307 | Put object into bucket, the content of object is from input_content 308 | ''' 309 | return self.put_object_from_string(bucket, object, input_content, content_type, headers) 310 | 311 | def put_object_from_string(self, bucket, object, input_content, content_type=DefaultContentType, headers = {}): 312 | ''' 313 | Put object into bucket, the content of object is from input_content 314 | 315 | :type bucket: string 316 | :param 317 | 318 | :type object: string 319 | :param 320 | 321 | :type input_content: string 322 | :param 323 | 324 | :type content_type: string 325 | :param: the object content type that supported by HTTP 326 | 327 | :type headers: dict 328 | :param: HTTP header 329 | 330 | Returns: 331 | HTTP Response 332 | ''' 333 | 334 | headers['Content-Type'] = content_type 335 | headers['Content-Length'] = str(len(input_content)) 336 | fp = StringIO.StringIO(input_content) 337 | res = self.put_object_from_fp(bucket, object, fp, content_type, headers) 338 | fp.close() 339 | return res 340 | 341 | def _open_conn_to_put_object(self, bucket, object, filesize, content_type=DefaultContentType, headers = {}): 342 | ''' 343 | NOT public API 344 | Open a connectioon to put object 345 | 346 | :type bucket: string 347 | :param 348 | 349 | :type filesize: int 350 | :param 351 | 352 | :type object: string 353 | :param 354 | 355 | :type input_content: string 356 | :param 357 | 358 | :type content_type: string 359 | :param: the object content type that supported by HTTP 360 | 361 | :type headers: dict 362 | :param: HTTP header 363 | 364 | Returns: 365 | HTTP Response 366 | ''' 367 | 368 | method = "PUT" 369 | if isinstance(object, unicode): 370 | object = object.encode('utf-8') 371 | resource = "/" + bucket + "/" 372 | resource = resource.encode('utf-8') + object 373 | object = urllib.quote(object) 374 | url = "/" + bucket + "/" + object 375 | date = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime()) 376 | 377 | conn = httplib.HTTPConnection(self.host) 378 | conn.putrequest(method, url) 379 | headers["Content-Type"] = content_type 380 | headers["Content-Length"] = filesize 381 | headers["Date"] = date 382 | headers["Host"] = self.host 383 | headers["Expect"] = "100-Continue" 384 | for k in headers.keys(): 385 | conn.putheader(k, headers[k]) 386 | if "" != self.secret_access_key and "" != self.access_id: 387 | auth = self._create_sign_for_normal_auth(method, headers, resource) 388 | conn.putheader("Authorization", auth) 389 | conn.endheaders() 390 | return conn 391 | 392 | def put_object_from_file(self, bucket, object, filename, content_type=DefaultContentType, headers = {}): 393 | ''' 394 | put object into bucket, the content of object is read from file 395 | 396 | :type bucket: string 397 | :param 398 | 399 | :type object: string 400 | :param 401 | 402 | :type fllename: string 403 | :param: the name of the read file 404 | 405 | :type content_type: string 406 | :param: the object content type that supported by HTTP 407 | 408 | :type headers: dict 409 | :param: HTTP header 410 | 411 | Returns: 412 | HTTP Response 413 | ''' 414 | fp = open(filename, 'rb') 415 | res = self.put_object_from_fp(bucket, object, fp, content_type, headers) 416 | fp.close() 417 | return res 418 | 419 | def put_object_from_fp(self, bucket, object, fp, content_type=DefaultContentType, headers = {}): 420 | ''' 421 | Put object into bucket, the content of object is read from file pointer 422 | 423 | :type bucket: string 424 | :param 425 | 426 | :type object: string 427 | :param 428 | 429 | :type fp: file 430 | :param: the pointer of the read file 431 | 432 | :type content_type: string 433 | :param: the object content type that supported by HTTP 434 | 435 | :type headers: dict 436 | :param: HTTP header 437 | 438 | Returns: 439 | HTTP Response 440 | ''' 441 | 442 | fp.seek(os.SEEK_SET, os.SEEK_END) 443 | filesize = fp.tell() 444 | fp.seek(os.SEEK_SET) 445 | 446 | conn = self._open_conn_to_put_object(bucket, object, filesize, content_type, headers) 447 | l = fp.read(self.SendBufferSize) 448 | while len(l) > 0: 449 | conn.send(l) 450 | l = fp.read(self.SendBufferSize) 451 | return conn.getresponse() 452 | 453 | def get_object(self, bucket, object, headers = {}): 454 | ''' 455 | Get object 456 | 457 | :type bucket: string 458 | :param 459 | 460 | :type object: string 461 | :param 462 | 463 | :type headers: dict 464 | :param: HTTP header 465 | 466 | Returns: 467 | HTTP Response 468 | ''' 469 | 470 | return self.object_operation("GET", bucket, object, headers) 471 | 472 | def get_object_to_file(self, bucket, object, filename, headers = {}): 473 | ''' 474 | Get object and write the content of object into a file 475 | 476 | :type bucket: string 477 | :param 478 | 479 | :type object: string 480 | :param 481 | 482 | :type filename: string 483 | :param 484 | 485 | :type headers: dict 486 | :param: HTTP header 487 | 488 | Returns: 489 | HTTP Response 490 | ''' 491 | 492 | res = self.get_object(bucket, object, headers) 493 | f = file(filename, 'wb') 494 | data = "" 495 | while True: 496 | data = res.read(self.GetBufferSize) 497 | if len(data) != 0: 498 | f.write(data) 499 | else: 500 | break 501 | f.close() 502 | # TODO: get object with flow 503 | return res 504 | 505 | def delete_object(self, bucket, object, headers = {}): 506 | ''' 507 | Delete object 508 | 509 | :type bucket: string 510 | :param 511 | 512 | :type object: string 513 | :param 514 | 515 | :type headers: dict 516 | :param: HTTP header 517 | 518 | Returns: 519 | HTTP Response 520 | ''' 521 | 522 | return self.object_operation("DELETE", bucket, object, headers) 523 | 524 | def head_object(self, bucket, object, headers = {}): 525 | ''' 526 | Head object, to get the meta message of object without the content 527 | 528 | :type bucket: string 529 | :param 530 | 531 | :type object: string 532 | :param 533 | 534 | :type headers: dict 535 | :param: HTTP header 536 | 537 | Returns: 538 | HTTP Response 539 | ''' 540 | 541 | return self.object_operation("HEAD", bucket, object, headers) 542 | 543 | def post_object_group(self, bucket, object, object_group_msg_xml, headers = {}, params = {}): 544 | ''' 545 | Post object group, merge all objects in object_group_msg_xml into one object 546 | :type bucket: string 547 | :param 548 | 549 | :type object: string 550 | :param 551 | 552 | :type object_group_msg_xml: string 553 | :param: xml format string, like 554 | 555 | 556 | N 557 | objectN 558 | "47BCE5C74F589F4867DBD57E9CA9F808" 559 | 560 | 561 | :type headers: dict 562 | :param: HTTP header 563 | 564 | :type params: dict 565 | :param: parameters 566 | 567 | Returns: 568 | HTTP Response 569 | ''' 570 | method = "POST" 571 | url = "/" + bucket + "/" + object + "?group" 572 | date = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime()) 573 | # Create REST header 574 | headers['Date'] = date 575 | headers['Host'] = self.host 576 | headers['Content-Type'] = 'text/xml' 577 | headers['Content-Length'] = len(object_group_msg_xml) 578 | resource = "/" + bucket + "/" + object + "?group" 579 | if "" != self.secret_access_key and "" != self.access_id: 580 | headers['Authorization'] = self._create_sign_for_normal_auth(method, headers, resource) 581 | elif "" != self.access_id: 582 | headers['Authorization'] = self.access_id 583 | 584 | conn = httplib.HTTPConnection(self.host) 585 | conn.request(method, url, object_group_msg_xml, headers) 586 | return conn.getresponse() 587 | 588 | def get_object_group_index(self, bucket, object, headers = {}): 589 | ''' 590 | Get object group_index 591 | 592 | :type bucket: string 593 | :param 594 | 595 | :type object: string 596 | :param 597 | 598 | :type headers: dict 599 | :param: HTTP header 600 | 601 | Returns: 602 | HTTP Response 603 | ''' 604 | 605 | headers["x-oss-file-group"] = "" 606 | return self.object_operation("GET", bucket, object, headers) 607 | 608 | 609 | def put_object_from_file_given_pos(self, bucket, object, filename, offset, partsize, content_type=DefaultContentType, headers = {}): 610 | ''' 611 | Put object into bucket, the content of object is read from given posision of filename 612 | :type bucket: string 613 | :param 614 | 615 | :type object: string 616 | :param 617 | 618 | :type fllename: string 619 | :param: the name of the read file 620 | 621 | :type offset: int 622 | :param: the given position of file 623 | 624 | :type partsize: int 625 | :param: the size of read content 626 | 627 | :type content_type: string 628 | :param: the object content type that supported by HTTP 629 | 630 | :type headers: dict 631 | :param: HTTP header 632 | 633 | Returns: 634 | HTTP Response 635 | ''' 636 | fp = open(filename, 'rb') 637 | if offset > os.path.getsize(filename): 638 | fp.seek(os.SEEK_SET, os.SEEK_END) 639 | else: 640 | fp.seek(offset) 641 | 642 | conn = self._open_conn_to_put_object(bucket, object, partsize, content_type, headers) 643 | 644 | left_len = partsize 645 | while True: 646 | if left_len <= 0: 647 | break 648 | elif left_len < self.SendBufferSize: 649 | buffer_content = fp.read(left_len) 650 | else: 651 | buffer_content = fp.read(self.SendBufferSize) 652 | 653 | if len(buffer_content) > 0: 654 | conn.send(buffer_content) 655 | 656 | left_len = left_len - len(buffer_content) 657 | 658 | fp.close() 659 | return conn.getresponse() 660 | 661 | def upload_large_file(self, bucket, object, filename, thread_num = 10, max_part_num = 1000): 662 | ''' 663 | Upload large file, the content is read from filename. The large file is splitted into many parts. It will put the many parts into bucket and then merge all the parts into one object. 664 | 665 | :type bucket: string 666 | :param 667 | 668 | :type object: string 669 | :param 670 | 671 | :type fllename: string 672 | :param: the name of the read file 673 | 674 | ''' 675 | #split the large file into 1000 parts or many parts 676 | #get part_msg_list 677 | if isinstance(filename, unicode): 678 | filename = filename.encode('utf-8') 679 | 680 | part_msg_list = split_large_file(filename, object, max_part_num) 681 | 682 | #make sure all the parts are put into same bucket 683 | if len(part_msg_list) < thread_num and len(part_msg_list) != 0: 684 | thread_num = len(part_msg_list) 685 | 686 | step = len(part_msg_list) / thread_num 687 | threadpool = [] 688 | for i in range(0, thread_num): 689 | if i == thread_num - 1: 690 | end = len(part_msg_list) 691 | else: 692 | end = i * step + step 693 | 694 | begin = i * step 695 | oss = OssAPI(self.host, self.access_id, self.secret_access_key) 696 | current = PutObjectGroupWorker(oss, bucket, filename, part_msg_list[begin:end]) 697 | threadpool.append(current) 698 | current.start() 699 | 700 | for item in threadpool: 701 | item.join() 702 | 703 | #get xml string that contains msg of object group 704 | object_group_msg_xml = create_object_group_msg_xml(part_msg_list) 705 | 706 | return self.post_object_group(bucket, object, object_group_msg_xml) 707 | -------------------------------------------------------------------------------- /osscmd: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ## OSSCMD 3 | ## Author: linjiudui 4 | ## Email:linjd828917@gmail.com 5 | ## License: GPL Version 2 6 | ## learn from s3tools(s3tools.org) 7 | 8 | import sys 9 | 10 | reload(sys) 11 | sys.setdefaultencoding('utf-8') 12 | if float("%d.%d" % (sys.version_info[0], sys.version_info[1])) < 2.4: 13 | sys.stderr.write("ERROR: Python 2.4 or higher required, sorry.\n") 14 | sys.exit(1) 15 | 16 | import logging 17 | import time 18 | import os 19 | import errno 20 | import traceback 21 | import codecs 22 | import locale 23 | 24 | from copy import copy 25 | from optparse import OptionParser, Option, OptionValueError, IndentedHelpFormatter 26 | from logging import debug, info, warning, error 27 | from distutils.spawn import find_executable 28 | 29 | class ParameterError(Exception): 30 | pass 31 | 32 | def output(message): 33 | sys.stdout.write(message + "\n") 34 | 35 | def show_error_msg(xml_string, msg): 36 | error(u"%s" % msg) 37 | if xml_string is not None and xml_string != "": 38 | e = ErrorXml(xml_string) 39 | e.show() 40 | 41 | def login(access_id, secret_access_key): 42 | '''login''' 43 | oss = OssAPI(cfg.host_base, access_id, secret_access_key) 44 | res = oss.get_service() 45 | if (res.status / 100) == 2: 46 | return oss 47 | else: 48 | return None 49 | 50 | def _get_filelist_local(local_uri): 51 | info(u"Compiling list of local files...") 52 | if local_uri.isdir(): 53 | local_base = local_uri.basename() 54 | local_path = local_uri.path() 55 | filelist = os.walk(local_path) 56 | single_file = False 57 | else: 58 | local_base = "" 59 | local_path = (local_uri.dirname()) 60 | filelist = [( local_path, [], [local_uri.basename()] )] 61 | single_file = True 62 | loc_list = {} 63 | for root, dirs, files in filelist: 64 | rel_root = root.replace(local_path, local_base, 1) 65 | if(not single_file): 66 | if(os.path.sep != '/'): 67 | rel_root = '/'.join(rel_root.split(os.path.sep)) 68 | 69 | full_name = root 70 | if not rel_root.endswith('/'): 71 | rel_root = rel_root+'/' 72 | ## get directory list 73 | loc_list[rel_root] = { 74 | 'is_dir' : True, 75 | 'full_name_unicode' : full_name, 76 | 'full_name' : full_name, 77 | 'size' : 0, 78 | } 79 | 80 | ## get filelist for upload 81 | for f in files: 82 | full_name = os.path.join(root, f) 83 | if not os.path.isfile(full_name): 84 | continue 85 | 86 | relative_file = os.path.join(rel_root, f) 87 | if os.path.sep != "/": 88 | relative_file = "/".join(relative_file.split(os.path.sep)) 89 | if cfg.urlencoding_mode == "normal": 90 | relative_file = replace_nonprintables(relative_file) 91 | if relative_file.startswith('./'): 92 | relative_file = relative_file[2:] 93 | sr = os.stat_result(os.lstat(full_name)) 94 | loc_list[relative_file] = { 95 | 'is_dir' : False, 96 | 'full_name_unicode' : full_name, 97 | 'full_name' : full_name, 98 | 'size' : sr.st_size, 99 | 'mtime' : sr.st_mtime, 100 | } 101 | return loc_list, single_file 102 | 103 | def fetch_local_list(args, recursive = None): 104 | local_uris = [] 105 | local_list = {} 106 | single_file = False 107 | 108 | if type(args) not in (list, tuple): 109 | args = [args] 110 | 111 | if recursive == None: 112 | recursive = cfg.recursive 113 | 114 | for arg in args: 115 | uri = OSSUri(arg) 116 | if not uri.type == 'file': 117 | raise ParameterError("Expecting filename or directory instead of: %s" % arg) 118 | if uri.isdir() and not recursive: 119 | raise ParameterError("Use --recursive to upload a directory: %s" % arg) 120 | local_uris.append(uri) 121 | 122 | for uri in local_uris: 123 | list_for_uri, single_file = _get_filelist_local(uri) 124 | local_list.update(list_for_uri) 125 | 126 | ## Single file is True if and only if the user 127 | ## specified one local URI and that URI represents 128 | ## a FILE. Ie it is False if the URI was of a DIR 129 | ## and that dir contained only one FILE. That's not 130 | ## a case of single_file==True. 131 | if len(local_list) > 1: 132 | single_file = False 133 | 134 | return local_list, single_file 135 | 136 | def _get_remote_key(common_prefix, obj_name): 137 | """ 138 | four type of common_prefix 139 | oss://bucket-name 140 | oss://bucket-name/dir/ 141 | oss://bucket-name/file 142 | oss://bucket-name/prefix 143 | """ 144 | ## bucket-name 145 | if common_prefix == "": 146 | return obj_name 147 | ## directory 148 | if common_prefix.endswith('/'): 149 | key = obj_name.replace(common_prefix, '', 1) 150 | else: 151 | ## file and prefix 152 | common_prefix = common_prefix[:common_prefix.rfind('/')+1] 153 | key = obj_name.replace(common_prefix, '', 1) 154 | 155 | if len(key) == 0: 156 | key = u'.' 157 | return key 158 | 159 | 160 | def _get_filelist_remote(oss, remote_uri, recursive = True): 161 | info(u"Retrieving list of remote files for %s ..." % remote_uri) 162 | delimiter = '/' 163 | if(recursive): 164 | delimiter = '' 165 | 166 | re = oss.list_bucket(remote_uri.bucket(), remote_uri.object(), '', delimiter, maxkeys='', headers={}) 167 | xml_string = re.read() 168 | if(re.status/100 != 2): 169 | show_error_msg(xml_string, u"list bucket[%s] failed" %remote_uri.bucket()) 170 | return 171 | cl, pl = GetBucketXml(xml_string).list() 172 | 173 | remote_list = {} 174 | ##oss://bucket-name, oss://bucket-name/prefix, oss://bucket-name/dir/, oss://bucket-name/file 175 | for obj in cl: 176 | ## remove the common prefix 177 | key = _get_remote_key(remote_uri.object(), unicodise(obj[0])) 178 | 179 | # print key 180 | remote_list[key] = { 181 | 'is_dir' : False, 182 | 'size' : int(obj[3]), 183 | 'timestamp' : obj[1], 184 | 'md5' : obj[2].strip('"').lower(), 185 | 'base_uri' : remote_uri, 186 | 'object_uri': '/'.join(["oss:/", remote_uri.bucket(), unicodise(obj[0])]) 187 | } 188 | for obj in pl: 189 | ## remove the common prefix 190 | key = _get_remote_key(remote_uri.object(), unicodise(obj)) 191 | ## root directory 192 | if(len(key) == 0): 193 | key = u"." 194 | elif(key[0] == '/'): 195 | key = key[1:] 196 | remote_list[unicodise(obj)] = {'is_dir':True, 197 | 'object_uri': '/'.join(["oss:/", remote_uri.bucket(), unicodise(obj)])} 198 | return remote_list 199 | 200 | def fetch_remote_list(oss, args, recursive = None): 201 | remote_uris = [] 202 | remote_list = {} 203 | 204 | if type(args) not in (list, tuple): 205 | args = [args] 206 | 207 | if recursive == None: 208 | recursive = cfg.recursive 209 | 210 | for arg in args: 211 | if(arg.endswith('*')): 212 | arg = arg[:-1] 213 | uri = OSSUri(arg) 214 | if not uri.type == 'oss': 215 | raise ParameterError("Expecting oss URI instead of '%s'" % arg) 216 | remote_uris.append(uri) 217 | 218 | for uri in remote_uris: 219 | cur_list = _get_filelist_remote(oss, uri, recursive) 220 | if cur_list is not None: 221 | remote_list.update(cur_list) 222 | 223 | return remote_list 224 | 225 | def subcmd_object_del_uri(oss, uri_str, recursive = None): 226 | if recursive is None: 227 | recursive = cfg.recursive 228 | 229 | remote_list = fetch_remote_list(oss, uri_str, recursive = recursive) 230 | 231 | remote_count = len(remote_list) 232 | 233 | info(u"Summary: %d remote files to delete" % remote_count) 234 | 235 | if cfg.dry_run: 236 | for key in remote_list: 237 | output(u"delete: %s" % remote_list[key]['object_uri']) 238 | 239 | warning(u"Exitting now because of --dry-run") 240 | return 241 | 242 | for key in remote_list: 243 | item = remote_list[key] 244 | uri = OSSUri(item['object_uri']) 245 | re = oss.delete_object(uri.bucket(), uri.object()) 246 | if(re.status/100 != 2): 247 | show_error_msg(re.read(), u"Delete file %s failed" %item['object_uri']) 248 | continue 249 | output(u"File %s deleted" % item['object_uri']) 250 | 251 | def cmd_bucket_create(args): 252 | '''Create bucket''' 253 | oss = OssAPI(cfg.host_base, cfg.access_id, cfg.secret_access_key) 254 | if(cfg.acl_public): 255 | acl = "public-read" 256 | else: 257 | acl = "private" 258 | 259 | for arg in args: 260 | uri = OSSUri(arg) 261 | re = oss.put_bucket(uri.bucket(), acl=acl, headers={}) 262 | if (re.status / 100) == 2: 263 | output(u"Bucket '%s' created" % arg) 264 | return True 265 | else: 266 | show_error_msg(re.read(), u"create bucket [%s] failed" %arg) 267 | return False 268 | 269 | def cmd_bucket_delete(args): 270 | oss = OssAPI(cfg.host_base, cfg.access_id, cfg.secret_access_key) 271 | for arg in args: 272 | uri = OSSUri(arg) 273 | re = oss.delete_bucket(uri.bucket()) 274 | if (re.status/100 != 2): 275 | xml_string = re.read() 276 | e = ErrorXml(xml_string) 277 | if(e.code == "BucketNotEmpty" and cfg.recursive): 278 | warning(u"Bucket is not empty. Removing all the objects from it first. This may take some time...") 279 | subcmd_object_del_uri(oss, str(uri), recursive=True) 280 | if cfg.dry_run: 281 | continue 282 | re = oss.delete_bucket(uri.bucket()) 283 | if(re.status/100 != 2): 284 | show_error_msg(re.read(), u"delete bucket[%s] failed" %arg) 285 | continue 286 | else: 287 | show_error_msg(xml_string, u"delete bucket[%s] failed" %arg) 288 | continue 289 | output(u"Bucket '%s' removed" % arg) 290 | 291 | def cmd_buckets_list_all_all(args): 292 | oss = OssAPI(cfg.host_base, cfg.access_id, cfg.secret_access_key) 293 | 294 | re = oss.get_service() 295 | xml_string = re.read() 296 | if(re.status/100 != 2): 297 | show_error_msg(xml_string, "Get Service failed:") 298 | return 299 | bl = [l[0] for l in GetServiceXml(xml_string).list()] 300 | for bucket in bl: 301 | subcmd_bucket_list(oss, OSSUri("oss://" + bucket)) 302 | output(u"") 303 | 304 | def subcmd_buckets_list_all(oss): 305 | re = oss.get_service() 306 | xml_string = re.read() 307 | if(re.status / 100 != 2): 308 | show_error_msg(xml_string, "Get Service failed:") 309 | return 310 | bl = GetServiceXml(xml_string).list() 311 | for (name, cdate) in bl: 312 | output(u"%s oss://%s" % (formatDateTime(cdate), name)) 313 | 314 | def subcmd_bucket_list(oss, uri): 315 | bucket = uri.bucket() 316 | prefix = uri.object() 317 | 318 | debug(u"Bucket 'oss://%s':" % bucket) 319 | 320 | delimiter = '/' 321 | if prefix.endswith('*'): 322 | prefix = prefix[:-1] 323 | if cfg.recursive: 324 | delimiter = '' 325 | 326 | output(u"%s %s %s" %(bucket, prefix, delimiter)) 327 | re = oss.list_bucket(bucket, prefix, '', delimiter, maxkeys='', headers={}) 328 | xml_string = re.read() 329 | if(re.status / 100 != 2): 330 | show_error_msg(xml_string, u"list bucket[%s/%s] failed" %(bucket, prefix)) 331 | return 332 | cl, pl = GetBucketXml(xml_string).list() 333 | # print cl, pl 334 | if(delimiter == '/'): 335 | output(u"Bucket 'oss://%s':%d files, %d directory" % (bucket, len(cl), len(pl))) 336 | else: 337 | output(u"Bucket 'oss://%s':%d files" % (bucket, len(cl))) 338 | if cfg.list_md5: 339 | format_string = u"%(timestamp)16s %(size)9s%(coeff)1s %(md5)32s %(uri)s" 340 | else: 341 | format_string = u"%(timestamp)16s %(size)9s%(coeff)1s %(uri)s" 342 | 343 | 344 | if cfg.list_md5: 345 | format_string = u"%(timestamp)16s %(size)9s%(coeff)1s %(md5)32s %(uri)s" 346 | else: 347 | format_string = u"%(timestamp)16s %(size)9s%(coeff)1s %(uri)s" 348 | 349 | for prefix in pl: 350 | output(format_string % {"timestamp":"", "size":"DIR", "coeff":"", "md5":"", 351 | "uri":prefix}) 352 | 353 | for obj in cl: 354 | size, size_coeff = formatSize(obj[3], cfg.human_readable_sizes) 355 | output(format_string % { 356 | "timestamp":obj[1], 357 | "size" : str(size), 358 | "coeff": size_coeff, 359 | "md5" : obj[2].strip('"'), 360 | "uri": obj[0], 361 | }) 362 | 363 | 364 | def cmd_ls(args): 365 | oss = OssAPI(cfg.host_base, cfg.access_id, cfg.secret_access_key) 366 | if len(args) > 0: 367 | try: 368 | uri = OSSUri(args[0]) 369 | except ValueError, e: 370 | error(u"Parameter Error, %s", e) 371 | return 372 | if uri.type == "oss" and uri.has_bucket(): 373 | subcmd_bucket_list(oss, uri) 374 | return 375 | else: 376 | error("Parameter Error:%s", args[0]) 377 | return 378 | subcmd_buckets_list_all(oss) 379 | 380 | def cmd_object_put(args): 381 | """ 382 | put the local file or directory to the oss 383 | empty directory will not put into the oss 384 | """ 385 | if len(args) == 0: 386 | raise ParameterError("You must specify a local file or directory and a OSS URI destination.") 387 | 388 | try: 389 | dst_uri = OSSUri(args.pop()) 390 | except ValueError, e: 391 | error(u"Parameter Error, %s", e) 392 | return 393 | if dst_uri.type != 'oss': 394 | raise ParameterError("Destination must be OSSUri. Got: %s" % dst_uri) 395 | dst_base = str(dst_uri) 396 | 397 | if len(args) == 0: 398 | raise ParameterError("Nothing to upload. Expecting a local file or directory.") 399 | 400 | local_list, single_file_local = fetch_local_list(args) 401 | 402 | local_count = len(local_list) 403 | 404 | info(u"Summary: %d local files to upload" % local_count) 405 | 406 | if local_count > 0: 407 | if not dst_base.endswith("/"): 408 | if not single_file_local: 409 | raise ParameterError("Destination OSS URI must end with '/' (ie must refer to a directory on the remote side).") 410 | local_list[local_list.keys()[0]]['remote_uri'] = dst_base 411 | else: 412 | for key in local_list: 413 | local_list[key]['remote_uri'] = dst_base + key 414 | 415 | if cfg.dry_run: 416 | for key in local_list: 417 | output(u"upload: %s -> %s" % (local_list[key]['full_name_unicode'], local_list[key]['remote_uri'])) 418 | 419 | warning(u"\nThe files didn't uploaded now because of --dry-run") 420 | return 421 | 422 | oss = OssAPI(cfg.host_base, cfg.access_id, cfg.secret_access_key) 423 | seq = 0 424 | success = 0 425 | for key in local_list: 426 | seq += 1 427 | uri_final = OSSUri(local_list[key]['remote_uri']) 428 | full_file_name = local_list[key]['full_name'] 429 | file_size = local_list[key]['size'] 430 | is_dir = local_list[key]['is_dir'] 431 | info(u"Sending file '%s[%d of %d]', please wait..." % (full_file_name, seq, local_count)) 432 | 433 | start_time = time.time() 434 | if is_dir: 435 | re = oss.put_object_from_string(uri_final.bucket(), uri_final.object(), "") 436 | else: 437 | re = oss.put_object_from_file(uri_final.bucket(), uri_final.object(), full_file_name) 438 | end_time = time.time() 439 | send_time = end_time-start_time 440 | 441 | if(re.status/100 != 2): 442 | show_error_msg(re.read(), u"send file %s failed" %full_file_name) 443 | continue 444 | 445 | output(u"File '%s' stored as '%s' (%d bytes in %0.1fs)" % 446 | (full_file_name, uri_final, file_size, send_time)) 447 | success += 1 448 | if cfg.acl_public: 449 | output(u"ACL is public, Public URL of the object is: %s" % (uri_final.public_url())) 450 | output("\nTotal %d files transfered, successed:%d, failed:%d" %( 451 | local_count, success, local_count-success) ) 452 | 453 | def cmd_object_get(args): 454 | if len(args) == 0: 455 | raise ParameterError("Nothing to download. Expecting OSS URI.") 456 | 457 | if OSSUri(args[-1]).type == 'file': 458 | destination_base = args.pop() 459 | else: 460 | destination_base = "." 461 | 462 | if len(args) == 0: 463 | raise ParameterError("Nothing to download. Expecting OSS URI.") 464 | 465 | oss = OssAPI(cfg.host_base, cfg.access_id, cfg.secret_access_key) 466 | 467 | remote_list = fetch_remote_list(oss, args) 468 | 469 | remote_count = len(remote_list) 470 | 471 | info(u"Summary: %d remote files to download" % remote_count) 472 | 473 | if destination_base.endswith(os.path.sep) and not os.path.exists(destination_base): 474 | os.makedirs(destination_base) 475 | if remote_count > 0: 476 | if not os.path.isdir(destination_base): 477 | if remote_count > 1: 478 | raise ParameterError("Destination must be a directory when downloading multiple sources.") 479 | remote_list[remote_list.keys()[0]]['local_filename'] = destination_base 480 | else : 481 | if destination_base[-1] != os.path.sep: 482 | destination_base += os.path.sep 483 | for key in remote_list: 484 | local_filename = destination_base + key 485 | if(os.name == 'nt'): 486 | local_filename = os.path.sep.join(local_filename.split('/')) 487 | remote_list[key]['local_filename'] = local_filename 488 | if local_filename.endswith(os.path.sep): 489 | remote_list[key]['is_dir'] = True 490 | 491 | if cfg.dry_run: 492 | for key in remote_list: 493 | output(u"download: %s -> %s" % (remote_list[key]['object_uri'], remote_list[key]['local_filename'])) 494 | 495 | warning(u"The files have not downloaded now because of --dry-run") 496 | return 497 | 498 | seq = 0 499 | success = 0 500 | skip = 0 501 | for key in remote_list: 502 | seq += 1 503 | item = remote_list[key] 504 | uri = OSSUri(item['object_uri']) 505 | ## Encode / Decode destination with "replace" to make sure it's compatible with current encoding 506 | destination = item['local_filename'] 507 | 508 | if item['is_dir']: 509 | ##Directory 510 | if(not os.path.exists(destination)): 511 | os.makedirs(destination) 512 | info(u"Creating directory: %s" % destination) 513 | success += 1 514 | else:skip += 1 515 | continue 516 | else: 517 | ## File 518 | try: 519 | file_exists = os.path.exists(destination) 520 | try: 521 | fp = open(destination, "ab") 522 | except IOError, e: 523 | if e.errno == errno.ENOENT: 524 | basename = destination[:destination.rindex(os.path.sep)] 525 | info(u"Creating directory: %s" % basename) 526 | os.makedirs(basename) 527 | fp = open(destination, "ab") 528 | else: 529 | raise 530 | if file_exists: 531 | if cfg.force: 532 | fp.seek(0L) 533 | fp.truncate() 534 | elif cfg.skip_existing: 535 | skip += 1 536 | info(u"Skipping over existing file: %s" % (destination)) 537 | continue 538 | else: 539 | fp.close() 540 | raise ParameterError(u"File %s already exists. Use either of --force / --skip-existing or give it a new name." % destination) 541 | except IOError, e: 542 | error(u"Skipping %s: %s" % (destination, e.strerror)) 543 | continue 544 | 545 | info("Download file '%s[%d of %d]', please wait..." % (key, seq, remote_count)) 546 | start_time = time.time() 547 | try: 548 | re = oss.get_object_to_file(uri.bucket(), uri.object(), item['local_filename'], headers={}) 549 | except IOError, e: 550 | output(u"debug %s, %s, %s" %(str(uri), item['local_filename'], e)) 551 | continue 552 | if(re.status/100 != 2): 553 | show_error_msg(re.read(), u"download file %s from oss failed" % key) 554 | continue 555 | end_time = time.time() 556 | send_time = end_time-start_time 557 | success += 1 558 | output(u"File %s saved as '%s' (%d bytes in %0.1f seconds)" % 559 | (uri, os.path.abspath(destination), item['size'], send_time)) 560 | output("\nTotal %d files downloaded, successed:%d, skip:%d, failed:%d" %( 561 | remote_count, success, skip, remote_count-success-skip) ) 562 | 563 | def cmd_object_del(args): 564 | for arg in args: 565 | uri = OSSUri(arg) 566 | if uri.type != "oss": 567 | raise ParameterError("Expecting OSS URI instead of '%s'" % uri) 568 | if not uri.has_object(): 569 | if cfg.recursive and not cfg.force: 570 | raise ParameterError("Please use --force to delete ALL contents of %s" % arg) 571 | elif not cfg.recursive: 572 | raise ParameterError("File name required, not only the bucket name. Alternatively use --recursive") 573 | oss = OssAPI(cfg.host_base, cfg.access_id, cfg.secret_access_key) 574 | subcmd_object_del_uri(oss, arg) 575 | 576 | def cmd_setacl(args): 577 | oss = OssAPI(cfg.host_base, cfg.access_id, cfg.secret_access_key) 578 | 579 | if(len(args) == 0): 580 | raise ParameterError("You must specify a bucket or object.") 581 | if (cfg.acl_public is None): 582 | return 583 | elif(cfg.acl_public): 584 | acl = "public-read" 585 | else: 586 | acl = "private" 587 | 588 | for arg in args: 589 | uri = OSSUri(arg) 590 | if uri.type != "oss": 591 | raise ParameterError("Expecting OSS URI instead of '%s'" % uri) 592 | re = oss.put_bucket(uri.bucket(), acl, headers={}) 593 | if re.status/100 != 2: 594 | show_error_msg(re.read(), u"set acl for bucket[%s] failed" %uri.bucket()) 595 | continue 596 | output(u"set %s for bucket[oss://%s] success" %(acl, uri.bucket())) 597 | 598 | def cmd_getacl(args): 599 | oss = OssAPI(cfg.host_base, cfg.access_id, cfg.secret_access_key) 600 | 601 | for arg in args: 602 | uri = OSSUri(arg) 603 | if uri.type != "oss": 604 | raise ParameterError("Expecting OSS URI instead of '%s'" % uri) 605 | re = oss.get_bucket_acl(uri.bucket()) 606 | if re.status/100 != 2: 607 | show_error_msg(re.read(), u"get acl of object[%s] failed" %arg) 608 | continue 609 | bucket_acl = GetBucketAclXml(re.read()).grant 610 | output(u"The acl of obcjet[%s] is %s" %(arg, bucket_acl)) 611 | 612 | def cmd_head_object(args): 613 | oss = OssAPI(cfg.host_base, cfg.access_id, cfg.secret_access_key) 614 | 615 | for arg in args: 616 | uri = OSSUri(arg) 617 | if uri.type != "oss": 618 | raise ParameterError("Expecting OSS URI instead of '%s'" % uri) 619 | re = oss.head_object(uri.bucket(), uri.object(), headers={}) 620 | 621 | if re.status/100 != 2: 622 | error("get info of object %s failed, No Suck Object?" %arg) 623 | continue 624 | headers = dict(re.getheaders()) 625 | output(u"The info of object %s:" %arg) 626 | for key in headers: 627 | output(u"%-20s: %s" %(key, headers[key])) 628 | 629 | def subcmd_cp_mv(args, action_str): 630 | if len(args) < 2: 631 | raise ParameterError("Expecting two or more OSS URIs for %s" %action_str) 632 | 633 | dst_base_uri = OSSUri(args.pop()) 634 | if dst_base_uri.type != "oss": 635 | raise ParameterError("Destination must be OSS URI.") 636 | destination_base = dst_base_uri.uri() 637 | 638 | oss = OssAPI(cfg.host_base, cfg.access_id, cfg.secret_access_key) 639 | remote_list = fetch_remote_list(oss, args, recursive=cfg.recursive) 640 | 641 | remote_count = len(remote_list) 642 | 643 | info(u"Summary: %d remote files to %s" % (remote_count, action_str)) 644 | 645 | if(remote_count == 0): 646 | error(u"No object find in OSS, check the source OSSURi") 647 | return 648 | if(remote_count > 1 and not cfg.recursive): 649 | error(u"Please use -r(--recursive)option to copy/move more than one objects") 650 | return 651 | 652 | if cfg.recursive: 653 | if not destination_base.endswith("/"): 654 | destination_base += "/" 655 | for key in remote_list: 656 | remote_list[key]['dest_name'] = destination_base + key 657 | else: 658 | #print remote_list.keys() 659 | key = remote_list.keys()[0] 660 | if destination_base.endswith("/"): 661 | ##copy directory 662 | if key.endswith('/'): 663 | dest_file_name = key[(key[:-1].rfind('/')+1):] 664 | else: 665 | dest_file_name = key[(key.rfind('/')+1):] 666 | remote_list[key]['dest_name'] = (destination_base + dest_file_name) 667 | else: 668 | remote_list[key]['dest_name'] = (destination_base) 669 | 670 | if cfg.dry_run: 671 | for key in remote_list: 672 | output(u"%s: %s -> %s" % (action_str, remote_list[key]['object_uri'], remote_list[key]['dest_name'])) 673 | 674 | warning(u"OSS didn't perform %s action now because of --dry-run" %action_str) 675 | return 676 | 677 | for key in remote_list: 678 | item = remote_list[key] 679 | #print item 680 | src_uri = OSSUri(item['object_uri']) 681 | dst_uri = OSSUri(item['dest_name']) 682 | # print (dst_uri.object()) 683 | headers={"x-oss-copy-source":item['object_uri'][5:]} 684 | 685 | bkt = dst_uri.bucket() 686 | obj = dst_uri.object() 687 | # print type(obj), obj 688 | re = oss.object_operation('PUT', bkt, obj, headers, "") 689 | if(re.status/100 != 2): 690 | show_error_msg(re.read(), u"%s from %s to %s failed" %(action_str, item['object_uri'], item['dest_name'])) 691 | continue 692 | 693 | if cfg.acl_public: 694 | info(u"ACL is public, Public URL is: %s" % dst_uri.public_url()) 695 | 696 | if action_str == 'move': 697 | re = oss.delete_object(src_uri.bucket(), src_uri.object(), headers={}) 698 | if re.status/100 != 2: 699 | show_error_msg(re.read(), u"Delete object %s failed" %item['object_uri']) 700 | continue 701 | output(u"%s: %s -> %s" % (action_str, item['object_uri'], item['dest_name'])) 702 | 703 | 704 | def _compare_filelists(src_list, dst_list, src_remote, dst_remote): 705 | def __direction_str(is_remote): 706 | return is_remote and "remote" or "local" 707 | 708 | info(u"Verifying attributes...") 709 | exists_list = {} 710 | 711 | debug("Comparing filelists (direction: %s -> %s)" % (__direction_str(src_remote), __direction_str(dst_remote))) 712 | src_list.keys().sort() 713 | dst_list.keys().sort() 714 | debug("src_list.keys: %s" % src_list.keys()) 715 | debug("dst_list.keys: %s" % dst_list.keys()) 716 | 717 | for file in src_list.keys(): 718 | debug(u"CHECK: %s" % file) 719 | ## Skip Directory 720 | if src_remote == False and src_list[file]['is_dir']: 721 | continue 722 | 723 | if dst_list.has_key(file): 724 | if dst_remote == False and dst_list[file]['is_dir']: 725 | continue 726 | ## Was --skip-existing requested? 727 | if cfg.skip_existing: 728 | debug(u"IGNR: %s (used --skip-existing)" % (file)) 729 | exists_list[file] = src_list[file] 730 | del(src_list[file]) 731 | ## Remove from destination-list, all that is left there will be deleted 732 | del(dst_list[file]) 733 | continue 734 | 735 | attribs_match = True 736 | ## Check size first 737 | if 'size' in cfg.sync_checks and dst_list[file]['size'] != src_list[file]['size']: 738 | debug(u"%s (size mismatch: src=%s dst=%s)" % (file, src_list[file]['size'], dst_list[file]['size'])) 739 | attribs_match = False 740 | 741 | if attribs_match and 'md5' in cfg.sync_checks: 742 | ## same size, check MD5 743 | if src_remote == False and dst_remote == True: 744 | src_md5 = hash_file_md5(src_list[file]['full_name']) 745 | dst_md5 = dst_list[file]['md5'] 746 | elif src_remote == True and dst_remote == False: 747 | src_md5 = src_list[file]['md5'] 748 | dst_md5 = hash_file_md5(dst_list[file]['full_name']) 749 | 750 | if src_md5 != dst_md5: 751 | ## Checksums are different. 752 | attribs_match = False 753 | debug(u"%s (md5 mismatch: src=%s dst=%s)" % (file, src_md5, dst_md5)) 754 | 755 | if attribs_match: 756 | ## Remove from source-list, all that is left there will be transferred 757 | debug(u"IGNR: %s (transfer not needed)" % file) 758 | exists_list[file] = src_list[file] 759 | del(src_list[file]) 760 | 761 | ## Remove from destination-list, all that is left there will be deleted 762 | del(dst_list[file]) 763 | 764 | return src_list, dst_list, exists_list 765 | 766 | def cmd_sync_remote2local(args): 767 | 768 | oss = OssAPI(cfg.host_base, cfg.access_id, cfg.secret_access_key) 769 | destination_base = args[-1] 770 | local_list, single_file_local = fetch_local_list(destination_base, recursive = True) 771 | remote_list = fetch_remote_list(oss, args[:-1], recursive = True) 772 | 773 | local_count = len(local_list) 774 | remote_count = len(remote_list) 775 | 776 | info(u"Found %d remote files, %d local files" % (remote_count, local_count)) 777 | 778 | remote_list, local_list, existing_list = _compare_filelists(remote_list, local_list, src_remote = True, dst_remote = False) 779 | 780 | local_count = len(local_list) 781 | remote_count = len(remote_list) 782 | 783 | info(u"Summary: %d remote files to download, %d local files to delete" % (remote_count, local_count)) 784 | 785 | 786 | if remote_count > 0: 787 | if not os.path.isdir(destination_base): 788 | if remote_count > 1: 789 | raise ParameterError("Destination must be a directory when downloading multiple sources.") 790 | remote_list[remote_list.keys()[0]]['local_filename'] = destination_base 791 | else: 792 | if destination_base[-1] != os.path.sep: 793 | destination_base += os.path.sep 794 | for key in remote_list: 795 | local_filename = destination_base + key 796 | if os.path.sep != "/": 797 | local_filename = os.path.sep.join(local_filename.split("/")) 798 | remote_list[key]['local_filename'] = local_filename 799 | if local_filename.endswith(os.path.sep): 800 | remote_list[key]['is_dir'] = True 801 | 802 | if cfg.dry_run: 803 | if cfg.delete_removed: 804 | for key in local_list: 805 | if(local_list[key]['is_dir']): 806 | continue 807 | output(u"delete: %s" % local_list[key]['full_name_unicode']) 808 | # print list([remote_list[key]['local_filename']]) 809 | for key in remote_list: 810 | output(u"download: %s -> %s" % (remote_list[key]['object_uri'], remote_list[key]['local_filename'])) 811 | 812 | warning(u"Exitting now because of --dry-run") 813 | return 814 | 815 | if cfg.delete_removed: 816 | for key in local_list: 817 | if(local_list[key]['is_dir']): 818 | continue 819 | os.unlink(local_list[key]['full_name']) 820 | output(u"deleted: %s" % local_list[key]['full_name_unicode']) 821 | 822 | total_size = 0 823 | timestamp_start = time.time() 824 | file_list = remote_list.keys() 825 | file_list.sort() 826 | success = 0 827 | seq = 0 828 | 829 | for file in file_list: 830 | seq += 1 831 | item = remote_list[file] 832 | uri = OSSUri(item['object_uri']) 833 | ## Encode / Decode destination with "replace" to make sure it's compatible with current encoding 834 | destination = item['local_filename'] 835 | 836 | if item['is_dir']: 837 | ##Directory 838 | if(not os.path.exists(destination)): 839 | os.makedirs(destination) 840 | info(u"Creating directory: %s" % destination) 841 | success += 1 842 | continue 843 | else: 844 | ## File 845 | try: 846 | file_exists = os.path.exists(destination) 847 | try: 848 | fp = open(destination, "ab") 849 | except IOError, e: 850 | if e.errno == errno.ENOENT: 851 | basename = destination[:destination.rindex(os.path.sep)] 852 | info(u"Creating directory: %s" % basename) 853 | os.makedirs(basename) 854 | fp = open(destination, "ab") 855 | else: 856 | raise 857 | if file_exists: 858 | fp.seek(0L) 859 | fp.truncate() 860 | else: 861 | fp.close() 862 | raise ParameterError(u"File %s already exists failed to truncate it" % destination) 863 | except IOError, e: 864 | error(u"Skipping %s: %s" % (destination, e.strerror)) 865 | continue 866 | 867 | info("sync file '%s[%d of %d]', please wait..." % (file, seq, remote_count)) 868 | start_time = time.time() 869 | try: 870 | re = oss.get_object_to_file(uri.bucket(), uri.object(), destination) 871 | except IOError, e: 872 | output(u"debug %s, %s, %s" %(str(uri), item['local_filename'], e)) 873 | continue 874 | if(re.status/100 != 2): 875 | show_error_msg(re.read(), u"sync file %s from oss failed" % file) 876 | continue 877 | end_time = time.time() 878 | send_time = end_time-start_time 879 | success += 1 880 | output(u"File %s synchronized as '%s' (%d bytes in %0.1f seconds)" % 881 | (uri, os.path.abspath(destination), item['size'], send_time)) 882 | 883 | total_elapsed = time.time() - timestamp_start 884 | 885 | output(u"Done. Sync %d files of %d bytes in %0.1f seconds" % (success, total_size, total_elapsed)) 886 | 887 | 888 | def cmd_sync_local2remote(args): 889 | destination_base_uri = OSSUri(args[-1]) 890 | if destination_base_uri.type != 'oss': 891 | raise ParameterError("Destination must be OSSUri. Got: %s" % destination_base_uri) 892 | destination_base = str(destination_base_uri) 893 | 894 | oss = OssAPI(cfg.host_base, cfg.access_id, cfg.secret_access_key) 895 | local_list, single_file_local = fetch_local_list(args[:-1], recursive = True) 896 | remote_list = fetch_remote_list(oss, destination_base, recursive = True) 897 | 898 | local_count = len(local_list) 899 | remote_count = len(remote_list) 900 | 901 | info(u"Found %d local files, %d remote files" % (local_count, remote_count)) 902 | 903 | ## only one file for sync 904 | if single_file_local and len(local_list) == 1 and len(remote_list) == 1: 905 | remote_list_entry = remote_list[remote_list.keys()[0]] 906 | remote_list = { local_list.keys()[0] : remote_list_entry } 907 | 908 | local_list, remote_list, existing_list = _compare_filelists(local_list, remote_list, src_remote = False, dst_remote = True) 909 | 910 | local_count = len(local_list) 911 | remote_count = len(remote_list) 912 | 913 | info(u"Summary: %d local files to upload, %d remote files to delete" % (local_count, remote_count)) 914 | 915 | if local_count > 0: 916 | ## Populate 'remote_uri' only if we've got something to upload 917 | if not destination_base.endswith("/"): 918 | if not single_file_local: 919 | raise ParameterError("Destination OSS URI must end with '/' (ie must refer to a directory on the remote side).") 920 | local_list[local_list.keys()[0]]['remote_uri'] = destination_base 921 | else: 922 | for key in local_list: 923 | local_list[key]['remote_uri'] = destination_base + key 924 | 925 | if cfg.dry_run: 926 | if cfg.delete_removed: 927 | for key in remote_list: 928 | output(u"delete: %s" % remote_list[key]['object_uri']) 929 | for key in local_list: 930 | output(u"upload: %s -> %s" % (local_list[key]['full_name_unicode'], local_list[key]['remote_uri'])) 931 | 932 | warning(u"Exitting now because of --dry-run") 933 | return 934 | 935 | if cfg.delete_removed: 936 | for key in remote_list: 937 | uri = OSSUri(remote_list[key]['object_uri']) 938 | oss.delete_object(uri.bucket(), uri.object(), headers={}) 939 | output(u"deleted: '%s'" % uri) 940 | 941 | total_size = 0 942 | timestamp_start = time.time() 943 | success = 0 944 | file_list = local_list.keys() 945 | 946 | file_list.sort() 947 | for file_name in file_list: 948 | item = local_list[file_name] 949 | src = item['full_name'] 950 | uri = OSSUri(item['remote_uri']) 951 | is_dir = item['is_dir'] 952 | if is_dir: 953 | re = oss.put_object_from_string(uri.bucket(), uri.object(), "") 954 | else: 955 | re = oss.put_object_from_file(uri.bucket(), uri.object(), src) 956 | if(re.status/100 != 2): 957 | show_error_msg(re.read(), u"upload file %s failed" %item['full_name_unicode']) 958 | continue 959 | total_size += item["size"] 960 | success += 1 961 | output(u"File '%s' stored as '%s'" % ( 962 | item['full_name_unicode'], uri)) 963 | 964 | 965 | total_elapsed = (time.time() - timestamp_start) 966 | output(u"Done. Uploaded %d bytes of %d files in %0.1f seconds" % (total_size, success, total_elapsed)) 967 | 968 | def cmd_sync(args): 969 | if (len(args) < 2): 970 | raise ParameterError("Need two parameters(src, dst): LOCAL_DIR OSSUri or OSSUri LOCAL_DIR") 971 | 972 | if OSSUri(args[0]).type == "file" and OSSUri(args[-1]).type == "oss": 973 | return cmd_sync_local2remote(args) 974 | if OSSUri(args[0]).type == "oss" and OSSUri(args[-1]).type == "file": 975 | return cmd_sync_remote2local(args) 976 | raise ParameterError("Invalid source/destination: '%s'" % "' '".join(args)) 977 | 978 | def cmd_cp(args): 979 | subcmd_cp_mv(args, "copy") 980 | pass 981 | 982 | def cmd_mv(args): 983 | subcmd_cp_mv(args, "move") 984 | pass 985 | 986 | def cmd_sign(args): 987 | string_to_sign = args.pop() 988 | debug("string-to-sign: %r" % string_to_sign) 989 | signature = sign_string(string_to_sign) 990 | output("Signature: %s" % signature) 991 | 992 | def run_configure(config_file): 993 | cfg = Config() 994 | options = [ 995 | ("access_id", "Access ID", "Access ID and Secret Access key are your identifiers for OSS"), 996 | ("secret_access_key", "Secret Access Key"), 997 | ] 998 | 999 | try: 1000 | while 1: 1001 | output(u"\nEnter new values or accept defaults in brackets with Enter.") 1002 | output(u"Refer to user manual for detailed description of all options.") 1003 | for option in options: 1004 | prompt = option[1] 1005 | 1006 | try: 1007 | val = getattr(cfg, option[0]) 1008 | if type(val) is bool: 1009 | val = val and "Yes" or "No" 1010 | if val not in (None, ""): 1011 | prompt += " [%s]" % val 1012 | except AttributeError: 1013 | pass 1014 | 1015 | if len(option) >= 3: 1016 | output(u"\n%s" % option[2]) 1017 | 1018 | val = raw_input(prompt + ": ") 1019 | if val != "": 1020 | setattr(cfg, option[0], val) 1021 | output(u"\nNew settings:") 1022 | for option in options: 1023 | output(u" %s: %s" % (option[1], getattr(cfg, option[0]))) 1024 | val = raw_input("\nTest access with supplied credentials? [Y/n] ") 1025 | if val.lower().startswith("y") or val == "": 1026 | try: 1027 | output(u"Please wait...") 1028 | oss = login(cfg.access_id, cfg.secret_access_key) 1029 | if oss is None: 1030 | error(u"Test failed") 1031 | else: 1032 | output(u"Success. Your identity has been verified successful.") 1033 | 1034 | except Exception, e: 1035 | error(u"Test failed: %s" % (e)) 1036 | val = raw_input("\nRetry configuration? [Y/n] ") 1037 | if val.lower().startswith("y") or val == "": 1038 | continue 1039 | 1040 | 1041 | val = raw_input("\nSave settings? [y/N] ") 1042 | if val.lower().startswith("y"): 1043 | break 1044 | val = raw_input("Retry configuration? [Y/n] ") 1045 | if val.lower().startswith("n"): 1046 | raise EOFError() 1047 | 1048 | ## Overwrite existing config file, make it user-readable only 1049 | old_mask = os.umask(0077) 1050 | try: 1051 | os.remove(config_file) 1052 | except OSError, e: 1053 | if e.errno != errno.ENOENT: 1054 | raise 1055 | f = open(config_file, "w") 1056 | os.umask(old_mask) 1057 | cfg.dump_config(f) 1058 | f.close() 1059 | output(u"Configuration saved to '%s'" % config_file) 1060 | 1061 | except (EOFError, KeyboardInterrupt): 1062 | output(u"\nConfiguration aborted. Changes were NOT saved.") 1063 | return 1064 | 1065 | except IOError, e: 1066 | error(u"Writing config file failed: %s: %s" % (config_file, e.strerror)) 1067 | sys.exit(1) 1068 | 1069 | def get_commands_list(): 1070 | return [ 1071 | {"cmd":"mb", "label":"Make bucket", "param":"BUCKET [BUCKET]", "func":cmd_bucket_create, "argc":1}, 1072 | {"cmd":"rb", "label":"Remove bucket", "param":"BUCKET", "func":cmd_bucket_delete, "argc":1}, 1073 | {"cmd":"ls", "label":"List objects or buckets", "param":"[oss://BUCKET[/PREFIX]]", "func":cmd_ls, "argc":0}, 1074 | {"cmd":"la", "label":"List all object in all buckets", "param":"", "func":cmd_buckets_list_all_all, "argc":0}, 1075 | {"cmd":"put", "label":"Put file into bucket", "param":"FILE [FILE...] oss://BUCKET[/PREFIX]", "func":cmd_object_put, "argc":2}, 1076 | {"cmd":"get", "label":"Get file from bucket", "param":"oss://BUCKET/OBJECT LOCAL_FILE", "func":cmd_object_get, "argc":1}, 1077 | {"cmd":"del", "label":"Delete file from bucket", "param":"oss://BUCKET/OBJECT", "func":cmd_object_del, "argc":1}, 1078 | {"cmd":"sync", "label":"Synchronize a directory tree to OSS", "param":"LOCAL_DIR oss://BUCKET[/PREFIX] or oss://BUCKET[/PREFIX] LOCAL_DIR", "func":cmd_sync, "argc":2}, 1079 | {"cmd":"info", "label":"Get various information about objects", "param":"oss://BUCKET/OBJECT", "func":cmd_head_object, "argc":1}, 1080 | {"cmd":"cp", "label":"Copy object", "param":"oss://BUCKET1/OBJECT1 oss://BUCKET2[/OBJECT2]", "func":cmd_cp, "argc":2}, 1081 | {"cmd":"mv", "label":"Move object", "param":"oss://BUCKET1/OBJECT1 oss://BUCKET2[/OBJECT2]", "func":cmd_mv, "argc":2}, 1082 | {"cmd":"setacl", "label":"Modify Access control list for Bucket or Files", "param":"oss://BUCKET[/OBJECT]", "func":cmd_setacl, "argc":1}, 1083 | {"cmd":"getacl", "label":"Get Access control of Bucket or Files", "param":"oss://BUCKET[/OBJECT]", "func":cmd_getacl, "argc":1}, 1084 | {"cmd":"sign", "label":"Sign arbitrary string using the secret key", "param":"STRING-TO-SIGN", "func":cmd_sign, "argc":1}, 1085 | ] 1086 | 1087 | 1088 | def format_commands(progname, commands_list): 1089 | help = "Commands:\n" 1090 | for cmd in commands_list: 1091 | help += " %s\n %s %s %s\n" % (cmd["label"], progname, cmd["cmd"], cmd["param"]) 1092 | return help 1093 | 1094 | class OptionMimeType(Option): 1095 | def check_mimetype(self, option, opt, value): 1096 | if re.compile("^[a-z0-9]+/[a-z0-9+\.-]+$", re.IGNORECASE).match(value): 1097 | return value 1098 | raise OptionValueError("option %s: invalid MIME-Type format: %r" % (opt, value)) 1099 | 1100 | class OptionAll(OptionMimeType): 1101 | TYPE_CHECKER = copy(Option.TYPE_CHECKER) 1102 | TYPE_CHECKER["mimetype"] = OptionMimeType.check_mimetype 1103 | TYPES = Option.TYPES + ("mimetype",) 1104 | 1105 | class MyHelpFormatter(IndentedHelpFormatter): 1106 | def format_epilog(self, epilog): 1107 | if epilog: 1108 | return "\n" + epilog + "\n" 1109 | else: 1110 | return "" 1111 | def main(): 1112 | global cfg 1113 | 1114 | commands_list = get_commands_list() 1115 | commands = {} 1116 | 1117 | ## Populate "commands" from "commands_list" 1118 | for cmd in commands_list: 1119 | if cmd.has_key("cmd"): 1120 | commands[cmd["cmd"]] = cmd 1121 | 1122 | default_verbosity = Config().verbosity 1123 | optparser = OptionParser(option_class=OptionAll, formatter=MyHelpFormatter()) 1124 | #optparser.disable_interspersed_args() 1125 | 1126 | config_file = None 1127 | if os.getenv("HOME"): 1128 | config_file = os.path.join(os.getenv("HOME"), ".osscfg") 1129 | elif os.name == "nt" and os.getenv("USERPROFILE"): 1130 | config_file = os.path.join(os.getenv("USERPROFILE").decode('mbcs'), "Application Data", "osscmd.ini") 1131 | 1132 | preferred_encoding = locale.getpreferredencoding() or "UTF-8" 1133 | 1134 | optparser.set_defaults(encoding=preferred_encoding) 1135 | optparser.set_defaults(config=config_file) 1136 | optparser.set_defaults(verbosity=default_verbosity) 1137 | 1138 | optparser.add_option("--configure", dest="run_configure", action="store_true", help="Invoke interactive (re)configuration tool.") 1139 | optparser.add_option("-c", "--config", dest="config", metavar="FILE", help="Config file name. Defaults to %default") 1140 | optparser.add_option("--dump-config", dest="dump_config", action="store_true", help="Dump current configuration after parsing config files and command line options and exit.") 1141 | 1142 | optparser.add_option("-n", "--dry-run", dest="dry_run", action="store_true", help="Only show what should be uploaded or downloaded but don't actually do it. May still perform OSS requests to get bucket listings and other information though (only for file transfer commands)") 1143 | 1144 | optparser.add_option("-f", "--force", dest="force", action="store_true", help="Force overwrite and other dangerous operations.") 1145 | # optparser.add_option("--continue", dest="get_continue", action="store_true", help="Continue getting a partially downloaded file (only for [get] command).") 1146 | optparser.add_option("--skip-existing", dest="skip_existing", action="store_true", help="Skip over files that exist at the destination (only for [get] and [sync] commands).") 1147 | optparser.add_option("-r", "--recursive", dest="recursive", action="store_true", help="Recursive upload, download or removal.") 1148 | optparser.add_option("--check-md5", dest="check_md5", action="store_true", help="Check MD5 sums when comparing files for [sync]. (default)") 1149 | optparser.add_option("--no-check-md5", dest="check_md5", action="store_false", help="Do not check MD5 sums when comparing files for [sync]. Only size will be compared. May significantly speed up transfer but may also miss some changed files.") 1150 | optparser.add_option("-P", "--acl-public", dest="acl_public", action="store_true", help="Store objects with ACL allowing read for anyone.") 1151 | optparser.add_option("--acl-private", dest="acl_public", action="store_false", help="Store objects with default ACL allowing access for you only.") 1152 | 1153 | optparser.add_option("--delete-removed", dest="delete_removed", action="store_true", help="Delete remote objects with no corresponding local file [sync]") 1154 | optparser.add_option("--no-delete-removed", dest="delete_removed", action="store_false", help="Don't delete remote objects.") 1155 | 1156 | optparser.add_option("--list-md5", dest="list_md5", action="store_true", help="Include MD5 sums in bucket listings (only for 'ls' command).") 1157 | optparser.add_option("-H", "--human-readable-sizes", dest="human_readable_sizes", action="store_true", help="Print sizes in human readable form (eg 1kB instead of 1234).") 1158 | 1159 | optparser.add_option("-v", "--verbose", dest="verbosity", action="store_const", const=logging.INFO, help="Enable verbose output.") 1160 | optparser.add_option("-d", "--debug", dest="verbosity", action="store_const", const=logging.DEBUG, help="Enable debug output.") 1161 | optparser.add_option("--version", dest="show_version", action="store_true", help="Show osscmd version (%s) and exit." % (pkginfo.version)) 1162 | 1163 | optparser.set_usage(optparser.usage + " COMMAND [parameters]") 1164 | optparser.set_description('osscmd is a tool for managing objects in ' + 1165 | 'Alibaba oss(open storage service). It allows for making and removing ' + 1166 | '"buckets" and uploading, downloading and removing ' + 1167 | '"objects" from these buckets.') 1168 | optparser.epilog = format_commands(optparser.get_prog_name(), commands_list) 1169 | 1170 | (options, args) = optparser.parse_args() 1171 | 1172 | logging.basicConfig(level=options.verbosity, 1173 | format='%(levelname)s: %(message)s', 1174 | stream=sys.stderr) 1175 | 1176 | if options.show_version: 1177 | output(u"osscmd version %s" % pkginfo.version) 1178 | sys.exit(0) 1179 | 1180 | if not options.config: 1181 | error(u"Can't find a config file. Please use --config option.") 1182 | sys.exit(1) 1183 | 1184 | try: 1185 | cfg = Config(options.config) 1186 | except IOError, e: 1187 | if options.run_configure: 1188 | cfg = Config() 1189 | else: 1190 | error(u"%s: %s" % (options.config, e.strerror)) 1191 | error(u"Configuration file not available.") 1192 | error(u"Consider using --configure parameter to create one.") 1193 | sys.exit(1) 1194 | 1195 | ## logging level adjustments 1196 | ## according to configfile and command line parameters 1197 | if options.verbosity != default_verbosity: 1198 | cfg.verbosity = options.verbosity 1199 | logging.root.setLevel(cfg.verbosity) 1200 | 1201 | ## Process --(no-)check-md5 1202 | if options.check_md5 == False: 1203 | try: 1204 | cfg.sync_checks.remove("md5") 1205 | except: 1206 | pass 1207 | if options.check_md5 == True and cfg.sync_checks.count("md5") == 0: 1208 | cfg.sync_checks.append("md5") 1209 | 1210 | ## Update Config with other parameters 1211 | for option in cfg.option_list(): 1212 | try: 1213 | if getattr(options, option) != None: 1214 | debug(u"Updating Config.Config %s -> %s" % (option, getattr(options, option))) 1215 | cfg.update_option(option, getattr(options, option)) 1216 | except AttributeError: 1217 | ## Some Config() options are not settable from command line 1218 | pass 1219 | 1220 | ## Special handling for tri-state options (True, False, None) 1221 | cfg.update_option("acl_public", options.acl_public) 1222 | 1223 | ## Set output and filesystem encoding for printing out filenames. 1224 | sys.stdout = codecs.getwriter(cfg.encoding)(sys.stdout, "replace") 1225 | sys.stderr = codecs.getwriter(cfg.encoding)(sys.stderr, "replace") 1226 | 1227 | 1228 | if options.dump_config: 1229 | cfg.dump_config(sys.stdout) 1230 | sys.exit(0) 1231 | 1232 | if options.run_configure: 1233 | run_configure(options.config) 1234 | sys.exit(0) 1235 | 1236 | if len(args) < 1: 1237 | error(u"Missing command. Please run with --help for more information.") 1238 | sys.exit(1) 1239 | 1240 | args = [unicodise(arg) for arg in args] 1241 | 1242 | command = args.pop(0) 1243 | try: 1244 | debug(u"Command: %s" % commands[command]["cmd"]) 1245 | cmd_func = commands[command]["func"] 1246 | except KeyError, e: 1247 | error(u"Invalid command: %s" % e) 1248 | sys.exit(1) 1249 | 1250 | if len(args) < commands[command]["argc"]: 1251 | error(u"Not enough paramters for command '%s'" % command) 1252 | sys.exit(1) 1253 | 1254 | cmd_func(args) 1255 | 1256 | def report_exception(e): 1257 | tb = traceback.format_exc(sys.exc_info()) 1258 | e_class = str(e.__class__) 1259 | e_class = e_class[e_class.rfind(".") + 1 :-2] 1260 | sys.stderr.write(u"Problem: %s: %s\n" % (e_class, e)) 1261 | try: 1262 | sys.stderr.write("osscmd: %s\n" % pkginfo.version) 1263 | except NameError: 1264 | sys.stderr.write("osscmd: unknown version. Module import problem?\n") 1265 | sys.stderr.write("\n") 1266 | sys.stderr.write(unicode(tb, errors="replace")) 1267 | 1268 | if type(e) == ImportError: 1269 | sys.stderr.write("\n") 1270 | sys.stderr.write("Your sys.path contains these entries:\n") 1271 | for path in sys.path: 1272 | sys.stderr.write(u"\t%s\n" % path) 1273 | sys.stderr.write("Now the question is where have the osscmd modules been installed?\n") 1274 | 1275 | if __name__ == '__main__': 1276 | try: 1277 | import osscmdlib.pkginfo as pkginfo 1278 | from oss.oss_api import OssAPI 1279 | from oss.oss_xml_handler import * 1280 | from osscmdlib.utils import * 1281 | from osscmdlib.Config import Config 1282 | from osscmdlib.ossuri import * 1283 | main() 1284 | sys.exit(0) 1285 | 1286 | except ImportError, e: 1287 | report_exception(e) 1288 | sys.exit(1) 1289 | 1290 | except ParameterError, e: 1291 | error(u"Parameter problem: %s" % e) 1292 | sys.exit(1) 1293 | 1294 | except SystemExit, e: 1295 | sys.exit(e.code) 1296 | 1297 | except KeyboardInterrupt: 1298 | sys.stderr.write("The program stoped by interupt!\n") 1299 | sys.exit(1) 1300 | 1301 | except Exception, e: 1302 | report_exception(e) 1303 | sys.exit(1) 1304 | --------------------------------------------------------------------------------