├── .functests ├── .gitignore ├── .gitmodules ├── .gitreview ├── .unittests ├── HISTORY ├── README.md ├── bin ├── gluster-swift-gen-builders ├── gluster-swift-migrate-metadata ├── gluster-swift-object-expirer └── gluster-swift-print-metadata ├── doc ├── man │ └── gluster-swift-gen-builders.8 └── markdown │ ├── apache-deploy.md │ ├── auth_guide.md │ ├── dev_guide.md │ ├── object-expiration.md │ ├── openstack_swift_sync.md │ ├── quick_start_guide.md │ └── s3.md ├── etc ├── account-server.conf-gluster ├── container-server.conf-gluster ├── fs.conf-gluster ├── object-expirer.conf-gluster ├── object-server.conf-gluster ├── proxy-server.conf-gluster └── swift.conf-gluster ├── extras ├── apache-deploy │ ├── conf │ │ ├── account-server.wsgi │ │ ├── container-server.wsgi │ │ ├── object-server.wsgi │ │ ├── proxy-server.wsgi │ │ └── swift_wsgi.conf │ ├── install.sh │ └── uninstall.sh └── hook-scripts │ └── S40ufo-stop.py ├── gluster ├── __init__.py └── swift │ ├── __init__.py │ ├── account │ ├── __init__.py │ ├── server.py │ └── utils.py │ ├── common │ ├── DiskDir.py │ ├── Glusterfs.py │ ├── __init__.py │ ├── constraints.py │ ├── exceptions.py │ ├── fs_utils.py │ ├── middleware │ │ ├── __init__.py │ │ └── gswauth │ │ │ ├── .gitignore │ │ │ ├── .unittests │ │ │ ├── AUTHORS │ │ │ ├── CHANGELOG │ │ │ ├── LICENSE │ │ │ ├── MANIFEST.in │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── babel.cfg │ │ │ ├── bin │ │ │ ├── gswauth-add-account │ │ │ ├── gswauth-add-user │ │ │ ├── gswauth-cleanup-tokens │ │ │ ├── gswauth-delete-account │ │ │ ├── gswauth-delete-user │ │ │ ├── gswauth-list │ │ │ ├── gswauth-prep │ │ │ └── gswauth-set-account-service │ │ │ ├── doc │ │ │ └── source │ │ │ │ ├── _static │ │ │ │ └── .empty │ │ │ │ ├── _templates │ │ │ │ └── .empty │ │ │ │ ├── api.rst │ │ │ │ ├── authtypes.rst │ │ │ │ ├── conf.py │ │ │ │ ├── details.rst │ │ │ │ ├── index.rst │ │ │ │ ├── license.rst │ │ │ │ ├── middleware.rst │ │ │ │ └── swauth.rst │ │ │ ├── etc │ │ │ └── proxy-server.conf-sample │ │ │ ├── locale │ │ │ └── swauth.pot │ │ │ ├── setup.cfg │ │ │ ├── setup.py │ │ │ ├── swauth │ │ │ ├── __init__.py │ │ │ ├── authtypes.py │ │ │ ├── middleware.py │ │ │ └── swift_version.py │ │ │ └── webadmin │ │ │ └── index.html │ ├── ring.py │ └── utils.py │ ├── container │ ├── __init__.py │ └── server.py │ ├── obj │ ├── __init__.py │ ├── diskfile.py │ ├── expirer.py │ └── server.py │ └── proxy │ ├── __init__.py │ └── server.py ├── glusterfs-openstack-swift.spec ├── makerpm.sh ├── pkgconfig.py ├── requirements.txt ├── setup.py ├── test-requirements.txt ├── test ├── __init__.py ├── functional │ ├── __init__.py │ ├── gluster_swift_tests.py │ ├── swift_test_client.py │ ├── swift_testing.py │ ├── test_account.py │ ├── test_container.py │ ├── test_object.py │ └── tests.py ├── functional_auth │ ├── __init__.py │ ├── common_conf │ │ ├── account-server.conf │ │ ├── container-server.conf │ │ ├── fs.conf │ │ ├── object-expirer.conf │ │ ├── object-server.conf │ │ ├── swift.conf │ │ └── test.conf │ ├── gswauth │ │ ├── __init__.py │ │ ├── conf │ │ │ ├── proxy-server.conf │ │ │ └── test.conf │ │ ├── test_gswauth.py │ │ └── test_gswauth_cli.py │ ├── keystone │ │ └── conf │ │ │ ├── proxy-server.conf │ │ │ └── test.conf │ ├── swiftkerbauth │ │ └── conf │ │ │ └── proxy-server.conf │ └── tempauth │ │ └── conf │ │ ├── proxy-server.conf │ │ └── test.conf ├── object_expirer_functional │ ├── __init__.py │ ├── test_object_expirer_gluster_swift.py │ └── test_object_expirer_swift.py └── unit │ ├── __init__.py │ ├── account │ ├── __init__.py │ └── test_server.py │ ├── common │ ├── __init__.py │ ├── data │ │ ├── README.rings │ │ ├── account.builder │ │ ├── account.ring.gz │ │ ├── account_tree.tar.bz2 │ │ ├── container.builder │ │ ├── container.ring.gz │ │ ├── container_tree.tar.bz2 │ │ ├── object.builder │ │ └── object.ring.gz │ ├── middleware │ │ ├── __init__.py │ │ └── gswauth │ │ │ ├── __init__.py │ │ │ └── swauth │ │ │ ├── __init__.py │ │ │ ├── test_authtypes.py │ │ │ └── test_middleware.py │ ├── test_Glusterfs.py │ ├── test_constraints.py │ ├── test_diskdir.py │ ├── test_fs_utils.py │ ├── test_ring.py │ └── test_utils.py │ ├── container │ ├── __init__.py │ └── test_server.py │ ├── obj │ ├── __init__.py │ ├── test_diskfile.py │ ├── test_expirer.py │ └── test_server.py │ ├── proxy │ ├── __init__.py │ ├── controllers │ │ ├── __init__.py │ │ ├── test_account.py │ │ ├── test_base.py │ │ ├── test_container.py │ │ ├── test_info.py │ │ └── test_obj.py │ └── test_server.py │ └── test_swift.py ├── tools ├── gswauth_functional_tests.sh ├── keystone_functional_tests.sh ├── object_expirer_functional.sh ├── tempauth_functional_tests.sh └── tox_run.sh └── tox.ini /.functests: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (c) 2013 Red Hat, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 14 | # implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | # This program expects to be run by tox in a virtual python environment 19 | # so that it does not pollute the host development system 20 | 21 | EXIT_STATUS=0 22 | 23 | # Run functional tests with tempauth as auth middleware 24 | bash tools/tempauth_functional_tests.sh || EXIT_STATUS=$? 25 | 26 | # Run functional tests with gswauth as auth middleware 27 | bash tools/gswauth_functional_tests.sh || EXIT_STATUS=$? 28 | 29 | # Run object expirer functional tests if gsexpiring volume is found. 30 | if mount | grep "gsexpiring on /mnt/gluster-object/gsexpiring type fuse.glusterfs"; then 31 | echo "Running object expirer functional tests" 32 | bash tools/object_expirer_functional.sh || EXIT_STATUS=$? 33 | fi 34 | 35 | exit $EXIT_STATUS 36 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[co] 2 | *.sw? 3 | *~ 4 | dist 5 | build 6 | cover 7 | functional_tests_result 8 | .coverage 9 | *.egg 10 | *.egg-info 11 | .tox 12 | pycscope.* 13 | cscope.* 14 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "modules/swift"] 2 | path = modules/swift 3 | url = git://github.com/openstack/swift.git 4 | -------------------------------------------------------------------------------- /.gitreview: -------------------------------------------------------------------------------- 1 | [gerrit] 2 | host=review.gluster.org 3 | port=29418 4 | project=gluster-swift.git 5 | defaultbranch=master 6 | defaultremote=gerrit 7 | -------------------------------------------------------------------------------- /.unittests: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (c) 2013 Red Hat, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 14 | # implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | TOP_DIR=$(python -c "import os; print os.path.dirname(os.path.realpath('$0'))") 19 | 20 | python -c 'from distutils.version import LooseVersion as Ver; import nose, sys; sys.exit(0 if Ver(nose.__version__) >= Ver("1.2.0") else 1)' 21 | if [ $? != 0 ]; then 22 | cover_branches="" 23 | else 24 | # Having the HTML reports is REALLY useful for achieving 100% branch 25 | # coverage. 26 | cover_branches="--cover-branches --cover-html --cover-html-dir=$TOP_DIR/cover" 27 | fi 28 | cd $TOP_DIR/test/unit 29 | nosetests -v --exe --with-coverage --cover-package gluster. --cover-erase $cover_branches $@ 30 | rvalue=$? 31 | rm -f .coverage 32 | cd - 33 | exit $rvalue 34 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Gluster-Swift 2 | Gluster-Swift provides object interface to GlusterFS volumes. It allows files 3 | and directories created on GlusterFS volume to be accessed as objects via the 4 | OpenStack Swift and S3 API. 5 | 6 | Please refer to [quick start guide](doc/markdown/quick_start_guide.md) 7 | to get started. 8 | -------------------------------------------------------------------------------- /bin/gluster-swift-gen-builders: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Note that these port numbers must match the configured values for the 4 | # various servers in their configuration files. 5 | declare -A port=(["account.builder"]=6012 ["container.builder"]=6011 \ 6 | ["object.builder"]=6010) 7 | 8 | print_usage() { 9 | echo " 10 | NAME 11 | gluster-swift-gen-builders - Registers GlusterFS volumes to be accessed by 12 | object storage. 13 | SYNOPSIS 14 | gluster-swift-gen-builders [-v] [-h] volumes... 15 | DESCRIPTION 16 | Register GlusterFS volumes to be accessed over OpenStack Swift object 17 | storage. 18 | OPTIONS 19 | -v or --verbose 20 | Verbose 21 | -h or --help 22 | Prints help screen 23 | EXAMPLES 24 | gluster-swift-gen-builders myvol1 myvol2 25 | -Creates new ring files with myvol1 and myvol2 26 | 27 | gluster-swift-gen-builders myvol2 28 | -Creates new ring files by removing myvol1 29 | " 30 | } 31 | 32 | builder_files="account.builder container.builder object.builder" 33 | 34 | function create { 35 | swift-ring-builder $1 create 1 1 1 >> /tmp/out 36 | } 37 | 38 | function add { 39 | swift-ring-builder $1 add --region 1 --zone $2 --ip 127.0.0.1 --port $3 \ 40 | --device $4 --weight 100.0 --meta "_" 41 | } 42 | 43 | function rebalance { 44 | swift-ring-builder $1 rebalance 45 | } 46 | 47 | function build { 48 | swift-ring-builder $1 49 | } 50 | 51 | verbose=0 52 | outdev="/dev/null" 53 | 54 | if [ "$1" = "-v" ] || [ "$1" = "--verbose" ]; then 55 | verbose=1 56 | outdev="/dev/stdout" 57 | shift 58 | fi 59 | 60 | if [ "x$1" = "x" ]; then 61 | echo "Please specify the gluster volume name to use." 62 | print_usage 63 | exit 1 64 | fi 65 | 66 | if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then 67 | print_usage 68 | exit 0 69 | fi 70 | 71 | 72 | if ! cd "/etc/swift"; then 73 | echo "The path /etc/swift not accessible. Please check if it exists." 74 | exit 1 75 | fi 76 | 77 | for builder_file in $builder_files 78 | do 79 | create $builder_file 80 | 81 | zone=1 82 | for volname in $@ 83 | do 84 | add $builder_file $zone ${port[$builder_file]} $volname >& $outdev 85 | zone=$(expr $zone + 1) 86 | done 87 | 88 | rebalance $builder_file >& $outdev 89 | build $builder_file >& $outdev 90 | 91 | done 92 | 93 | echo "Ring files are prepared in /etc/swift. Please restart object store services" 94 | -------------------------------------------------------------------------------- /bin/gluster-swift-migrate-metadata: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright (c) 2015 Red Hat, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 14 | # implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | import os 19 | import pwd 20 | import sys 21 | import stat 22 | import errno 23 | import xattr 24 | import cPickle as pickle 25 | import multiprocessing 26 | 27 | from optparse import OptionParser 28 | from gluster.swift.common.utils import write_metadata, SafeUnpickler, \ 29 | METADATA_KEY, MAX_XATTR_SIZE 30 | 31 | 32 | ORIGINAL_EUID = os.geteuid() 33 | NOBODY_UID = pwd.getpwnam('nobody').pw_uid 34 | 35 | 36 | def print_msg(s): 37 | global options 38 | if options.verbose: 39 | print(s) 40 | 41 | 42 | def clean_metadata(path, key_count): 43 | """ 44 | Can only be used when you know the key_count. Saves one unnecessarry 45 | removexattr() call. Ignores error when file or metadata isn't found. 46 | """ 47 | for key in xrange(0, key_count): 48 | try: 49 | xattr.removexattr(path, '%s%s' % (METADATA_KEY, (key or ''))) 50 | except IOError as err: 51 | if err.errno not in (errno.ENOENT, errno.ESTALE, errno.ENODATA): 52 | print_msg("xattr.removexattr(%s, %s%s) failed: %s" % 53 | (path, METADATA_KEY, (key or ''), err.errno)) 54 | 55 | 56 | def process_object(path): 57 | 58 | metastr = '' 59 | key_count = 0 60 | try: 61 | while True: 62 | metastr += xattr.getxattr(path, '%s%s' % 63 | (METADATA_KEY, (key_count or ''))) 64 | key_count += 1 65 | if len(metastr) < MAX_XATTR_SIZE: 66 | # Prevent further getxattr calls 67 | break 68 | except IOError as err: 69 | if err.errno not in (errno.ENOENT, errno.ESTALE, errno.ENODATA): 70 | print_msg("xattr.getxattr(%s, %s%s) failed: %s" % 71 | (path, METADATA_KEY, (key_count or ''), err.errno)) 72 | 73 | if not metastr: 74 | return 75 | 76 | if metastr.startswith('\x80\x02}') and metastr.endswith('.'): 77 | # It's pickled. If unpickling is successful and metadata is 78 | # not stale write back the metadata by serializing it. 79 | try: 80 | os.seteuid(NOBODY_UID) # Drop privileges 81 | metadata = SafeUnpickler.loads(metastr) 82 | os.seteuid(ORIGINAL_EUID) # Restore privileges 83 | assert isinstance(metadata, dict) 84 | except (pickle.UnpicklingError, EOFError, AttributeError, 85 | IndexError, ImportError, AssertionError): 86 | clean_metadata(path, key_count) 87 | else: 88 | try: 89 | # Remove existing metadata first before writing new metadata 90 | clean_metadata(path, key_count) 91 | write_metadata(path, metadata) 92 | print_msg("%s MIGRATED" % (path)) 93 | except IOError as err: 94 | if err.errno not in (errno.ENOENT, errno.ESTALE): 95 | raise 96 | elif metastr.startswith("{") and metastr.endswith("}"): 97 | # It's not pickled and is already serialized, just return 98 | print_msg("%s SKIPPED" % (path)) 99 | else: 100 | # Metadata is malformed 101 | clean_metadata(path, key_count) 102 | print_msg("%s CLEANED" % (path)) 103 | 104 | 105 | def walktree(top, pool, root=True): 106 | """ 107 | Recursively walk the filesystem tree and migrate metadata of each object 108 | found. Unlike os.walk(), this method performs stat() sys call on a 109 | file/directory at most only once. 110 | """ 111 | 112 | if root: 113 | # The root of volume is account which also contains metadata 114 | pool.apply_async(process_object, (top, )) 115 | 116 | for f in os.listdir(top): 117 | if root and f in (".trashcan", ".glusterfs", "async_pending", "tmp"): 118 | continue 119 | path = os.path.join(top, f) 120 | try: 121 | s = os.stat(path) 122 | except OSError as err: 123 | if err.errno in (errno.ENOENT, errno.ESTALE): 124 | continue 125 | raise 126 | if stat.S_ISLNK(s.st_mode): 127 | pass 128 | elif stat.S_ISDIR(s.st_mode): 129 | pool.apply_async(process_object, (path, )) 130 | # Recurse into directory 131 | walktree(path, pool, root=False) 132 | elif stat.S_ISREG(s.st_mode): 133 | pool.apply_async(process_object, (path, )) 134 | 135 | 136 | if __name__ == '__main__': 137 | 138 | global options 139 | 140 | usage = "usage: %prog [options] volume1_mountpath volume2_mountpath..." 141 | description = """Account, container and object metadata are stored as \ 142 | extended attributes of files and directories. This utility migrates metadata \ 143 | stored in pickled format to JSON format.""" 144 | parser = OptionParser(usage=usage, description=description) 145 | parser.add_option("-v", "--verbose", dest="verbose", 146 | action="store_true", default=False, 147 | help="Print object paths as they are processed.") 148 | (options, mount_paths) = parser.parse_args() 149 | 150 | if len(mount_paths) < 1: 151 | print "Mountpoint path(s) missing." 152 | parser.print_usage() 153 | sys.exit(-1) 154 | 155 | pool = multiprocessing.Pool(multiprocessing.cpu_count() * 2) 156 | 157 | for path in mount_paths: 158 | if os.path.isdir(path): 159 | walktree(path, pool) 160 | 161 | pool.close() 162 | pool.join() 163 | -------------------------------------------------------------------------------- /bin/gluster-swift-object-expirer: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2016 Red Hat 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | # implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from swift.common.daemon import run_daemon 18 | from swift.common.utils import parse_options 19 | from gluster.swift.obj.expirer import ObjectExpirer 20 | from optparse import OptionParser 21 | 22 | if __name__ == '__main__': 23 | parser = OptionParser("%prog CONFIG [options]") 24 | parser.add_option('--processes', dest='processes', 25 | help="Number of processes to use to do the work, don't " 26 | "use this option to do all the work in one process") 27 | parser.add_option('--process', dest='process', 28 | help="Process number for this process, don't use " 29 | "this option to do all the work in one process, this " 30 | "is used to determine which part of the work this " 31 | "process should do") 32 | conf_file, options = parse_options(parser=parser, once=True) 33 | run_daemon(ObjectExpirer, conf_file, **options) 34 | -------------------------------------------------------------------------------- /bin/gluster-swift-print-metadata: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright (c) 2013 Red Hat, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 14 | # implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | import sys 19 | import pprint 20 | import os 21 | import json 22 | from optparse import OptionParser 23 | from gluster.swift.common.utils import read_metadata 24 | 25 | # Parser Setup 26 | USAGE = "Usage: %prog [options] OBJECT" 27 | DESCRIPTION = "Where OBJECT is a file or directory which "\ 28 | "its Gluster for Swift metadata will be printed "\ 29 | "to standard out" 30 | 31 | parser = OptionParser(usage=USAGE, description=DESCRIPTION) 32 | parser.add_option("-J", "--json", dest="json_format", action="store_true", 33 | default=False, help="Print output in JSON format") 34 | 35 | (options, args) = parser.parse_args() 36 | 37 | if len(args) < 1 or not os.path.exists(args[0]): 38 | parser.error("Filename missing") 39 | 40 | if options.json_format: 41 | print json.dumps(read_metadata(args[0])) 42 | else: 43 | pprint.pprint(read_metadata(args[0])) 44 | 45 | -------------------------------------------------------------------------------- /doc/man/gluster-swift-gen-builders.8: -------------------------------------------------------------------------------- 1 | .TH gluster-swift-gen-builders 8 "gluster-swift helper utility" "18 November 2013" "Red Hat Inc." 2 | .SH NAME 3 | \fBgluster-swift-gen-builders \fP- Registers GlusterFS volumes to be accessed by 4 | \fBOpenStack Swift. 5 | \fB 6 | .SH SYNOPSIS 7 | .nf 8 | .fam C 9 | \fBgluster-swift-gen-builders\fP [\fB-v\fP] [\fB-h\fP] volumes\.\.\. 10 | 11 | .fam T 12 | .fi 13 | .fam T 14 | .fi 15 | .SH DESCRIPTION 16 | Register GlusterFS volumes to be accessed over OpenStack Swift. 17 | .SH OPTIONS 18 | \fB-v\fP or \fB--verbose\fP 19 | .PP 20 | .nf 21 | .fam C 22 | Verbose 23 | 24 | .fam T 25 | .fi 26 | \fB-h\fP or \fB--help\fP 27 | .PP 28 | .nf 29 | .fam C 30 | Prints help screen 31 | 32 | .fam T 33 | .fi 34 | .SH EXAMPLES 35 | 36 | \fBgluster-swift-gen-builders\fP vol1 vol2 37 | .PP 38 | .nf 39 | .fam C 40 | Creates new ring files with vol1 and vol2 41 | 42 | .fam T 43 | .fi 44 | \fBgluster-swift-gen-builders\fP vol2 45 | .PP 46 | .nf 47 | .fam C 48 | Creates new ring files by removing vol1 49 | 50 | .fam T 51 | .fi 52 | \fBgluster-swift-gen-builders\fP \fB-v\fP vol1 53 | .PP 54 | .nf 55 | .fam C 56 | Create new ring files with vol1, (Verbose). 57 | 58 | .fam T 59 | .fi 60 | \fBgluster-swift-gen-builders\fP \fB-h\fP 61 | .PP 62 | .nf 63 | .fam C 64 | Displays help screen 65 | 66 | .fam T 67 | .fi 68 | .SH COPYRIGHT 69 | \fBCopyright\fP(c) 2013 RedHat, Inc. 70 | -------------------------------------------------------------------------------- /doc/markdown/object-expiration.md: -------------------------------------------------------------------------------- 1 | # Object Expiration 2 | 3 | ## Contents 4 | * [Overview](#overview) 5 | * [Setup](#setup) 6 | * [Using object expiration](#using) 7 | * [Running object-expirer daemon](#running-daemon) 8 | 9 | 10 | 11 | ## Overview 12 | The Object Expiration feature offers **scheduled deletion of objects**. The client would use the *X-Delete-At* or *X-Delete-After* headers during an object PUT or POST and the cluster would automatically quit serving that object at the specified time and would shortly thereafter remove the object from the GlusterFS volume. 13 | 14 | Expired objects however do appear in container listings until they are deleted by object-expirer daemon. This behaviour is expected: https://bugs.launchpad.net/swift/+bug/1069849 15 | 16 | 17 | 18 | ## Setup 19 | Object expirer uses a seprate account (a GlusterFS volume, for now, until multiple accounts per volume is implemented) named *gsexpiring*. You will have to [create a GlusterFS volume](quick_start_guide.md#gluster-volume-setup) by that name. 20 | 21 | Object-expirer uses the */etc/swift/object-expirer.conf* configuration file. Make sure that it exists. If not, you can copy it from */etc* directory of gluster-swift source repo. 22 | 23 | 24 | 25 | ## Using object expiration 26 | 27 | **PUT an object with X-Delete-At header using curl** 28 | 29 | ~~~ 30 | curl -v -X PUT -H 'X-Delete-At: 1392013619' http://127.0.0.1:8080/v1/AUTH_test/container1/object1 -T ./localfile 31 | ~~~ 32 | 33 | **PUT an object with X-Delete-At header using swift client** 34 | 35 | ~~~ 36 | swift --os-auth-token=AUTH_tk99a39aecc3dd4f80b2b1e801d00df846 --os-storage-url=http://127.0.0.1:8080/v1/AUTH_test upload container1 ./localfile --header 'X-Delete-At: 1392013619' 37 | ~~~ 38 | 39 | where *X-Delete-At* header takes a Unix Epoch timestamp in integer. For example, the current time in Epoch notation can be found by running this command: 40 | 41 | ~~~ 42 | date +%s 43 | ~~~ 44 | 45 | 46 | **PUT an object with X-Delete-After header using curl** 47 | 48 | ~~~ 49 | curl -v -X PUT -H 'X-Delete-After: 3600' http://127.0.0.1:8080/v1/AUTH_test/container1/object1 -T ./localfile 50 | ~~~ 51 | 52 | **PUT an object with X-Delete-At header using swift client** 53 | 54 | ~~~ 55 | swift --os-auth-token=AUTH_tk99a39aecc3dd4f80b2b1e801d00df846 --os-storage-url=http://127.0.0.1:8080/v1/AUTH_test upload container1 ./localfile --header 'X-Delete-After: 3600' 56 | ~~~ 57 | 58 | where *X-Delete-After* header takes a integer number of seconds, after which the object expires. The proxy server that receives the request will convert this header into an X-Delete-At header using its current time plus the value given. 59 | 60 | 61 | 62 | ## Running object-expirer daemon 63 | The object-expirer daemon runs a pass once every X seconds (configurable using *interval* option in config file). For every pass it makes, it queries the *gsexpiring* account for "tracker objects". Based on (timestamp, path) present in name of "tracker objects", object-expirer then deletes the actual object and the corresponding tracker object. 64 | 65 | 66 | To run object-expirer forever as a daemon: 67 | ~~~ 68 | swift-init object-expirer start 69 | ~~~ 70 | 71 | To run just once: 72 | ~~~ 73 | swift-object-expirer -o -v /etc/swift/object-expirer.conf 74 | ~~~ 75 | 76 | **For more information, visit:** 77 | http://docs.openstack.org/developer/swift/overview_expiring_objects.html 78 | 79 | 80 | -------------------------------------------------------------------------------- /doc/markdown/openstack_swift_sync.md: -------------------------------------------------------------------------------- 1 | # Syncing Gluster-Swift with Swift 2 | 3 | ## Create a release 4 | Create a release in launchpad.net so that we can place the latest swift source for download. We'll place the source here, and it will allow tox in gluster-swift to download the latest code. 5 | 6 | ## Upload swift release 7 | 8 | * Clone the git swift repo 9 | * Go to the release tag or just use the latest 10 | * Type the following to package the swift code: 11 | 12 | ``` 13 | $ python setup.py sdist 14 | $ ls dist 15 | ``` 16 | 17 | * Take the file in the `dist` directory and upload it to the new release we created it on launchpad.net. 18 | * Alternatively, if we are syncing with a Swift version which is already released, we can get the tar.gz file from Swift launchpad page and upload the same to gluster-swift launchpad. 19 | 20 | ## Setup Tox 21 | Now that the swift source is availabe on launchpad.net, copy its link location and update tox.ini in gluster-swift with the new link. 22 | 23 | ## Update tests 24 | This part is a little more complicated and now we need to *merge* the latest tests with ours. 25 | 26 | [meld](http://meldmerge.org/) is a great tool to make this work easier. The 3-way comparison feature of meld comes handy to compare 3 version of same file from: 27 | 28 | * Latest swift (say v1.13) 29 | * Previous swift (say v1.12) 30 | * gluster-swift (v1.12) 31 | 32 | Files that need to be merged: 33 | 34 | * Update unit tests 35 | 36 | ``` 37 | $ export SWIFTDIR=../swift 38 | $ meld $SWIFTDIR/tox.ini tox.ini 39 | $ meld $SWIFTDIR/test-requirements.txt tools/test-requires 40 | $ meld $SWIFTDIR/requirements.txt tools/requirements.txt 41 | $ meld $SWIFTDIR/test/unit/proxy/test_servers.py test/unit/proxy/test_server.py 42 | $ cp $SWIFTDIR/test/unit/proxy/controllers/*.py test/unit/proxy/controllers 43 | $ meld $SWIFTDIR/test/unit/__init__.py test/unit/__init__.py 44 | ``` 45 | 46 | * Update all the functional tests 47 | First check if there are any new files in the swift functional test directory. If there are, copy them over. 48 | 49 | * Remember to `git add` any new files 50 | 51 | * Now merge the existing ones: 52 | 53 | ``` 54 | for i in $SWIFTDIR/test/functional/*.py ; do 55 | meld $i test/functional/`basename $i` 56 | done 57 | ``` 58 | 59 | ## Update the version 60 | If needed, update the version now in `gluster/swift/__init__.py`. 61 | 62 | ## Upload the patch 63 | Upload the patch to Gerrit. 64 | 65 | ## Update the release in launchpad.net 66 | Upload the gluster-swift*.tar.gz built by Jenkins to launchpad.net once the fix has been commited to the main branch. 67 | 68 | -------------------------------------------------------------------------------- /etc/account-server.conf-gluster: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | # 3 | # Default gluster mount point to be used for object store,can be changed by 4 | # setting the following value in {account,container,object}-server.conf files. 5 | # It is recommended to keep this value same for all the three services but can 6 | # be kept different if environment demands. 7 | devices = /mnt/gluster-object 8 | # 9 | # Once you are confident that your startup processes will always have your 10 | # gluster volumes properly mounted *before* the account-server workers start, 11 | # you can *consider* setting this value to "false" to reduce the per-request 12 | # overhead it can incur. 13 | mount_check = true 14 | bind_port = 6012 15 | # 16 | # Override swift's default behaviour for fallocate. 17 | disable_fallocate = true 18 | # 19 | # One or two workers should be sufficient for almost any installation of 20 | # Gluster. 21 | workers = 1 22 | 23 | [pipeline:main] 24 | pipeline = account-server 25 | 26 | [app:account-server] 27 | use = egg:gluster_swift#account 28 | user = root 29 | log_facility = LOG_LOCAL2 30 | log_level = WARN 31 | # The following parameter is used by object-expirer and needs to be same 32 | # across all conf files! 33 | auto_create_account_prefix = gs 34 | # 35 | # After ensuring things are running in a stable manner, you can turn off 36 | # normal request logging for the account server to unclutter the log 37 | # files. Warnings and errors will still be logged. 38 | log_requests = off 39 | 40 | -------------------------------------------------------------------------------- /etc/container-server.conf-gluster: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | # 3 | # Default gluster mount point to be used for object store,can be changed by 4 | # setting the following value in {account,container,object}-server.conf files. 5 | # It is recommended to keep this value same for all the three services but can 6 | # be kept different if environment demands. 7 | devices = /mnt/gluster-object 8 | # 9 | # Once you are confident that your startup processes will always have your 10 | # gluster volumes properly mounted *before* the container-server workers 11 | # start, you can *consider* setting this value to "false" to reduce the 12 | # per-request overhead it can incur. 13 | mount_check = true 14 | bind_port = 6011 15 | # 16 | # Override swift's default behaviour for fallocate. 17 | disable_fallocate = true 18 | # 19 | # One or two workers should be sufficient for almost any installation of 20 | # Gluster. 21 | workers = 1 22 | 23 | [pipeline:main] 24 | pipeline = container-server 25 | 26 | [app:container-server] 27 | use = egg:gluster_swift#container 28 | user = root 29 | log_facility = LOG_LOCAL2 30 | log_level = WARN 31 | # The following parameters is used by object-expirer and needs to be same 32 | # across all conf files! 33 | auto_create_account_prefix = gs 34 | # 35 | # After ensuring things are running in a stable manner, you can turn off 36 | # normal request logging for the container server to unclutter the log 37 | # files. Warnings and errors will still be logged. 38 | log_requests = off 39 | 40 | -------------------------------------------------------------------------------- /etc/fs.conf-gluster: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | # 3 | # IP address of a node in the GlusterFS server cluster hosting the 4 | # volumes to be served via Swift API. 5 | mount_ip = localhost 6 | 7 | # Performance optimization parameter. When turned off, the filesystem will 8 | # see a reduced number of stat calls, resulting in substantially faster 9 | # response time for GET and HEAD container requests on containers with large 10 | # numbers of objects, at the expense of an accurate count of combined bytes 11 | # used by all objects in the container. For most installations "off" works 12 | # fine. 13 | accurate_size_in_listing = off 14 | 15 | # In older versions of gluster-swift, metadata stored as xattrs of dirs/files 16 | # were serialized using PICKLE format. The PICKLE format is vulnerable to 17 | # exploits in deployments where a user has access to backend filesystem over 18 | # FUSE/SMB. Deserializing pickled metadata can result in malicious code being 19 | # executed if an attacker has stored malicious code as xattr from filesystem 20 | # interface. Although, new metadata is always serialized using JSON format, 21 | # existing metadata already stored in PICKLE format are loaded by default. 22 | # You can turn this option to 'off' once you have migrated all your metadata 23 | # from PICKLE format to JSON format using gluster-swift-migrate-metadata tool. 24 | read_pickled_metadata = on 25 | -------------------------------------------------------------------------------- /etc/object-expirer.conf-gluster: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | user = root 3 | # Default gluster mount point to be used for object store,can be changed by 4 | # setting the following value in {account,container,object}-server.conf files. 5 | devices = /mnt/gluster-object 6 | 7 | [object-expirer] 8 | user = root 9 | log_facility = LOG_LOCAL2 10 | log_level = INFO 11 | 12 | # The following parameters are used by object-expirer and needs to be same 13 | # across all conf files! 14 | auto_create_account_prefix = gs 15 | expiring_objects_account_name = expiring 16 | 17 | # The expirer will re-attempt expiring if the source object is not available 18 | # up to reclaim_age seconds before it gives up and deletes the entry in the 19 | # queue. In gluster-swift, you'd almost always want to set this to zero. 20 | reclaim_age = 0 21 | 22 | # Do not retry DELETEs on getting 404. Hence default is set to 1. 23 | request_tries = 1 24 | 25 | # The swift-object-expirer daemon will run every 'interval' number of seconds 26 | # interval = 300 27 | 28 | # Emit a log line report of the progress so far every 'report_interval' 29 | # number of seconds. 30 | # report_interval = 300 31 | 32 | # concurrency is the level of concurrency to use to do the work, this value 33 | # must be set to at least 1 34 | # concurrency = 1 35 | 36 | # processes is how many parts to divide the work into, one part per process 37 | # that will be doing the work 38 | # processes set 0 means that a single process will be doing all the work 39 | # processes can also be specified on the command line and will override the 40 | # config value 41 | # processes = 0 42 | 43 | # process is which of the parts a particular process will work on 44 | # process can also be specified on the command line and will overide the config 45 | # value 46 | # process is "zero based", if you want to use 3 processes, you should run 47 | # processes with process set to 0, 1, and 2 48 | # process = 0 49 | 50 | 51 | [pipeline:main] 52 | pipeline = catch_errors cache proxy-server 53 | 54 | [app:proxy-server] 55 | use = egg:gluster_swift#proxy 56 | 57 | [filter:cache] 58 | use = egg:swift#memcache 59 | 60 | [filter:catch_errors] 61 | use = egg:swift#catch_errors 62 | -------------------------------------------------------------------------------- /etc/object-server.conf-gluster: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | # 3 | # Default gluster mount point to be used for object store,can be changed by 4 | # setting the following value in {account,container,object}-server.conf files. 5 | # It is recommended to keep this value same for all the three services but can 6 | # be kept different if environment demands. 7 | devices = /mnt/gluster-object 8 | # 9 | # Once you are confident that your startup processes will always have your 10 | # gluster volumes properly mounted *before* the object-server workers start, 11 | # you can *consider* setting this value to "false" to reduce the per-request 12 | # overhead it can incur. 13 | mount_check = true 14 | bind_port = 6010 15 | # 16 | # Maximum number of clients one worker can process simultaneously (it will 17 | # actually accept N + 1). Setting this to one (1) will only handle one request 18 | # at a time, without accepting another request concurrently. By increasing the 19 | # number of workers to a much higher value, one can prevent slow file system 20 | # operations for one request from starving other requests. 21 | max_clients = 1024 22 | # 23 | # If not doing the above, setting this value initially to match the number of 24 | # CPUs is a good starting point for determining the right value. 25 | workers = 1 26 | # Override swift's default behaviour for fallocate. 27 | disable_fallocate = true 28 | 29 | [pipeline:main] 30 | pipeline = object-server 31 | 32 | [app:object-server] 33 | use = egg:gluster_swift#object 34 | user = root 35 | log_facility = LOG_LOCAL2 36 | log_level = WARN 37 | # The following parameters are used by object-expirer and needs to be same 38 | # across all conf files! 39 | auto_create_account_prefix = gs 40 | expiring_objects_account_name = expiring 41 | # 42 | # For performance, after ensuring things are running in a stable manner, you 43 | # can turn off normal request logging for the object server to reduce the 44 | # per-request overhead and unclutter the log files. Warnings and errors will 45 | # still be logged. 46 | log_requests = off 47 | # 48 | # Adjust this value to match the stripe width of the underlying storage array 49 | # (not the stripe element size). This will provide a reasonable starting point 50 | # for tuning this value. 51 | disk_chunk_size = 65536 52 | # 53 | # Adjust this value match whatever is set for the disk_chunk_size initially. 54 | # This will provide a reasonable starting point for tuning this value. 55 | network_chunk_size = 65536 56 | -------------------------------------------------------------------------------- /etc/proxy-server.conf-gluster: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | bind_port = 8080 3 | user = root 4 | # Consider using 1 worker per CPU 5 | workers = 1 6 | 7 | [pipeline:main] 8 | pipeline = catch_errors healthcheck proxy-logging cache proxy-logging proxy-server 9 | 10 | [app:proxy-server] 11 | use = egg:gluster_swift#proxy 12 | log_facility = LOG_LOCAL1 13 | log_level = WARN 14 | # The API allows for account creation and deletion, but since Gluster/Swift 15 | # automounts a Gluster volume for a given account, there is no way to create 16 | # or delete an account. So leave this off. 17 | allow_account_management = false 18 | account_autocreate = true 19 | # The following parameters are used by object-expirer and needs to be same 20 | # across all conf files! 21 | auto_create_account_prefix = gs 22 | expiring_objects_account_name = expiring 23 | # Ensure the proxy server uses fast-POSTs since we don't need to make a copy 24 | # of the entire object given that all metadata is stored in the object 25 | # extended attributes (no .meta file used after creation) and no container 26 | # sync feature to present. 27 | object_post_as_copy = false 28 | # Only need to recheck the account exists once a day 29 | recheck_account_existence = 86400 30 | # May want to consider bumping this up if containers are created and destroyed 31 | # infrequently. 32 | recheck_container_existence = 60 33 | # Timeout clients that don't read or write to the proxy server after 5 34 | # seconds. 35 | client_timeout = 5 36 | # Give more time to connect to the object, container or account servers in 37 | # cases of high load. 38 | conn_timeout = 5 39 | # For high load situations, once connected to an object, container or account 40 | # server, allow for delays communicating with them. 41 | node_timeout = 60 42 | # May want to consider bumping up this value to 1 - 4 MB depending on how much 43 | # traffic is for multi-megabyte or gigabyte requests; perhaps matching the 44 | # stripe width (not stripe element size) of your storage volume is a good 45 | # starting point. See below for sizing information. 46 | object_chunk_size = 65536 47 | # If you do decide to increase the object_chunk_size, then consider lowering 48 | # this value to one. Up to "put_queue_length" object_chunk_size'd buffers can 49 | # be queued to the object server for processing. Given one proxy server worker 50 | # can handle up to 1,024 connections, by default, it will consume 10 * 65,536 51 | # * 1,024 bytes of memory in the worse case (default values). Be sure the 52 | # amount of memory available on the system can accommodate increased values 53 | # for object_chunk_size. 54 | put_queue_depth = 10 55 | 56 | [filter:catch_errors] 57 | use = egg:swift#catch_errors 58 | 59 | [filter:proxy-logging] 60 | use = egg:swift#proxy_logging 61 | access_log_level = WARN 62 | 63 | [filter:healthcheck] 64 | use = egg:swift#healthcheck 65 | 66 | [filter:cache] 67 | use = egg:swift#memcache 68 | # Update this line to contain a comma separated list of memcache servers 69 | # shared by all nodes running the proxy-server service. 70 | memcache_servers = localhost:11211 71 | -------------------------------------------------------------------------------- /etc/swift.conf-gluster: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | 3 | 4 | [swift-hash] 5 | # random unique string that can never change (DO NOT LOSE) 6 | swift_hash_path_suffix = gluster 7 | 8 | 9 | # The swift-constraints section sets the basic constraints on data 10 | # saved in the swift cluster. 11 | 12 | [swift-constraints] 13 | 14 | # max_file_size is the largest "normal" object that can be saved in 15 | # the cluster. This is also the limit on the size of each segment of 16 | # a "large" object when using the large object manifest support. 17 | # This value is set in bytes. Setting it to lower than 1MiB will cause 18 | # some tests to fail. 19 | # Default is 1 TiB = 2**30*1024 20 | max_file_size = 1099511627776 21 | 22 | 23 | # max_meta_name_length is the max number of bytes in the utf8 encoding 24 | # of the name portion of a metadata header. 25 | 26 | #max_meta_name_length = 128 27 | 28 | 29 | # max_meta_value_length is the max number of bytes in the utf8 encoding 30 | # of a metadata value 31 | 32 | #max_meta_value_length = 256 33 | 34 | 35 | # max_meta_count is the max number of metadata keys that can be stored 36 | # on a single account, container, or object 37 | 38 | #max_meta_count = 90 39 | 40 | 41 | # max_meta_overall_size is the max number of bytes in the utf8 encoding 42 | # of the metadata (keys + values) 43 | 44 | #max_meta_overall_size = 4096 45 | 46 | 47 | # max_object_name_length is the max number of bytes in the utf8 encoding of an 48 | # object name: Gluster FS can handle much longer file names, but the length 49 | # between the slashes of the URL is handled below. Remember that most web 50 | # clients can't handle anything greater than 2048, and those that do are 51 | # rather clumsy. 52 | 53 | max_object_name_length = 2048 54 | 55 | # max_object_name_component_length (GlusterFS) is the max number of bytes in 56 | # the utf8 encoding of an object name component (the part between the 57 | # slashes); this is a limit imposed by the underlying file system (for XFS it 58 | # is 255 bytes). 59 | 60 | max_object_name_component_length = 255 61 | 62 | # container_listing_limit is the default (and max) number of items 63 | # returned for a container listing request 64 | 65 | #container_listing_limit = 10000 66 | 67 | 68 | # account_listing_limit is the default (and max) number of items returned 69 | # for an account listing request 70 | 71 | #account_listing_limit = 10000 72 | 73 | 74 | # max_account_name_length is the max number of bytes in the utf8 encoding of 75 | # an account name: Gluster FS Filename limit (XFS limit?), must be the same 76 | # size as max_object_name_component_length above. 77 | 78 | max_account_name_length = 255 79 | 80 | 81 | # max_container_name_length is the max number of bytes in the utf8 encoding 82 | # of a container name: Gluster FS Filename limit (XFS limit?), must be the same 83 | # size as max_object_name_component_length above. 84 | 85 | max_container_name_length = 255 86 | -------------------------------------------------------------------------------- /extras/apache-deploy/conf/account-server.wsgi: -------------------------------------------------------------------------------- 1 | from swift.common.wsgi import init_request_processor 2 | application, conf, logger, log_name = \ 3 | init_request_processor('/etc/swift/account-server.conf','account-server') 4 | -------------------------------------------------------------------------------- /extras/apache-deploy/conf/container-server.wsgi: -------------------------------------------------------------------------------- 1 | from swift.common.wsgi import init_request_processor 2 | application, conf, logger, log_name = \ 3 | init_request_processor('/etc/swift/container-server.conf','container-server') 4 | -------------------------------------------------------------------------------- /extras/apache-deploy/conf/object-server.wsgi: -------------------------------------------------------------------------------- 1 | from swift.common.wsgi import init_request_processor 2 | application, conf, logger, log_name = \ 3 | init_request_processor('/etc/swift/object-server.conf','object-server') 4 | -------------------------------------------------------------------------------- /extras/apache-deploy/conf/proxy-server.wsgi: -------------------------------------------------------------------------------- 1 | from swift.common.wsgi import init_request_processor 2 | application, conf, logger, log_name = \ 3 | init_request_processor('/etc/swift/proxy-server.conf','proxy-server') 4 | -------------------------------------------------------------------------------- /extras/apache-deploy/conf/swift_wsgi.conf: -------------------------------------------------------------------------------- 1 | WSGISocketPrefix /var/run/wsgi 2 | 3 | #Proxy Service 4 | Listen 8080 5 | 6 | ServerName proxy-server 7 | LimitRequestBody 5368709122 8 | WSGIDaemonProcess proxy-server processes=5 threads=1 user=swift 9 | WSGIProcessGroup proxy-server 10 | WSGIScriptAlias / /var/www/swift/proxy-server.wsgi 11 | LimitRequestFields 200 12 | ErrorLog /var/log/httpd/proxy-server.log 13 | LogLevel debug 14 | CustomLog /var/log/httpd/proxy.log combined 15 | 16 | 17 | #Object Service 18 | Listen 6010 19 | 20 | ServerName object-server 21 | WSGIDaemonProcess object-server processes=5 threads=1 user=swift 22 | WSGIProcessGroup object-server 23 | WSGIScriptAlias / /var/www/swift/object-server.wsgi 24 | LimitRequestFields 200 25 | ErrorLog /var/log/httpd/object-server.log 26 | LogLevel debug 27 | CustomLog /var/log/httpd/access.log combined 28 | 29 | 30 | #Container Service 31 | Listen 6011 32 | 33 | ServerName container-server 34 | WSGIDaemonProcess container-server processes=5 threads=1 user=swift 35 | WSGIProcessGroup container-server 36 | WSGIScriptAlias / /var/www/swift/container-server.wsgi 37 | LimitRequestFields 200 38 | ErrorLog /var/log/httpd/container-server.log 39 | LogLevel debug 40 | CustomLog /var/log/httpd/access.log combined 41 | 42 | 43 | #Account Service 44 | Listen 6012 45 | 46 | ServerName account-server 47 | WSGIDaemonProcess account-server processes=5 threads=1 user=swift 48 | WSGIProcessGroup account-server 49 | WSGIScriptAlias / /var/www/swift/account-server.wsgi 50 | LimitRequestFields 200 51 | ErrorLog /var/log/httpd/account-server.log 52 | LogLevel debug 53 | CustomLog /var/log/httpd/access.log combined 54 | 55 | -------------------------------------------------------------------------------- /extras/apache-deploy/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -x 2 | 3 | # For Fedora/RHEL ONLY 4 | 5 | if [ $EUID -ne 0 ]; then 6 | echo "This script must be run as root" 7 | exit 1 8 | fi 9 | 10 | # Stop Apache and Swift services if running 11 | swift-init main stop 12 | service httpd stop 13 | 14 | # Install Apache and mod_wsgi 15 | yum install httpd mod_wsgi 16 | 17 | # Create a directory for Apache wsgi files 18 | mkdir -p /var/www/swift 19 | 20 | # Create a directory for swift which it'll use as home 21 | mkdir -p /var/lib/swift 22 | 23 | # Copy wsgi files for each of the four swift services 24 | cp ./conf/*wsgi /var/www/swift/ 25 | 26 | # Copy swift httpd config file 27 | cp ./conf/swift_wsgi.conf /etc/httpd/conf.d/ 28 | 29 | # Change owner of conf files to swift 30 | chown swift:swift /etc/swift/* 31 | 32 | # Check if SElinux is set to permissive/disabled 33 | selinux_mode=$(getenforce) 34 | if [ $selinux_mode == "Enforcing" ]; then 35 | echo "SElinux is set to Enforcing. Change it to Permissive or Disabled \ 36 | by editing /etc/sysconfig/selinux" 37 | echo "You will need to reboot your system for the changed value to take \ 38 | effect." 39 | exit 1 40 | fi 41 | 42 | echo "Successfully configured Apache as frontend for Swift." 43 | echo "Make sure GlusterFS volume is mounted at /mnt/gluster-object/ \ 44 | before starting httpd" 45 | -------------------------------------------------------------------------------- /extras/apache-deploy/uninstall.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -x 2 | 3 | # For Fedora/RHEL ONLY 4 | 5 | if [ $EUID -ne 0 ]; then 6 | echo "This script must be run as root" 7 | exit 1 8 | fi 9 | 10 | # Stop Apache service 11 | service httpd stop 12 | 13 | # Remove swift wsgi files 14 | rm -rf /var/www/swift 15 | 16 | # Remove swift httpd config file 17 | rm -f /etc/httpd/conf.d/swift_wsgi.conf 18 | 19 | echo -e "DONE.\nYou can now restart Swift." 20 | -------------------------------------------------------------------------------- /extras/hook-scripts/S40ufo-stop.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright (c) 2013 Red Hat, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 14 | # implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | 19 | from optparse import OptionParser 20 | 21 | if __name__ == '__main__': 22 | # check if swift is installed 23 | try: 24 | from gluster.swift.common.Glusterfs import get_mnt_point, unmount 25 | except ImportError: 26 | import sys 27 | sys.exit("Openstack Swift does not appear to be installed properly") 28 | 29 | op = OptionParser(usage="%prog [options...]") 30 | op.add_option('--volname', dest='vol', type=str) 31 | op.add_option('--last', dest='last', type=str) 32 | (opts, args) = op.parse_args() 33 | 34 | mnt_point = get_mnt_point(opts.vol) 35 | if mnt_point: 36 | unmount(mnt_point) 37 | else: 38 | sys.exit("get_mnt_point returned none for mount point") 39 | -------------------------------------------------------------------------------- /gluster/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2016 Red Hat, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # Share gluster namespace 17 | from pkgutil import extend_path 18 | __path__ = extend_path(__path__, __name__) 19 | -------------------------------------------------------------------------------- /gluster/swift/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2012-2013 Red Hat, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | """ Gluster for Swift """ 17 | 18 | 19 | class PkgInfo(object): 20 | def __init__(self, canonical_version, release, name, final): 21 | self.canonical_version = canonical_version 22 | self.release = release 23 | self.name = name 24 | self.final = final 25 | self.full_version = self.canonical_version + '-' + self.release 26 | 27 | def save_config(self, filename): 28 | """ 29 | Creates a file with the package configuration which can be sourced by 30 | a bash script. 31 | """ 32 | with open(filename, 'w') as fd: 33 | fd.write("NAME=%s\n" % self.name) 34 | fd.write("VERSION=%s\n" % self.canonical_version) 35 | fd.write("RELEASE=%s\n" % self.release) 36 | 37 | @property 38 | def pretty_version(self): 39 | if self.final: 40 | return self.canonical_version 41 | else: 42 | return '%s-dev' % (self.canonical_version,) 43 | 44 | 45 | # 46 | # Change the Package version here 47 | # 48 | _pkginfo = PkgInfo('2.15.1', '0', 'gluster_swift', False) 49 | __version__ = _pkginfo.pretty_version 50 | __canonical_version__ = _pkginfo.canonical_version 51 | -------------------------------------------------------------------------------- /gluster/swift/account/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gluster/gluster-swift/488744a005fb399af8d094ad7a62c1917410398c/gluster/swift/account/__init__.py -------------------------------------------------------------------------------- /gluster/swift/account/server.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2012-2013 Red Hat, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | """ Account Server for Gluster Swift UFO """ 17 | 18 | # Simply importing this monkey patches the constraint handling to fit our 19 | # needs 20 | import gluster.swift.common.constraints # noqa 21 | 22 | from swift.account import server 23 | from gluster.swift.common.DiskDir import DiskAccount 24 | 25 | 26 | class AccountController(server.AccountController): 27 | 28 | def _get_account_broker(self, drive, part, account, **kwargs): 29 | """ 30 | Overriden to provide the GlusterFS specific broker that talks to 31 | Gluster for the information related to servicing a given request 32 | instead of talking to a database. 33 | 34 | :param drive: drive that holds the container 35 | :param part: partition the container is in 36 | :param account: account name 37 | :returns: DiskDir object 38 | """ 39 | return DiskAccount(self.root, drive, account, self.logger, **kwargs) 40 | 41 | 42 | def app_factory(global_conf, **local_conf): 43 | """paste.deploy app factory for creating WSGI account server apps.""" 44 | conf = global_conf.copy() 45 | conf.update(local_conf) 46 | return AccountController(conf) 47 | -------------------------------------------------------------------------------- /gluster/swift/account/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2016 Red Hat, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from swift.account.utils import FakeAccountBroker, get_response_headers 17 | from swift.common.swob import HTTPOk, HTTPNoContent 18 | from swift.common.utils import json, Timestamp 19 | from xml.sax import saxutils 20 | 21 | 22 | def account_listing_response(account, req, response_content_type, broker=None, 23 | limit='', marker='', end_marker='', prefix='', 24 | delimiter='', reverse=False): 25 | """ 26 | This is an exact copy of swift.account.utis.account_listing_response() 27 | except for one difference i.e this method passes response_content_type 28 | to broker.list_containers_iter() method. 29 | """ 30 | if broker is None: 31 | broker = FakeAccountBroker() 32 | 33 | resp_headers = get_response_headers(broker) 34 | 35 | account_list = broker.list_containers_iter(limit, marker, end_marker, 36 | prefix, delimiter, 37 | response_content_type, reverse) 38 | if response_content_type == 'application/json': 39 | data = [] 40 | for (name, object_count, bytes_used, put_tstamp, 41 | is_subdir) in account_list: 42 | if is_subdir: 43 | data.append({'subdir': name}) 44 | else: 45 | data.append({'name': name, 'count': object_count, 46 | 'bytes': bytes_used, 47 | 'last_modified': Timestamp(put_tstamp).isoformat}) 48 | account_list = json.dumps(data) 49 | elif response_content_type.endswith('/xml'): 50 | output_list = ['', 51 | '' % saxutils.quoteattr(account)] 52 | for (name, object_count, bytes_used, put_tstamp, 53 | is_subdir) in account_list: 54 | if is_subdir: 55 | output_list.append( 56 | '' % saxutils.quoteattr(name)) 57 | else: 58 | item = '%s%s' \ 59 | '%s%s \ 60 | ' % \ 61 | (saxutils.escape(name), object_count, bytes_used, 62 | Timestamp(put_tstamp).isoformat) 63 | output_list.append(item) 64 | output_list.append('') 65 | account_list = '\n'.join(output_list) 66 | else: 67 | if not account_list: 68 | resp = HTTPNoContent(request=req, headers=resp_headers) 69 | resp.content_type = response_content_type 70 | resp.charset = 'utf-8' 71 | return resp 72 | account_list = '\n'.join(r[0] for r in account_list) + '\n' 73 | ret = HTTPOk(body=account_list, request=req, headers=resp_headers) 74 | ret.content_type = response_content_type 75 | ret.charset = 'utf-8' 76 | return ret 77 | -------------------------------------------------------------------------------- /gluster/swift/common/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gluster/gluster-swift/488744a005fb399af8d094ad7a62c1917410398c/gluster/swift/common/__init__.py -------------------------------------------------------------------------------- /gluster/swift/common/constraints.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2012-2013 Red Hat, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import os 17 | from swift.common.swob import HTTPBadRequest 18 | import swift.common.constraints 19 | import swift.common.ring as _ring 20 | from gluster.swift.common import Glusterfs, ring 21 | 22 | MAX_OBJECT_NAME_COMPONENT_LENGTH = 255 23 | 24 | 25 | def set_object_name_component_length(len=None): 26 | global MAX_OBJECT_NAME_COMPONENT_LENGTH 27 | 28 | if len: 29 | MAX_OBJECT_NAME_COMPONENT_LENGTH = len 30 | elif hasattr(swift.common.constraints, 'constraints_conf_int'): 31 | MAX_OBJECT_NAME_COMPONENT_LENGTH = \ 32 | swift.common.constraints.constraints_conf_int( 33 | 'max_object_name_component_length', 255) 34 | else: 35 | MAX_OBJECT_NAME_COMPONENT_LENGTH = 255 36 | return 37 | 38 | set_object_name_component_length() 39 | 40 | 41 | def get_object_name_component_length(): 42 | return MAX_OBJECT_NAME_COMPONENT_LENGTH 43 | 44 | 45 | def validate_obj_name_component(obj): 46 | if not obj: 47 | return 'cannot begin, end, or have contiguous %s\'s' % os.path.sep 48 | if len(obj) > MAX_OBJECT_NAME_COMPONENT_LENGTH: 49 | return 'too long (%d)' % len(obj) 50 | if obj == '.' or obj == '..': 51 | return 'cannot be . or ..' 52 | return '' 53 | 54 | 55 | # Save the original check object creation 56 | __check_object_creation = swift.common.constraints.check_object_creation 57 | 58 | 59 | # Define our new one which invokes the original 60 | def gluster_check_object_creation(req, object_name): 61 | """ 62 | Check to ensure that everything is alright about an object to be created. 63 | Monkey patches swift.common.constraints.check_object_creation, invoking 64 | the original, and then adding an additional check for individual object 65 | name components. 66 | 67 | :param req: HTTP request object 68 | :param object_name: name of object to be created 69 | :raises HTTPRequestEntityTooLarge: the object is too large 70 | :raises HTTPLengthRequered: missing content-length header and not 71 | a chunked request 72 | :raises HTTPBadRequest: missing or bad content-type header, or 73 | bad metadata 74 | """ 75 | ret = __check_object_creation(req, object_name) 76 | 77 | if ret is None: 78 | for obj in object_name.split(os.path.sep): 79 | reason = validate_obj_name_component(obj) 80 | if reason: 81 | bdy = 'Invalid object name "%s", component "%s" %s' \ 82 | % (object_name, obj, reason) 83 | ret = HTTPBadRequest(body=bdy, 84 | request=req, 85 | content_type='text/plain') 86 | 87 | return ret 88 | 89 | # Replace the original checks with ours 90 | swift.common.constraints.check_object_creation = gluster_check_object_creation 91 | 92 | # Replace the original check mount with ours 93 | swift.common.constraints.check_mount = Glusterfs.mount 94 | 95 | # Save the original Ring class 96 | __Ring = _ring.Ring 97 | 98 | # Replace the original Ring class 99 | _ring.Ring = ring.Ring 100 | 101 | # Monkey patch account_listing_response 102 | import swift.account.utils 103 | from gluster.swift.account.utils import account_listing_response as gf_als 104 | swift.account.utils.account_listing_response = gf_als 105 | 106 | # Monkey patch StoragePolicy.load_ring as POLICIES are initialized already 107 | from swift.common.storage_policy import StoragePolicy 108 | 109 | 110 | def load_ring(self, swift_dir): 111 | if self.object_ring: 112 | return 113 | self.object_ring = ring.Ring(swift_dir, ring_name='object') 114 | 115 | StoragePolicy.load_ring = load_ring 116 | -------------------------------------------------------------------------------- /gluster/swift/common/exceptions.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2012-2013 Red Hat, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from swift.common.exceptions import SwiftException 17 | 18 | 19 | class GlusterFileSystemOSError(OSError): 20 | pass 21 | 22 | 23 | class GlusterFileSystemIOError(IOError): 24 | pass 25 | 26 | 27 | class GlusterfsException(Exception): 28 | pass 29 | 30 | 31 | class FailureToMountError(GlusterfsException): 32 | pass 33 | 34 | 35 | class FileOrDirNotFoundError(GlusterfsException): 36 | pass 37 | 38 | 39 | class NotDirectoryError(GlusterfsException): 40 | pass 41 | 42 | 43 | class AlreadyExistsAsDir(GlusterfsException): 44 | pass 45 | 46 | 47 | class AlreadyExistsAsFile(GlusterfsException): 48 | pass 49 | 50 | 51 | class DiskFileContainerDoesNotExist(GlusterfsException): 52 | pass 53 | 54 | 55 | class ThreadPoolDead(SwiftException): 56 | pass 57 | -------------------------------------------------------------------------------- /gluster/swift/common/middleware/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gluster/gluster-swift/488744a005fb399af8d094ad7a62c1917410398c/gluster/swift/common/middleware/__init__.py -------------------------------------------------------------------------------- /gluster/swift/common/middleware/gswauth/.gitignore: -------------------------------------------------------------------------------- 1 | *.egg-info 2 | *.py[co] 3 | .DS_Store 4 | -------------------------------------------------------------------------------- /gluster/swift/common/middleware/gswauth/.unittests: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | nosetests test_swauth/unit --exe --with-coverage --cover-package swauth --cover-erase 4 | rm -f .coverage 5 | -------------------------------------------------------------------------------- /gluster/swift/common/middleware/gswauth/AUTHORS: -------------------------------------------------------------------------------- 1 | Maintainer 2 | ---------- 3 | Greg Holt 4 | 5 | Original Authors 6 | ---------------- 7 | Chuck Thier 8 | Greg Holt 9 | Greg Lange 10 | Jay Payne 11 | John Dickinson 12 | Michael Barton 13 | Will Reese 14 | 15 | Contributors 16 | ------------ 17 | Andrew Clay Shafer 18 | Anne Gentle 19 | Brian K. Jones 20 | Caleb Tennis 21 | Chmouel Boudjnah 22 | Christian Schwede 23 | Chris Wedgwood 24 | Clay Gerrard 25 | Colin Nicholson 26 | Conrad Weidenkeller 27 | Cory Wright 28 | David Goetz 29 | Ed Leafe 30 | Fujita Tomonori 31 | Kapil Thangavelu 32 | Monty Taylor 33 | Pablo Llopis 34 | Paul Jimenez 35 | Pete Zaitcev 36 | Russ Nelson 37 | Scott Simpson 38 | Soren Hansen 39 | Stephen Milton 40 | -------------------------------------------------------------------------------- /gluster/swift/common/middleware/gswauth/CHANGELOG: -------------------------------------------------------------------------------- 1 | swauth (1.0.8) 2 | 3 | Added request.environ[reseller_request] = True if request is coming from an 4 | user in .reseller_admin group 5 | 6 | Fixed to work with newer Swift versions whose memcache clients require a 7 | time keyword argument when the older versions required a timeout keyword 8 | argument. 9 | 10 | swauth (1.0.7) 11 | 12 | New X-Auth-Token-Lifetime header a user can set to how long they'd like 13 | their token to be good for. 14 | 15 | New max_token_life config value for capping the above. 16 | 17 | New X-Auth-Token-Expires header returned with the get token request. 18 | 19 | Switchover to swift.common.swob instead of WebOb; requires Swift >= 1.7.6 20 | now. 21 | 22 | swauth (1.0.6) 23 | 24 | Apparently I haven't been keeping up with this CHANGELOG. I'll try to be 25 | better onward. 26 | 27 | This release added passing OPTIONS requests through untouched, needed for 28 | CORS support in Swift. 29 | 30 | Also, Swauth is a bit more restrictive in deciding when it's the definitive 31 | auth for a request. 32 | 33 | swauth (1.0.3-dev) 34 | 35 | This release is still under development. A full change log will be made at 36 | release. Until then, you can see what has changed with: 37 | 38 | git log 1.0.2..HEAD 39 | 40 | swauth (1.0.2) 41 | 42 | Fixed bug rejecting requests when using multiple instances of Swauth or 43 | Swauth with other auth services. 44 | 45 | Fixed bug interpreting URL-encoded user names and keys. 46 | 47 | Added support for the Swift container sync feature. 48 | 49 | Allowed /not/ setting super_admin_key to disable Swauth administration 50 | features. 51 | 52 | Added swauth_remote mode so the Swauth middleware for one Swift cluster 53 | could be pointing to the Swauth service on another Swift cluster, sharing 54 | account/user data sets. 55 | 56 | Added ability to purge stored tokens. 57 | 58 | Added API documentation for internal Swauth API. 59 | 60 | swauth (1.0.1) 61 | 62 | Initial release after separation from Swift. 63 | -------------------------------------------------------------------------------- /gluster/swift/common/middleware/gswauth/MANIFEST.in: -------------------------------------------------------------------------------- 1 | include AUTHORS LICENSE README.md .unittests test_swauth/__init__.py 2 | include CHANGELOG 3 | graft doc 4 | graft etc 5 | -------------------------------------------------------------------------------- /gluster/swift/common/middleware/gswauth/README.md: -------------------------------------------------------------------------------- 1 | Swauth 2 | ------ 3 | 4 | An Auth Service for Swift as WSGI Middleware that uses Swift itself as a 5 | backing store. Sphinx-built docs at: 6 | 7 | See also for the standard OpenStack 8 | auth service. 9 | 10 | 11 | NOTE 12 | ---- 13 | 14 | **Be sure to review the Sphinx-built docs at: 15 | ** 16 | 17 | 18 | Quick Install 19 | ------------- 20 | 21 | 1) Install Swauth with ``sudo python setup.py install`` or ``sudo python 22 | setup.py develop`` or via whatever packaging system you may be using. 23 | 24 | 2) Alter your proxy-server.conf pipeline to have swauth instead of tempauth: 25 | 26 | Was: 27 | 28 | [pipeline:main] 29 | pipeline = catch_errors cache tempauth proxy-server 30 | 31 | Change To: 32 | 33 | [pipeline:main] 34 | pipeline = catch_errors cache swauth proxy-server 35 | 36 | 3) Add to your proxy-server.conf the section for the Swauth WSGI filter: 37 | 38 | [filter:swauth] 39 | use = egg:swauth#swauth 40 | set log_name = swauth 41 | super_admin_key = swauthkey 42 | 43 | 4) Be sure your proxy server allows account management: 44 | 45 | [app:proxy-server] 46 | ... 47 | allow_account_management = true 48 | 49 | 5) Restart your proxy server ``swift-init proxy reload`` 50 | 51 | 6) Initialize the Swauth backing store in Swift ``swauth-prep -K swauthkey`` 52 | 53 | 7) Add an account/user ``swauth-add-user -A http://127.0.0.1:8080/auth/ -K 54 | swauthkey -a test tester testing`` 55 | 56 | 8) Ensure it works ``swift -A http://127.0.0.1:8080/auth/v1.0 -U test:tester -K 57 | testing stat -v`` 58 | 59 | 60 | Web Admin Install 61 | ----------------- 62 | 63 | 1) If you installed from packages, you'll need to cd to the webadmin directory 64 | the package installed. This is ``/usr/share/doc/python-swauth/webadmin`` 65 | with the Lucid packages. If you installed from source, you'll need to cd to 66 | the webadmin directory in the source directory. 67 | 68 | 2) Upload the Web Admin files with ``swift -A http://127.0.0.1:8080/auth/v1.0 69 | -U .super_admin:.super_admin -K swauthkey upload .webadmin .`` 70 | 71 | 3) Open ``http://127.0.0.1:8080/auth/`` in your browser. 72 | -------------------------------------------------------------------------------- /gluster/swift/common/middleware/gswauth/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gluster/gluster-swift/488744a005fb399af8d094ad7a62c1917410398c/gluster/swift/common/middleware/gswauth/__init__.py -------------------------------------------------------------------------------- /gluster/swift/common/middleware/gswauth/babel.cfg: -------------------------------------------------------------------------------- 1 | [python: **.py] 2 | 3 | -------------------------------------------------------------------------------- /gluster/swift/common/middleware/gswauth/bin/gswauth-add-account: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2010-2011 OpenStack, LLC. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | # implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import gettext 18 | import socket 19 | 20 | from optparse import OptionParser 21 | from sys import argv, exit 22 | 23 | from swift.common.bufferedhttp import http_connect_raw as http_connect 24 | from swift.common.utils import urlparse 25 | 26 | 27 | if __name__ == '__main__': 28 | gettext.install('gswauth', unicode=1) 29 | parser = OptionParser(usage='Usage: %prog [options] ') 30 | parser.add_option('-A', '--admin-url', dest='admin_url', 31 | default='http://127.0.0.1:8080/auth/', help='The URL to the auth ' 32 | 'subsystem (default: http://127.0.0.1:8080/auth/)') 33 | parser.add_option('-U', '--admin-user', dest='admin_user', 34 | default='.super_admin', help='The user with admin rights to add ' 35 | 'accounts (default: .super_admin).') 36 | parser.add_option('-K', '--admin-key', dest='admin_key', 37 | help='The key for the user with admin rights is required.') 38 | args = argv[1:] 39 | if not args: 40 | args.append('-h') 41 | (options, args) = parser.parse_args(args) 42 | if len(args) != 1: 43 | parser.parse_args(['-h']) 44 | if options.admin_key is None: 45 | parser.parse_args(['-h']) 46 | account = args[0] 47 | parsed = urlparse(options.admin_url) 48 | if parsed.scheme not in ('http', 'https'): 49 | raise Exception('Cannot handle protocol scheme %s for url %s' % 50 | (parsed.scheme, repr(options.admin_url))) 51 | parsed_path = parsed.path 52 | if not parsed_path: 53 | parsed_path = '/' 54 | elif parsed_path[-1] != '/': 55 | parsed_path += '/' 56 | path = '%sv2/%s' % (parsed_path, account) 57 | headers = {'X-Auth-Admin-User': options.admin_user, 58 | 'X-Auth-Admin-Key': options.admin_key, 59 | 'Content-Length': '0'} 60 | try: 61 | conn = http_connect(parsed.hostname, parsed.port, 'PUT', path, headers, 62 | ssl=(parsed.scheme == 'https')) 63 | resp = conn.getresponse() 64 | except socket.gaierror, err: 65 | exit('Account creation failed: %s. ' \ 66 | 'Check that the admin_url is valid' % err) 67 | except socket.error, msg: 68 | exit('Account creation failed: %s. ' \ 69 | 'Check that the admin_url is valid' % msg) 70 | 71 | if resp.status // 100 != 2: 72 | if resp.status == 401: 73 | exit('Account creation failed: %s %s: Invalid user/key provided' % 74 | (resp.status, resp.reason)) 75 | elif resp.status == 403: 76 | exit('Account creation failed: %s %s: Insufficient privileges' % 77 | (resp.status, resp.reason)) 78 | else: 79 | exit('Account creation failed: %s %s' % 80 | (resp.status, resp.reason)) 81 | -------------------------------------------------------------------------------- /gluster/swift/common/middleware/gswauth/bin/gswauth-add-user: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2010-2011 OpenStack, LLC. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | # implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import gettext 18 | import socket 19 | 20 | from optparse import OptionParser 21 | from sys import argv, exit 22 | 23 | from swift.common.bufferedhttp import http_connect_raw as http_connect 24 | from swift.common.utils import urlparse 25 | 26 | 27 | if __name__ == '__main__': 28 | gettext.install('gswauth', unicode=1) 29 | parser = OptionParser( 30 | usage='Usage: %prog [options] ') 31 | parser.add_option('-a', '--admin', dest='admin', action='store_true', 32 | default=False, help='Give the user administrator access; otherwise ' 33 | 'the user will only have access to containers specifically allowed ' 34 | 'with ACLs.') 35 | parser.add_option('-r', '--reseller-admin', dest='reseller_admin', 36 | action='store_true', default=False, help='Give the user full reseller ' 37 | 'administrator access, giving them full access to all accounts within ' 38 | 'the reseller, including the ability to create new accounts. Creating ' 39 | 'a new reseller admin requires super_admin rights.') 40 | parser.add_option('-A', '--admin-url', dest='admin_url', 41 | default='http://127.0.0.1:8080/auth/', help='The URL to the auth ' 42 | 'subsystem (default: http://127.0.0.1:8080/auth/') 43 | parser.add_option('-U', '--admin-user', dest='admin_user', 44 | default='.super_admin', help='The user with admin rights to add users ' 45 | '(default: .super_admin).') 46 | parser.add_option('-K', '--admin-key', dest='admin_key', 47 | help='The key for the user with admin rights to add users is required.') 48 | args = argv[1:] 49 | if not args: 50 | args.append('-h') 51 | (options, args) = parser.parse_args(args) 52 | if len(args) != 3: 53 | parser.parse_args(['-h']) 54 | if options.admin_key is None: 55 | parser.parse_args(['-h']) 56 | account, user, password = args 57 | parsed = urlparse(options.admin_url) 58 | if parsed.scheme not in ('http', 'https'): 59 | raise Exception('Cannot handle protocol scheme %s for url %s' % 60 | (parsed.scheme, repr(options.admin_url))) 61 | parsed_path = parsed.path 62 | if not parsed_path: 63 | parsed_path = '/' 64 | elif parsed_path[-1] != '/': 65 | parsed_path += '/' 66 | 67 | # Check if user is changing his own password. This is carried out by 68 | # making sure that the user changing the password and the user whose 69 | # password is being changed are the same. 70 | # If not, ensure that the account exists before creating new user. 71 | if not options.admin_user == (account + ':' + user): 72 | # GET the account 73 | path = '%sv2/%s' % (parsed_path, account) 74 | headers = {'X-Auth-Admin-User': options.admin_user, 75 | 'X-Auth-Admin-Key': options.admin_key} 76 | try: 77 | conn = http_connect(parsed.hostname, parsed.port, 'GET', path, 78 | headers, ssl=(parsed.scheme == 'https')) 79 | resp = conn.getresponse() 80 | if resp.status // 100 != 2: 81 | # If the GET operation fails, it means the account does not 82 | # exist. Now we create the account by sending a PUT request. 83 | headers['Content-Length'] = '0' 84 | conn = http_connect(parsed.hostname, parsed.port, 'PUT', path, 85 | headers, ssl=(parsed.scheme == 'https')) 86 | resp = conn.getresponse() 87 | if resp.status // 100 != 2: 88 | print 'Account creation failed: %s %s' % \ 89 | (resp.status, resp.reason) 90 | except socket.gaierror, err: 91 | exit('User creation failed: %s. ' \ 92 | 'Check that the admin_url is valid' % err) 93 | except socket.error, msg: 94 | exit('User creation failed: %s. ' \ 95 | 'Check that the admin_url is valid' % msg) 96 | 97 | # Add the user 98 | path = '%sv2/%s/%s' % (parsed_path, account, user) 99 | headers = {'X-Auth-Admin-User': options.admin_user, 100 | 'X-Auth-Admin-Key': options.admin_key, 101 | 'X-Auth-User-Key': password, 102 | 'Content-Length': '0'} 103 | if options.admin: 104 | headers['X-Auth-User-Admin'] = 'true' 105 | if options.reseller_admin: 106 | headers['X-Auth-User-Reseller-Admin'] = 'true' 107 | try: 108 | conn = http_connect(parsed.hostname, parsed.port, 'PUT', path, headers, 109 | ssl=(parsed.scheme == 'https')) 110 | resp = conn.getresponse() 111 | except socket.gaierror, err: 112 | exit('User creation failed: %s. ' \ 113 | 'Check that the admin_url is valid' % err) 114 | except socket.error, msg: 115 | exit('User creation failed: %s. ' \ 116 | 'Check that the admin_url is valid' % msg) 117 | 118 | if resp.status // 100 != 2: 119 | if resp.status == 401: 120 | exit('User creation failed: %s %s: Invalid user/key provided' % 121 | (resp.status, resp.reason)) 122 | elif resp.status == 403: 123 | exit('User creation failed: %s %s: Insufficient privileges' % 124 | (resp.status, resp.reason)) 125 | else: 126 | exit('User creation failed: %s %s' % 127 | (resp.status, resp.reason)) 128 | -------------------------------------------------------------------------------- /gluster/swift/common/middleware/gswauth/bin/gswauth-delete-account: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2010-2011 OpenStack, LLC. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | # implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import gettext 18 | import socket 19 | 20 | from optparse import OptionParser 21 | from sys import argv, exit 22 | 23 | from swift.common.bufferedhttp import http_connect_raw as http_connect 24 | from swift.common.utils import urlparse 25 | 26 | 27 | if __name__ == '__main__': 28 | gettext.install('gswauth', unicode=1) 29 | parser = OptionParser(usage='Usage: %prog [options] ') 30 | parser.add_option('-A', '--admin-url', dest='admin_url', 31 | default='http://127.0.0.1:8080/auth/', help='The URL to the auth ' 32 | 'subsystem (default: http://127.0.0.1:8080/auth/') 33 | parser.add_option('-U', '--admin-user', dest='admin_user', 34 | default='.super_admin', 35 | help='The user with admin rights to delete accounts ' 36 | '(default: .super_admin).') 37 | parser.add_option('-K', '--admin-key', dest='admin_key', 38 | help='The key for the user with admin rights to delete accounts ' 39 | 'is required.') 40 | args = argv[1:] 41 | if not args: 42 | args.append('-h') 43 | (options, args) = parser.parse_args(args) 44 | if len(args) != 1: 45 | parser.parse_args(['-h']) 46 | if options.admin_key is None: 47 | parser.parse_args(['-h']) 48 | account = args[0] 49 | parsed = urlparse(options.admin_url) 50 | if parsed.scheme not in ('http', 'https'): 51 | raise Exception('Cannot handle protocol scheme %s for url %s' % 52 | (parsed.scheme, repr(options.admin_url))) 53 | parsed_path = parsed.path 54 | if not parsed_path: 55 | parsed_path = '/' 56 | elif parsed_path[-1] != '/': 57 | parsed_path += '/' 58 | path = '%sv2/%s' % (parsed_path, account) 59 | headers = {'X-Auth-Admin-User': options.admin_user, 60 | 'X-Auth-Admin-Key': options.admin_key} 61 | try: 62 | conn = http_connect(parsed.hostname, parsed.port, 'DELETE', path, headers, 63 | ssl=(parsed.scheme == 'https')) 64 | resp = conn.getresponse() 65 | except socket.gaierror, err: 66 | exit('Account deletion failed: %s. ' \ 67 | 'Check that the admin_url is valid' % err) 68 | except socket.error, msg: 69 | exit('Account deletion failed: %s. ' \ 70 | 'Check that the admin_url is valid' % msg) 71 | 72 | if resp.status // 100 != 2: 73 | if resp.status == 401: 74 | exit('Delete account failed: %s %s: Invalid user/key provided' % 75 | (resp.status, resp.reason)) 76 | elif resp.status == 403: 77 | exit('Delete account failed: %s %s: Insufficient privileges' % 78 | (resp.status, resp.reason)) 79 | elif resp.status == 404: 80 | exit('Delete account failed: %s %s: Account %s does not exist' % 81 | (resp.status, resp.reason, account)) 82 | elif resp.status == 409: 83 | exit('Delete account failed: %s %s: Account %s contains active users. ' 84 | 'Delete all users first.' % (resp.status, resp.reason, account)) 85 | else: 86 | exit('Delete account failed: %s %s' % (resp.status, resp.reason)) 87 | -------------------------------------------------------------------------------- /gluster/swift/common/middleware/gswauth/bin/gswauth-delete-user: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2010-2011 OpenStack, LLC. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | # implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import gettext 18 | import socket 19 | 20 | from optparse import OptionParser 21 | from sys import argv, exit 22 | 23 | from swift.common.bufferedhttp import http_connect_raw as http_connect 24 | from swift.common.utils import urlparse 25 | 26 | 27 | if __name__ == '__main__': 28 | gettext.install('gswauth', unicode=1) 29 | parser = OptionParser(usage='Usage: %prog [options] ') 30 | parser.add_option('-A', '--admin-url', dest='admin_url', 31 | default='http://127.0.0.1:8080/auth/', help='The URL to the auth ' 32 | 'subsystem (default: http://127.0.0.1:8080/auth/') 33 | parser.add_option('-U', '--admin-user', dest='admin_user', 34 | default='.super_admin', 35 | help='The user with admin rights to delete users ' 36 | '(default: .super_admin).') 37 | parser.add_option('-K', '--admin-key', dest='admin_key', 38 | help='The key for the user with admin rights to delete ' 39 | 'users is required.') 40 | args = argv[1:] 41 | if not args: 42 | args.append('-h') 43 | (options, args) = parser.parse_args(args) 44 | if len(args) != 2: 45 | parser.parse_args(['-h']) 46 | if options.admin_key is None: 47 | parser.parse_args(['-h']) 48 | account, user = args 49 | parsed = urlparse(options.admin_url) 50 | if parsed.scheme not in ('http', 'https'): 51 | raise Exception('Cannot handle protocol scheme %s for url %s' % 52 | (parsed.scheme, repr(options.admin_url))) 53 | parsed_path = parsed.path 54 | if not parsed_path: 55 | parsed_path = '/' 56 | elif parsed_path[-1] != '/': 57 | parsed_path += '/' 58 | path = '%sv2/%s/%s' % (parsed_path, account, user) 59 | headers = {'X-Auth-Admin-User': options.admin_user, 60 | 'X-Auth-Admin-Key': options.admin_key} 61 | try: 62 | conn = http_connect(parsed.hostname, parsed.port, 'DELETE', path, headers, 63 | ssl=(parsed.scheme == 'https')) 64 | resp = conn.getresponse() 65 | except socket.gaierror, err: 66 | exit('User deletion failed: %s. ' \ 67 | 'Check that the admin_url is valid' % err) 68 | except socket.error, msg: 69 | exit('User deletion failed: %s. ' \ 70 | 'Check that the admin_url is valid' % msg) 71 | 72 | if resp.status // 100 != 2: 73 | if resp.status == 401: 74 | exit('Delete user failed: %s %s: Invalid user/key provided' % 75 | (resp.status, resp.reason)) 76 | elif resp.status == 403: 77 | exit('Delete user failed: %s %s: Insufficient privileges' % 78 | (resp.status, resp.reason)) 79 | elif resp.status == 404: 80 | exit('Delete user failed: %s %s: User %s does not exist' % 81 | (resp.status, resp.reason, user)) 82 | else: 83 | exit('Delete user failed: %s %s' % (resp.status, resp.reason)) 84 | -------------------------------------------------------------------------------- /gluster/swift/common/middleware/gswauth/bin/gswauth-list: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2010-2011 OpenStack, LLC. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | # implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | try: 18 | import simplejson as json 19 | except ImportError: 20 | import json 21 | import gettext 22 | import socket 23 | import types 24 | 25 | from optparse import OptionParser 26 | from sys import argv, exit 27 | 28 | from swift.common.bufferedhttp import http_connect_raw as http_connect 29 | from swift.common.utils import urlparse 30 | 31 | from prettytable import PrettyTable 32 | 33 | if __name__ == '__main__': 34 | gettext.install('gswauth', unicode=1) 35 | parser = OptionParser(usage=''' 36 | Usage: %prog [options] [account] [user] 37 | 38 | If [account] and [user] are omitted, a list of accounts will be output. 39 | 40 | If [account] is included but not [user], a list of users within the account 41 | will be output. 42 | 43 | If [account] and [user] are included, a list of groups the user belongs to 44 | will be ouptput. 45 | 46 | If the [user] is '.groups', the active groups for the account will be listed. 47 | '''.strip()) 48 | parser.add_option('-p', '--plain-text', dest='plain_text', 49 | action='store_true', default=False, help='Changes the output from ' 50 | 'JSON to plain text. This will cause an account to list only the ' 51 | 'users and a user to list only the groups.') 52 | parser.add_option('-j', '--json', dest='json_format', 53 | action='store_true', default=False, help='Output in JSON format. ' 54 | 'This will print all information about given account or user, ' 55 | 'including stored password.') 56 | parser.add_option('-A', '--admin-url', dest='admin_url', 57 | default='http://127.0.0.1:8080/auth/', help='The URL to the auth ' 58 | 'subsystem (default: http://127.0.0.1:8080/auth/') 59 | parser.add_option('-U', '--admin-user', dest='admin_user', 60 | default='.super_admin', help='The user with admin rights ' 61 | '(default: .super_admin).') 62 | parser.add_option('-K', '--admin-key', dest='admin_key', 63 | help='The key for the user with admin rights is required.') 64 | args = argv[1:] 65 | if not args: 66 | args.append('-h') 67 | (options, args) = parser.parse_args(args) 68 | if len(args) > 2: 69 | parser.parse_args(['-h']) 70 | if options.admin_key is None: 71 | parser.parse_args(['-h']) 72 | parsed = urlparse(options.admin_url) 73 | if parsed.scheme not in ('http', 'https'): 74 | raise Exception('Cannot handle protocol scheme %s for url %s' % 75 | (parsed.scheme, repr(options.admin_url))) 76 | parsed_path = parsed.path 77 | if not parsed_path: 78 | parsed_path = '/' 79 | elif parsed_path[-1] != '/': 80 | parsed_path += '/' 81 | path = '%sv2/%s' % (parsed_path, '/'.join(args)) 82 | headers = {'X-Auth-Admin-User': options.admin_user, 83 | 'X-Auth-Admin-Key': options.admin_key} 84 | try: 85 | conn = http_connect(parsed.hostname, parsed.port, 'GET', path, headers, 86 | ssl=(parsed.scheme == 'https')) 87 | resp = conn.getresponse() 88 | except socket.gaierror, err: 89 | exit('List failed: %s. ' \ 90 | 'Check that the admin_url is valid' % err) 91 | except socket.error, msg: 92 | exit('List failed: %s. ' \ 93 | 'Check that the admin_url is valid' % msg) 94 | 95 | body = resp.read() 96 | if resp.status // 100 != 2: 97 | if resp.status == 401: 98 | exit('List failed: %s %s: Invalid user/key provided' % 99 | (resp.status, resp.reason)) 100 | elif resp.status == 403: 101 | exit('List failed: %s %s: Insufficient privileges' % 102 | (resp.status, resp.reason)) 103 | else: 104 | exit('List failed: %s %s' % (resp.status, resp.reason)) 105 | if options.plain_text: 106 | info = json.loads(body) 107 | for group in info[['accounts', 'users', 'groups'][len(args)]]: 108 | print group['name'] 109 | elif options.json_format: 110 | print body 111 | else: 112 | info = json.loads(body) 113 | h = ['accounts', 'users', 'groups'][len(args)] 114 | table = PrettyTable([h.title()]) 115 | for group in info[h]: 116 | table.add_row([group['name']]) 117 | print table 118 | -------------------------------------------------------------------------------- /gluster/swift/common/middleware/gswauth/bin/gswauth-prep: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2010-2011 OpenStack, LLC. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | # implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import gettext 18 | import socket 19 | 20 | from optparse import OptionParser 21 | from sys import argv, exit 22 | 23 | from swift.common.bufferedhttp import http_connect_raw as http_connect 24 | from swift.common.utils import urlparse 25 | 26 | 27 | if __name__ == '__main__': 28 | gettext.install('gswauth', unicode=1) 29 | parser = OptionParser(usage='Usage: %prog [options]') 30 | parser.add_option('-A', '--admin-url', dest='admin_url', 31 | default='http://127.0.0.1:8080/auth/', help='The URL to the auth ' 32 | 'subsystem (default: http://127.0.0.1:8080/auth/') 33 | parser.add_option('-U', '--admin-user', dest='admin_user', 34 | default='.super_admin', help='The user with admin rights ' 35 | '(default: .super_admin).') 36 | parser.add_option('-K', '--admin-key', dest='admin_key', 37 | help='The key for the user with admin rights is required.') 38 | args = argv[1:] 39 | if not args: 40 | args.append('-h') 41 | (options, args) = parser.parse_args(args) 42 | if args: 43 | parser.parse_args(['-h']) 44 | if options.admin_key is None: 45 | parser.parse_args(['-h']) 46 | parsed = urlparse(options.admin_url) 47 | if parsed.scheme not in ('http', 'https'): 48 | raise Exception('Cannot handle protocol scheme %s for url %s' % 49 | (parsed.scheme, repr(options.admin_url))) 50 | parsed_path = parsed.path 51 | if not parsed_path: 52 | parsed_path = '/' 53 | elif parsed_path[-1] != '/': 54 | parsed_path += '/' 55 | path = '%sv2/.prep' % parsed_path 56 | headers = {'X-Auth-Admin-User': options.admin_user, 57 | 'X-Auth-Admin-Key': options.admin_key} 58 | try: 59 | conn = http_connect(parsed.hostname, parsed.port, 'POST', path, headers, 60 | ssl=(parsed.scheme == 'https')) 61 | resp = conn.getresponse() 62 | except socket.gaierror, err: 63 | exit('gswauth preparation failed: %s. ' \ 64 | 'Check that the admin_url is valid' % err) 65 | except socket.error, msg: 66 | exit('gswauth preparation failed: %s. ' \ 67 | 'Check that the admin_url is valid' % msg) 68 | 69 | if resp.status // 100 != 2: 70 | if resp.status == 401: 71 | exit('gswauth preparation failed: %s %s: Invalid user/key provided' % 72 | (resp.status, resp.reason)) 73 | else: 74 | exit('gswauth preparation failed: %s %s' % 75 | (resp.status, resp.reason)) 76 | -------------------------------------------------------------------------------- /gluster/swift/common/middleware/gswauth/bin/gswauth-set-account-service: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2010-2011 OpenStack, LLC. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | # implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | try: 18 | import simplejson as json 19 | except ImportError: 20 | import json 21 | import gettext 22 | import socket 23 | from optparse import OptionParser 24 | from sys import argv, exit 25 | 26 | from swift.common.bufferedhttp import http_connect_raw as http_connect 27 | from swift.common.utils import urlparse 28 | 29 | 30 | if __name__ == '__main__': 31 | gettext.install('gswauth', unicode=1) 32 | parser = OptionParser(usage=''' 33 | Usage: %prog [options] 34 | 35 | Sets a service URL for an account. Can only be set by a reseller admin. 36 | 37 | Example: %prog -K gswauthkey test storage local http://127.0.0.1:8080/v1/AUTH_018c3946-23f8-4efb-a8fb-b67aae8e4162 38 | '''.strip()) 39 | parser.add_option('-A', '--admin-url', dest='admin_url', 40 | default='http://127.0.0.1:8080/auth/', help='The URL to the auth ' 41 | 'subsystem (default: http://127.0.0.1:8080/auth/)') 42 | parser.add_option('-U', '--admin-user', dest='admin_user', 43 | default='.super_admin', help='The user with admin rights ' 44 | '(default: .super_admin).') 45 | parser.add_option('-K', '--admin-key', dest='admin_key', 46 | help='The key for the user with admin rights is required.') 47 | args = argv[1:] 48 | if not args: 49 | args.append('-h') 50 | (options, args) = parser.parse_args(args) 51 | if len(args) != 4: 52 | parser.parse_args(['-h']) 53 | if options.admin_key is None: 54 | parser.parse_args(['-h']) 55 | account, service, name, url = args 56 | parsed = urlparse(options.admin_url) 57 | if parsed.scheme not in ('http', 'https'): 58 | raise Exception('Cannot handle protocol scheme %s for url %s' % 59 | (parsed.scheme, repr(options.admin_url))) 60 | parsed_path = parsed.path 61 | if not parsed_path: 62 | parsed_path = '/' 63 | elif parsed_path[-1] != '/': 64 | parsed_path += '/' 65 | path = '%sv2/%s/.services' % (parsed_path, account) 66 | body = json.dumps({service: {name: url}}) 67 | headers = {'Content-Length': str(len(body)), 68 | 'X-Auth-Admin-User': options.admin_user, 69 | 'X-Auth-Admin-Key': options.admin_key} 70 | try: 71 | conn = http_connect(parsed.hostname, parsed.port, 'POST', path, headers, 72 | ssl=(parsed.scheme == 'https')) 73 | conn.send(body) 74 | resp = conn.getresponse() 75 | except socket.gaierror, err: 76 | exit('Service set failed: %s. ' \ 77 | 'Check that the admin_url is valid' % err) 78 | except socket.error, msg: 79 | exit('Service set failed: %s. ' \ 80 | 'Check that the admin_url is valid' % msg) 81 | if resp.status // 100 != 2: 82 | if resp.status == 401: 83 | exit('Service set failed: %s %s: Invalid user/key provided' % 84 | (resp.status, resp.reason)) 85 | elif resp.status == 403: 86 | exit('Service set failed: %s %s: Insufficient privileges' % 87 | (resp.status, resp.reason)) 88 | else: 89 | exit('Service set failed: %s %s' % (resp.status, resp.reason)) 90 | -------------------------------------------------------------------------------- /gluster/swift/common/middleware/gswauth/doc/source/_static/.empty: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gluster/gluster-swift/488744a005fb399af8d094ad7a62c1917410398c/gluster/swift/common/middleware/gswauth/doc/source/_static/.empty -------------------------------------------------------------------------------- /gluster/swift/common/middleware/gswauth/doc/source/_templates/.empty: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gluster/gluster-swift/488744a005fb399af8d094ad7a62c1917410398c/gluster/swift/common/middleware/gswauth/doc/source/_templates/.empty -------------------------------------------------------------------------------- /gluster/swift/common/middleware/gswauth/doc/source/authtypes.rst: -------------------------------------------------------------------------------- 1 | .. _swauth_authtypes_module: 2 | 3 | swauth.authtypes 4 | ================= 5 | 6 | .. automodule:: swauth.authtypes 7 | :members: 8 | :undoc-members: 9 | :show-inheritance: 10 | :noindex: 11 | -------------------------------------------------------------------------------- /gluster/swift/common/middleware/gswauth/doc/source/middleware.rst: -------------------------------------------------------------------------------- 1 | .. _swauth_middleware_module: 2 | 3 | swauth.middleware 4 | ================= 5 | 6 | .. automodule:: swauth.middleware 7 | :members: 8 | :undoc-members: 9 | :show-inheritance: 10 | -------------------------------------------------------------------------------- /gluster/swift/common/middleware/gswauth/doc/source/swauth.rst: -------------------------------------------------------------------------------- 1 | .. _swauth_module: 2 | 3 | swauth 4 | ====== 5 | 6 | .. automodule:: swauth 7 | :members: 8 | :undoc-members: 9 | :show-inheritance: 10 | -------------------------------------------------------------------------------- /gluster/swift/common/middleware/gswauth/etc/proxy-server.conf-sample: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | # Standard from Swift 3 | 4 | [pipeline:main] 5 | # Standard from Swift, this is just an example of where to put swauth 6 | pipeline = catch_errors healthcheck cache ratelimit swauth proxy-server 7 | 8 | [app:proxy-server] 9 | # Standard from Swift, main point to note is the inclusion of 10 | # allow_account_management = true (only for the proxy servers where you want to 11 | # be able to create/delete accounts). 12 | use = egg:swift#proxy 13 | allow_account_management = true 14 | 15 | [filter:swauth] 16 | use = egg:swauth#swauth 17 | # You can override the default log routing for this filter here: 18 | # set log_name = swauth 19 | # set log_facility = LOG_LOCAL0 20 | # set log_level = INFO 21 | # set log_headers = False 22 | # The reseller prefix will verify a token begins with this prefix before even 23 | # attempting to validate it. Also, with authorization, only Swift storage 24 | # accounts with this prefix will be authorized by this middleware. Useful if 25 | # multiple auth systems are in use for one Swift cluster. 26 | # reseller_prefix = AUTH 27 | # If you wish to use a Swauth service on a remote cluster with this cluster: 28 | # swauth_remote = http://remotehost:port/auth 29 | # swauth_remote_timeout = 10 30 | # When using swauth_remote, the rest of these settings have no effect. 31 | # 32 | # The auth prefix will cause requests beginning with this prefix to be routed 33 | # to the auth subsystem, for granting tokens, creating accounts, users, etc. 34 | # auth_prefix = /auth/ 35 | # Cluster strings are of the format name#url where name is a short name for the 36 | # Swift cluster and url is the url to the proxy server(s) for the cluster. 37 | # default_swift_cluster = local#http://127.0.0.1:8080/v1 38 | # You may also use the format name#url#url where the first url is the one 39 | # given to users to access their account (public url) and the second is the one 40 | # used by swauth itself to create and delete accounts (private url). This is 41 | # useful when a load balancer url should be used by users, but swauth itself is 42 | # behind the load balancer. Example: 43 | # default_swift_cluster = local#https://public.com:8080/v1#http://private.com:8080/v1 44 | # Number of seconds a newly issued token should be valid for, by default. 45 | # token_life = 86400 46 | # Maximum number of seconds a newly issued token can be valid for. 47 | # max_token_life = 48 | # Specifies how the user key is stored. The default is 'plaintext', leaving the 49 | # key unsecured but available for key-signing features if such are ever added. 50 | # An alternative is 'sha1' which stores only a one-way hash of the key leaving 51 | # it secure but unavailable for key-signing. 52 | # auth_type = plaintext 53 | # Used if the auth_type is sha1 or another method that can make use of a salt. 54 | # auth_type_salt = swauthsalt 55 | # This allows middleware higher in the WSGI pipeline to override auth 56 | # processing, useful for middleware such as tempurl and formpost. If you know 57 | # you're not going to use such middleware and you want a bit of extra security, 58 | # you can set this to false. 59 | # allow_overrides = true 60 | # Highly recommended to change this. If you comment this out, the Swauth 61 | # administration features will be disabled for this proxy. 62 | super_admin_key = swauthkey 63 | 64 | [filter:ratelimit] 65 | # Standard from Swift 66 | use = egg:swift#ratelimit 67 | 68 | [filter:cache] 69 | # Standard from Swift 70 | use = egg:swift#memcache 71 | 72 | [filter:healthcheck] 73 | # Standard from Swift 74 | use = egg:swift#healthcheck 75 | 76 | [filter:catch_errors] 77 | # Standard from Swift 78 | use = egg:swift#catch_errors 79 | -------------------------------------------------------------------------------- /gluster/swift/common/middleware/gswauth/locale/swauth.pot: -------------------------------------------------------------------------------- 1 | # Translations template for swauth. 2 | # Copyright (C) 2011 ORGANIZATION 3 | # This file is distributed under the same license as the swauth project. 4 | # FIRST AUTHOR , 2011. 5 | # 6 | #, fuzzy 7 | msgid "" 8 | msgstr "" 9 | "Project-Id-Version: swauth 1.0.1.dev\n" 10 | "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" 11 | "POT-Creation-Date: 2011-05-26 10:35+0000\n" 12 | "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" 13 | "Last-Translator: FULL NAME \n" 14 | "Language-Team: LANGUAGE \n" 15 | "MIME-Version: 1.0\n" 16 | "Content-Type: text/plain; charset=utf-8\n" 17 | "Content-Transfer-Encoding: 8bit\n" 18 | "Generated-By: Babel 0.9.4\n" 19 | 20 | #: swauth/middleware.py:94 21 | msgid "No super_admin_key set in conf file! Exiting." 22 | msgstr "" 23 | 24 | #: swauth/middleware.py:637 25 | #, python-format 26 | msgid "" 27 | "ERROR: Exception while trying to communicate with " 28 | "%(scheme)s://%(host)s:%(port)s/%(path)s" 29 | msgstr "" 30 | 31 | -------------------------------------------------------------------------------- /gluster/swift/common/middleware/gswauth/setup.cfg: -------------------------------------------------------------------------------- 1 | [build_sphinx] 2 | all_files = 1 3 | build-dir = doc/build 4 | source-dir = doc/source 5 | 6 | [egg_info] 7 | tag_build = 8 | tag_date = 0 9 | tag_svn_revision = 0 10 | 11 | [compile_catalog] 12 | directory = locale 13 | domain = swauth 14 | 15 | [update_catalog] 16 | domain = swauth 17 | output_dir = locale 18 | input_file = locale/swauth.pot 19 | 20 | [extract_messages] 21 | keywords = _ l_ lazy_gettext 22 | mapping_file = babel.cfg 23 | output_file = locale/swauth.pot 24 | -------------------------------------------------------------------------------- /gluster/swift/common/middleware/gswauth/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # Copyright (c) 2010-2011 OpenStack, LLC. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | # implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from setuptools import setup, find_packages 18 | from setuptools.command.sdist import sdist 19 | import os 20 | import subprocess 21 | try: 22 | from babel.messages import frontend 23 | except ImportError: 24 | frontend = None 25 | 26 | from swauth import __version__ as version 27 | 28 | 29 | class local_sdist(sdist): 30 | """Customized sdist hook - builds the ChangeLog file from VC first""" 31 | 32 | def run(self): 33 | if os.path.isdir('.bzr'): 34 | # We're in a bzr branch 35 | 36 | log_cmd = subprocess.Popen(["bzr", "log", "--gnu"], 37 | stdout=subprocess.PIPE) 38 | changelog = log_cmd.communicate()[0] 39 | with open("ChangeLog", "w") as changelog_file: 40 | changelog_file.write(changelog) 41 | sdist.run(self) 42 | 43 | 44 | name = 'swauth' 45 | 46 | 47 | cmdclass = {'sdist': local_sdist} 48 | 49 | 50 | if frontend: 51 | cmdclass.update({ 52 | 'compile_catalog': frontend.compile_catalog, 53 | 'extract_messages': frontend.extract_messages, 54 | 'init_catalog': frontend.init_catalog, 55 | 'update_catalog': frontend.update_catalog, 56 | }) 57 | 58 | 59 | setup( 60 | name=name, 61 | version=version, 62 | description='Swauth', 63 | license='Apache License (2.0)', 64 | author='OpenStack, LLC.', 65 | author_email='swauth@brim.net', 66 | url='https://github.com/gholt/swauth', 67 | packages=find_packages(exclude=['test_swauth', 'bin']), 68 | test_suite='nose.collector', 69 | cmdclass=cmdclass, 70 | classifiers=[ 71 | 'Development Status :: 4 - Beta', 72 | 'License :: OSI Approved :: Apache Software License', 73 | 'Operating System :: POSIX :: Linux', 74 | 'Programming Language :: Python :: 2.6', 75 | 'Environment :: No Input/Output (Daemon)', 76 | ], 77 | install_requires=[], # removed for better compat 78 | scripts=[ 79 | 'bin/swauth-add-account', 'bin/swauth-add-user', 80 | 'bin/swauth-cleanup-tokens', 'bin/swauth-delete-account', 81 | 'bin/swauth-delete-user', 'bin/swauth-list', 'bin/swauth-prep', 82 | 'bin/swauth-set-account-service', 83 | ], 84 | entry_points={ 85 | 'paste.filter_factory': [ 86 | 'swauth=swauth.middleware:filter_factory', 87 | ], 88 | }, 89 | ) 90 | -------------------------------------------------------------------------------- /gluster/swift/common/middleware/gswauth/swauth/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2010-2013 OpenStack, LLC. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import gettext 17 | 18 | 19 | #: Version information (major, minor, revision[, 'dev']). 20 | version_info = (1, 0, 9, 'dev') 21 | #: Version string 'major.minor.revision'. 22 | version = __version__ = ".".join(map(str, version_info)) 23 | gettext.install('swauth') 24 | -------------------------------------------------------------------------------- /gluster/swift/common/middleware/gswauth/swauth/authtypes.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 10 | # implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | # Pablo Llopis 2011 15 | 16 | 17 | """ 18 | This module hosts available auth types for encoding and matching user keys. 19 | For adding a new auth type, simply write a class that satisfies the following 20 | conditions: 21 | 22 | - For the class name, capitalize first letter only. This makes sure the user 23 | can specify an all-lowercase config option such as "plaintext" or "sha1". 24 | Swauth takes care of capitalizing the first letter before instantiating it. 25 | - Write an encode(key) method that will take a single argument, the user's key, 26 | and returns the encoded string. For plaintext, this would be 27 | "plaintext:" 28 | - Write a match(key, creds) method that will take two arguments: the user's 29 | key, and the user's retrieved credentials. Return a boolean value that 30 | indicates whether the match is True or False. 31 | 32 | Note that, since some of the encodings will be hashes, swauth supports the 33 | notion of salts. Thus, self.salt will be set to either a user-specified salt 34 | value or to a default value. 35 | """ 36 | 37 | import hashlib 38 | import os 39 | 40 | 41 | #: Maximum length any valid token should ever be. 42 | MAX_TOKEN_LENGTH = 5000 43 | 44 | 45 | class Plaintext(object): 46 | """ 47 | Provides a particular auth type for encoding format for encoding and 48 | matching user keys. 49 | 50 | This class must be all lowercase except for the first character, which 51 | must be capitalized. encode and match methods must be provided and are 52 | the only ones that will be used by swauth. 53 | """ 54 | def encode(self, key): 55 | """ 56 | Encodes a user key into a particular format. The result of this method 57 | will be used by swauth for storing user credentials. 58 | 59 | :param key: User's secret key 60 | :returns: A string representing user credentials 61 | """ 62 | return "plaintext:%s" % key 63 | 64 | def match(self, key, creds): 65 | """ 66 | Checks whether the user-provided key matches the user's credentials 67 | 68 | :param key: User-supplied key 69 | :param creds: User's stored credentials 70 | :returns: True if the supplied key is valid, False otherwise 71 | """ 72 | return self.encode(key) == creds 73 | 74 | 75 | class Sha1(object): 76 | """ 77 | Provides a particular auth type for encoding format for encoding and 78 | matching user keys. 79 | 80 | This class must be all lowercase except for the first character, which 81 | must be capitalized. encode and match methods must be provided and are 82 | the only ones that will be used by swauth. 83 | """ 84 | 85 | def encode_w_salt(self, salt, key): 86 | """ 87 | Encodes a user key with salt into a particular format. The result of 88 | this method will be used internally. 89 | 90 | :param salt: Salt for hashing 91 | :param key: User's secret key 92 | :returns: A string representing user credentials 93 | """ 94 | enc_key = '%s%s' % (salt, key) 95 | enc_val = hashlib.sha1(enc_key).hexdigest() 96 | return "sha1:%s$%s" % (salt, enc_val) 97 | 98 | def encode(self, key): 99 | """ 100 | Encodes a user key into a particular format. The result of this method 101 | will be used by swauth for storing user credentials. 102 | 103 | :param key: User's secret key 104 | :returns: A string representing user credentials 105 | """ 106 | salt = self.salt or os.urandom(32).encode('base64').rstrip() 107 | return self.encode_w_salt(salt, key) 108 | 109 | def match(self, key, creds): 110 | """ 111 | Checks whether the user-provided key matches the user's credentials 112 | 113 | :param key: User-supplied key 114 | :param creds: User's stored credentials 115 | :returns: True if the supplied key is valid, False otherwise 116 | """ 117 | 118 | type, rest = creds.split(':') 119 | salt, enc = rest.split('$') 120 | 121 | return self.encode_w_salt(salt, key) == creds 122 | 123 | 124 | class Sha512(object): 125 | """ 126 | Provides a particular auth type for encoding format for encoding and 127 | matching user keys. 128 | 129 | This class must be all lowercase except for the first character, which 130 | must be capitalized. encode and match methods must be provided and are 131 | the only ones that will be used by swauth. 132 | """ 133 | 134 | def encode_w_salt(self, salt, key): 135 | """ 136 | Encodes a user key with salt into a particular format. The result of 137 | this method will be used internal. 138 | 139 | :param salt: Salt for hashing 140 | :param key: User's secret key 141 | :returns: A string representing user credentials 142 | """ 143 | enc_key = '%s%s' % (salt, key) 144 | enc_val = hashlib.sha512(enc_key).hexdigest() 145 | return "sha512:%s$%s" % (salt, enc_val) 146 | 147 | def encode(self, key): 148 | """ 149 | Encodes a user key into a particular format. The result of this method 150 | will be used by swauth for storing user credentials. 151 | 152 | If salt is not manually set in conf file, a random salt will be 153 | generated and used. 154 | 155 | :param key: User's secret key 156 | :returns: A string representing user credentials 157 | """ 158 | salt = self.salt or os.urandom(32).encode('base64').rstrip() 159 | return self.encode_w_salt(salt, key) 160 | 161 | def match(self, key, creds): 162 | """Checks whether the user-provided key matches the user's credentials 163 | 164 | :param key: User-supplied key 165 | :param creds: User's stored credentials 166 | :returns: True if the supplied key is valid, False otherwise 167 | """ 168 | 169 | type, rest = creds.split(':') 170 | salt, enc = rest.split('$') 171 | 172 | return self.encode_w_salt(salt, key) == creds 173 | -------------------------------------------------------------------------------- /gluster/swift/common/middleware/gswauth/swauth/swift_version.py: -------------------------------------------------------------------------------- 1 | import swift 2 | 3 | 4 | MAJOR = None 5 | MINOR = None 6 | REVISION = None 7 | FINAL = None 8 | 9 | 10 | def parse(value): 11 | parts = value.split('.') 12 | if parts[-1].endswith('-dev'): 13 | final = False 14 | parts[-1] = parts[-1][:-4] 15 | else: 16 | final = True 17 | major = int(parts.pop(0)) 18 | minor = int(parts.pop(0)) 19 | if parts: 20 | revision = int(parts.pop(0)) 21 | else: 22 | revision = 0 23 | return major, minor, revision, final 24 | 25 | 26 | def newer_than(value): 27 | global MAJOR, MINOR, REVISION, FINAL 28 | major, minor, revision, final = parse(value) 29 | if MAJOR is None: 30 | MAJOR, MINOR, REVISION, FINAL = parse(swift.__version__) 31 | if MAJOR < major: 32 | return False 33 | elif MAJOR == major: 34 | if MINOR < minor: 35 | return False 36 | elif MINOR == minor: 37 | if REVISION < revision: 38 | return False 39 | elif REVISION == revision: 40 | if not FINAL or final: 41 | return False 42 | return True 43 | 44 | 45 | def run_tests(): 46 | global MAJOR, MINOR, REVISION, FINAL 47 | MAJOR, MINOR, REVISION, FINAL = parse('1.3') 48 | assert(newer_than('1.2')) 49 | assert(newer_than('1.2.9')) 50 | assert(newer_than('1.3-dev')) 51 | assert(newer_than('1.3.0-dev')) 52 | assert(not newer_than('1.3')) 53 | assert(not newer_than('1.3.0')) 54 | assert(not newer_than('1.3.1-dev')) 55 | assert(not newer_than('1.3.1')) 56 | assert(not newer_than('1.4')) 57 | assert(not newer_than('2.0')) 58 | MAJOR, MINOR, REVISION, FINAL = parse('1.7.7-dev') 59 | assert(newer_than('1.6')) 60 | assert(newer_than('1.7')) 61 | assert(newer_than('1.7.6-dev')) 62 | assert(newer_than('1.7.6')) 63 | assert(not newer_than('1.7.7')) 64 | assert(not newer_than('1.7.8-dev')) 65 | assert(not newer_than('1.7.8')) 66 | assert(not newer_than('1.8.0')) 67 | assert(not newer_than('2.0')) 68 | 69 | 70 | if __name__ == '__main__': 71 | run_tests() 72 | -------------------------------------------------------------------------------- /gluster/swift/common/ring.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2013 Red Hat, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import os 17 | import errno 18 | from ConfigParser import ConfigParser 19 | from swift.common.ring import ring 20 | from swift.common.utils import search_tree 21 | from gluster.swift.common.Glusterfs import SWIFT_DIR 22 | 23 | reseller_prefix = "AUTH_" 24 | conf_files = search_tree(SWIFT_DIR, "proxy-server*", 'conf') 25 | if conf_files: 26 | conf_file = conf_files[0] 27 | 28 | _conf = ConfigParser() 29 | if conf_files and _conf.read(conf_file): 30 | if _conf.defaults().get("reseller_prefix", None): 31 | reseller_prefix = _conf.defaults().get("reseller_prefix") 32 | else: 33 | for key, value in _conf._sections.items(): 34 | if value.get("reseller_prefix", None): 35 | reseller_prefix = value["reseller_prefix"] 36 | break 37 | 38 | if not reseller_prefix.endswith('_'): 39 | reseller_prefix = reseller_prefix + '_' 40 | 41 | 42 | class Ring(ring.Ring): 43 | 44 | def __init__(self, serialized_path, reload_time=15, ring_name=None): 45 | self.false_node = {'zone': 1, 'weight': 100.0, 'ip': '127.0.0.1', 46 | 'id': 0, 'meta': '', 'device': 'volume_not_in_ring', 47 | 'port': 6012} 48 | self.account_list = [] 49 | 50 | if ring_name: 51 | _serialized_path = os.path.join(serialized_path, 52 | ring_name + '.ring.gz') 53 | else: 54 | _serialized_path = os.path.join(serialized_path) 55 | 56 | if not os.path.exists(_serialized_path): 57 | raise OSError(errno.ENOENT, 'No such file or directory', 58 | '%s ring file does not exists, aborting ' 59 | 'proxy-server start.' % _serialized_path) 60 | 61 | ring.Ring.__init__(self, serialized_path, reload_time, ring_name) 62 | 63 | def _get_part_nodes(self, part): 64 | seen_ids = set() 65 | 66 | try: 67 | account = self.account_list[part] 68 | except IndexError: 69 | return [self.false_node] 70 | else: 71 | nodes = [] 72 | for dev in self._devs: 73 | if dev['device'] == account: 74 | if dev['id'] not in seen_ids: 75 | seen_ids.add(dev['id']) 76 | nodes.append(dev) 77 | if not nodes: 78 | nodes = [self.false_node] 79 | return nodes 80 | 81 | def get_part_nodes(self, part): 82 | """ 83 | Get the nodes that are responsible for the partition. If one 84 | node is responsible for more than one replica of the same 85 | partition, it will only appear in the output once. 86 | 87 | :param part: partition to get nodes for 88 | :returns: list of node dicts 89 | 90 | See :func:`get_nodes` for a description of the node dicts. 91 | """ 92 | return self._get_part_nodes(part) 93 | 94 | def get_part(self, account, container=None, obj=None): 95 | """ 96 | Get the partition for an account/container/object. 97 | 98 | :param account: account name 99 | :param container: container name 100 | :param obj: object name 101 | :returns: the partition number 102 | """ 103 | if account.startswith(reseller_prefix): 104 | account = account.replace(reseller_prefix, '', 1) 105 | 106 | # Save the account name in the table 107 | # This makes part be the index of the location of the account 108 | # in the list 109 | try: 110 | part = self.account_list.index(account) 111 | except ValueError: 112 | self.account_list.append(account) 113 | part = self.account_list.index(account) 114 | 115 | return part 116 | 117 | def get_nodes(self, account, container=None, obj=None): 118 | """ 119 | Get the partition and nodes for an account/container/object. 120 | If a node is responsible for more than one replica, it will 121 | only appear in the output once. 122 | :param account: account name 123 | :param container: container name 124 | :param obj: object name 125 | :returns: a tuple of (partition, list of node dicts) 126 | 127 | Each node dict will have at least the following keys: 128 | ====== =============================================================== 129 | id unique integer identifier amongst devices 130 | weight a float of the relative weight of this device as compared to 131 | others; this indicates how many partitions the builder will try 132 | to assign to this device 133 | zone integer indicating which zone the device is in; a given 134 | partition will not be assigned to multiple devices within the 135 | same zone 136 | ip the ip address of the device 137 | port the tcp port of the device 138 | device the device's name on disk (sdb1, for example) 139 | meta general use 'extra' field; for example: the online date, the 140 | hardware description 141 | ====== =============================================================== 142 | """ 143 | part = self.get_part(account, container, obj) 144 | return part, self._get_part_nodes(part) 145 | 146 | def get_more_nodes(self, part): 147 | """ 148 | Generator to get extra nodes for a partition for hinted handoff. 149 | 150 | :param part: partition to get handoff nodes for 151 | :returns: generator of node dicts 152 | 153 | See :func:`get_nodes` for a description of the node dicts. 154 | Should never be called in the swift UFO environment, so yield nothing 155 | """ 156 | return [] 157 | -------------------------------------------------------------------------------- /gluster/swift/container/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gluster/gluster-swift/488744a005fb399af8d094ad7a62c1917410398c/gluster/swift/container/__init__.py -------------------------------------------------------------------------------- /gluster/swift/container/server.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2012-2013 Red Hat, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | """ Container Server for Gluster Swift UFO """ 17 | 18 | # Simply importing this monkey patches the constraint handling to fit our 19 | # needs 20 | import gluster.swift.common.constraints # noqa 21 | 22 | from swift.container import server 23 | from gluster.swift.common.DiskDir import DiskDir 24 | from swift.common.utils import public, timing_stats, config_true_value 25 | from swift.common.exceptions import DiskFileNoSpace 26 | from swift.common.swob import HTTPInsufficientStorage, HTTPNotFound, \ 27 | HTTPPreconditionFailed 28 | from swift.common.request_helpers import get_param, get_listing_content_type, \ 29 | split_and_validate_path 30 | from swift.common.constraints import check_mount 31 | from swift.container.server import gen_resp_headers 32 | from swift.common import constraints 33 | 34 | 35 | class ContainerController(server.ContainerController): 36 | """ 37 | Subclass of the container server's ContainerController which replaces the 38 | _get_container_broker() method so that we can use Gluster's DiskDir 39 | duck-type of the container DatabaseBroker object, and make the 40 | account_update() method a no-op (information is simply stored on disk and 41 | already updated by virtue of performaing the file system operations 42 | directly). 43 | """ 44 | 45 | def _get_container_broker(self, drive, part, account, container, **kwargs): 46 | """ 47 | Overriden to provide the GlusterFS specific broker that talks to 48 | Gluster for the information related to servicing a given request 49 | instead of talking to a database. 50 | 51 | :param drive: drive that holds the container 52 | :param part: partition the container is in 53 | :param account: account name 54 | :param container: container name 55 | :returns: DiskDir object, a duck-type of DatabaseBroker 56 | """ 57 | return DiskDir(self.root, drive, account, container, self.logger, 58 | **kwargs) 59 | 60 | def account_update(self, req, account, container, broker): 61 | """ 62 | Update the account server(s) with latest container info. 63 | 64 | For Gluster, this is just a no-op, since an account is just the 65 | directory holding all the container directories. 66 | 67 | :param req: swob.Request object 68 | :param account: account name 69 | :param container: container name 70 | :param broker: container DB broker object 71 | :returns: None. 72 | """ 73 | return None 74 | 75 | @public 76 | @timing_stats() 77 | def PUT(self, req): 78 | try: 79 | return server.ContainerController.PUT(self, req) 80 | except DiskFileNoSpace: 81 | # As container=directory in gluster-swift, we might run out of 82 | # space or exceed quota when creating containers. 83 | drive = req.split_path(1, 1, True) 84 | return HTTPInsufficientStorage(drive=drive, request=req) 85 | 86 | @public 87 | @timing_stats() 88 | def GET(self, req): 89 | """ 90 | Handle HTTP GET request. 91 | 92 | This method is exact copy of swift.container.server.GET() except 93 | that this version of it passes 'out_content_type' information to 94 | broker.list_objects_iter() 95 | """ 96 | drive, part, account, container, obj = split_and_validate_path( 97 | req, 4, 5, True) 98 | path = get_param(req, 'path') 99 | prefix = get_param(req, 'prefix') 100 | delimiter = get_param(req, 'delimiter') 101 | if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254): 102 | # delimiters can be made more flexible later 103 | return HTTPPreconditionFailed(body='Bad delimiter') 104 | marker = get_param(req, 'marker', '') 105 | end_marker = get_param(req, 'end_marker') 106 | limit = constraints.CONTAINER_LISTING_LIMIT 107 | given_limit = get_param(req, 'limit') 108 | reverse = config_true_value(get_param(req, 'reverse')) 109 | 110 | if given_limit and given_limit.isdigit(): 111 | limit = int(given_limit) 112 | if limit > constraints.CONTAINER_LISTING_LIMIT: 113 | return HTTPPreconditionFailed( 114 | request=req, 115 | body='Maximum limit is %d' 116 | % constraints.CONTAINER_LISTING_LIMIT) 117 | out_content_type = get_listing_content_type(req) 118 | if self.mount_check and not check_mount(self.root, drive): 119 | return HTTPInsufficientStorage(drive=drive, request=req) 120 | broker = self._get_container_broker(drive, part, account, container, 121 | pending_timeout=0.1, 122 | stale_reads_ok=True) 123 | info, is_deleted = broker.get_info_is_deleted() 124 | resp_headers = gen_resp_headers(info, is_deleted=is_deleted) 125 | if is_deleted: 126 | return HTTPNotFound(request=req, headers=resp_headers) 127 | container_list = broker.list_objects_iter( 128 | limit, marker, end_marker, prefix, delimiter, path, 129 | storage_policy_index=info['storage_policy_index'], 130 | out_content_type=out_content_type, reverse=reverse) 131 | return self.create_listing(req, out_content_type, info, resp_headers, 132 | broker.metadata, container_list, container) 133 | 134 | 135 | def app_factory(global_conf, **local_conf): 136 | """paste.deploy app factory for creating WSGI container server apps.""" 137 | conf = global_conf.copy() 138 | conf.update(local_conf) 139 | return ContainerController(conf) 140 | -------------------------------------------------------------------------------- /gluster/swift/obj/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gluster/gluster-swift/488744a005fb399af8d094ad7a62c1917410398c/gluster/swift/obj/__init__.py -------------------------------------------------------------------------------- /gluster/swift/obj/expirer.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2016 Red Hat 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # This import will monkey-patch Ring and other classes. 17 | # Do not remove. 18 | import gluster.swift.common.constraints # noqa 19 | 20 | import errno 21 | import os 22 | 23 | from gluster.swift.common.utils import delete_tracker_object 24 | 25 | from swift.obj.expirer import ObjectExpirer as SwiftObjectExpirer 26 | from swift.common.http import HTTP_NOT_FOUND 27 | from swift.common.internal_client import InternalClient, UnexpectedResponse 28 | from gluster.swift.common.utils import ThreadPool 29 | 30 | EXCLUDE_DIRS = ('.trashcan', '.glusterfs') 31 | 32 | 33 | class GlusterSwiftInternalClient(InternalClient): 34 | 35 | def __init__(self, conf_path, user_agent, request_tries, 36 | allow_modify_pipeline=False, devices=None): 37 | super(GlusterSwiftInternalClient, self).__init__( 38 | conf_path, user_agent, request_tries, allow_modify_pipeline) 39 | self.devices = devices 40 | 41 | def get_account_info(self, account): 42 | # Supposed to return container count and object count in gsexpiring 43 | # account. This is used by object expirer only for logging. 44 | return (0, 0) 45 | 46 | def delete_container(self, account, container, acceptable_statuses=None): 47 | container_path = os.path.join(self.devices, account, container) 48 | try: 49 | os.rmdir(container_path) 50 | except OSError as err: 51 | if err.errno != errno.ENOENT: 52 | raise 53 | 54 | def iter_containers(self, account): 55 | account_path = os.path.join(self.devices, account) 56 | for container in os.listdir(account_path): 57 | if container in EXCLUDE_DIRS: 58 | continue 59 | container_path = os.path.join(account_path, container) 60 | if os.path.isdir(container_path): 61 | yield {'name': container.encode('utf8')} 62 | 63 | def iter_objects(self, account, container): 64 | container_path = os.path.join(self.devices, account, container) 65 | # TODO: Use a slightly better implementation of os.walk() 66 | for (root, dirs, files) in os.walk(container_path): 67 | for f in files: 68 | obj_path = os.path.join(root, f) 69 | obj = obj_path[(len(container_path) + 1):] 70 | yield {'name': obj.encode('utf8')} 71 | 72 | 73 | class ObjectExpirer(SwiftObjectExpirer): 74 | 75 | def __init__(self, conf, logger=None, swift=None): 76 | 77 | conf_path = conf.get('__file__') or '/etc/swift/object-expirer.conf' 78 | self.devices = conf.get('devices', '/mnt/gluster-object') 79 | # Do not retry DELETEs on getting 404. Hence default is set to 1. 80 | request_tries = int(conf.get('request_tries') or 1) 81 | # Use our extended version of InternalClient 82 | swift = GlusterSwiftInternalClient( 83 | conf_path, 'Gluster Swift Object Expirer', request_tries, 84 | devices=self.devices) 85 | # Let the parent class initialize self.swift 86 | super(ObjectExpirer, self).__init__(conf, logger=logger, swift=swift) 87 | 88 | self.reseller_prefix = conf.get('reseller_prefix', 'AUTH').strip() 89 | if not self.reseller_prefix.endswith('_'): 90 | self.reseller_prefix = self.reseller_prefix + '_' 91 | 92 | # nthread=0 is intentional. This ensures that no green pool is 93 | # used. Call to force_run_in_thread() will ensure that the method 94 | # passed as arg is run in a real external thread using eventlet.tpool 95 | # which has a threadpool of 20 threads (default) 96 | self.threadpool = ThreadPool(nthreads=0) 97 | 98 | def pop_queue(self, container, obj): 99 | """ 100 | In Swift, this method removes tracker object entry directly from 101 | container database. In gluster-swift, this method deletes tracker 102 | object directly from filesystem. 103 | """ 104 | container_path = os.path.join(self.devices, 105 | self.expiring_objects_account, 106 | container) 107 | self.threadpool.force_run_in_thread(delete_tracker_object, 108 | container_path, obj) 109 | 110 | def delete_actual_object(self, actual_obj, timestamp): 111 | """ 112 | Swift's expirer will re-attempt expiring if the source object is not 113 | available (404 or ANY other error) up to self.reclaim_age seconds 114 | before it gives up and deletes the entry in the queue. 115 | 116 | Don't do this in gluster-swift. GlusterFS isn't eventually consistent 117 | and has no concept of hand-off nodes. If actual data object doesn't 118 | exist (404), remove tracker object from the queue (filesystem). 119 | 120 | However if DELETE fails due a reason other than 404, do not remove 121 | tracker object yet, follow Swift's behaviour of waiting till 122 | self.reclaim_age seconds. 123 | 124 | This method is just a wrapper around parent class's method. All this 125 | wrapper does is ignore 404 failures. 126 | """ 127 | try: 128 | super(ObjectExpirer, self).delete_actual_object( 129 | actual_obj, timestamp) 130 | except UnexpectedResponse as err: 131 | if err.resp.status_int != HTTP_NOT_FOUND: 132 | raise 133 | -------------------------------------------------------------------------------- /gluster/swift/proxy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gluster/gluster-swift/488744a005fb399af8d094ad7a62c1917410398c/gluster/swift/proxy/__init__.py -------------------------------------------------------------------------------- /gluster/swift/proxy/server.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2012-2013 Red Hat, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | # Simply importing this monkey patches the constraint handling to fit our 18 | # needs 19 | import gluster.swift.common.constraints # noqa 20 | 21 | from swift.proxy.server import Application, mimetypes # noqa 22 | from swift.proxy.controllers import AccountController # noqa 23 | from swift.proxy.controllers import ObjectControllerRouter # noqa 24 | from swift.proxy.controllers import ContainerController # noqa 25 | 26 | 27 | def app_factory(global_conf, **local_conf): # noqa 28 | """paste.deploy app factory for creating WSGI proxy apps.""" 29 | conf = global_conf.copy() 30 | conf.update(local_conf) 31 | return Application(conf) 32 | -------------------------------------------------------------------------------- /glusterfs-openstack-swift.spec: -------------------------------------------------------------------------------- 1 | %define _confdir %{_sysconfdir}/swift 2 | 3 | # The following values are provided by passing the following arguments 4 | # to rpmbuild. For example: 5 | # --define "_version 1.0" --define "_release 1" --define "_name g4s" 6 | # 7 | %{!?_version:%define _version __PKG_VERSION__} 8 | %{!?_name:%define _name __PKG_NAME__} 9 | %{!?_release:%define _release __PKG_RELEASE__} 10 | 11 | Summary : GlusterFS Integration with OpenStack Object Storage (Swift). 12 | Name : %{_name} 13 | Version : %{_version} 14 | Release : %{_release}%{?dist} 15 | Group : Application/File 16 | URL : http://github.com/gluster/gluster-swift 17 | Vendor : Gluster Community 18 | Source0 : %{_name}-%{_version}-%{_release}.tar.gz 19 | License : ASL 2.0 20 | BuildArch: noarch 21 | BuildRequires: python 22 | BuildRequires: python-setuptools 23 | Requires : memcached 24 | Requires : openssl 25 | Requires : python 26 | Requires : python-prettytable 27 | Requires : openstack-swift = 2.15.1 28 | Requires : openstack-swift-account = 2.15.1 29 | Requires : openstack-swift-container = 2.15.1 30 | Requires : openstack-swift-object = 2.15.1 31 | Requires : openstack-swift-proxy = 2.15.1 32 | # gluster-swift has no hard-dependency on particular version of glusterfs 33 | # so don't bump this up unless you want to force users to upgrade their 34 | # glusterfs deployment 35 | Requires : python-gluster >= 3.8.0 36 | Obsoletes: glusterfs-swift-plugin 37 | Obsoletes: glusterfs-swift 38 | Obsoletes: glusterfs-ufo 39 | Obsoletes: glusterfs-swift-container 40 | Obsoletes: glusterfs-swift-object 41 | Obsoletes: glusterfs-swift-proxy 42 | Obsoletes: glusterfs-swift-account 43 | 44 | %description 45 | Gluster-Swift integrates GlusterFS as an alternative back end for OpenStack 46 | Object Storage (Swift) leveraging the existing front end OpenStack Swift code. 47 | Gluster volumes are used to store objects in files, containers are maintained 48 | as top-level directories of volumes, where accounts are mapped one-to-one to 49 | gluster volumes. 50 | 51 | %prep 52 | %setup -q -n gluster_swift-%{_version} 53 | 54 | %build 55 | %{__python} setup.py build 56 | 57 | %install 58 | rm -rf %{buildroot} 59 | 60 | %{__python} setup.py install -O1 --skip-build --root %{buildroot} 61 | 62 | mkdir -p %{buildroot}/%{_confdir}/ 63 | cp -r etc/* %{buildroot}/%{_confdir}/ 64 | 65 | # Man Pages 66 | install -d -m 755 %{buildroot}%{_mandir}/man8 67 | for page in doc/man/*.8; do 68 | install -p -m 0644 $page %{buildroot}%{_mandir}/man8 69 | done 70 | 71 | # Remove tests 72 | %{__rm} -rf %{buildroot}/%{python_sitelib}/test 73 | 74 | # Remove files provided by python-gluster (glusterfs-api earlier) 75 | %{__rm} -rf %{buildroot}/%{python_sitelib}/gluster/__init__.p* 76 | 77 | %files 78 | %defattr(-,root,root) 79 | %{python_sitelib}/gluster 80 | %{python_sitelib}/gluster_swift-%{_version}*.egg-info 81 | %{_bindir}/gluster-swift-gen-builders 82 | %{_bindir}/gluster-swift-print-metadata 83 | %{_bindir}/gluster-swift-migrate-metadata 84 | %{_bindir}/gluster-swift-object-expirer 85 | %{_bindir}/gswauth-add-account 86 | %{_bindir}/gswauth-add-user 87 | %{_bindir}/gswauth-cleanup-tokens 88 | %{_bindir}/gswauth-delete-account 89 | %{_bindir}/gswauth-delete-user 90 | %{_bindir}/gswauth-list 91 | %{_bindir}/gswauth-prep 92 | %{_bindir}/gswauth-set-account-service 93 | %{_mandir}/man8/* 94 | 95 | %dir %{_confdir} 96 | %config(noreplace) %{_confdir}/account-server.conf-gluster 97 | %config(noreplace) %{_confdir}/container-server.conf-gluster 98 | %config(noreplace) %{_confdir}/object-server.conf-gluster 99 | %config(noreplace) %{_confdir}/swift.conf-gluster 100 | %config(noreplace) %{_confdir}/proxy-server.conf-gluster 101 | %config(noreplace) %{_confdir}/fs.conf-gluster 102 | %config(noreplace) %{_confdir}/object-expirer.conf-gluster 103 | 104 | %changelog 105 | * Wed Nov 22 2017 Venkata R Edara - 2.15.1 106 | - Rebase to Swift 2.15.1 (pike) 107 | 108 | * Wed May 10 2017 Venkata R Edara - 2.10.1 109 | - Rebase to Swift 2.10.1 (newton) 110 | 111 | * Tue Mar 15 2016 Prashanth Pai - 2.3.0-0 112 | - Rebase to swift kilo (2.3.0) 113 | 114 | * Fri May 23 2014 Thiago da Silva - 1.13.1-1 115 | - Update to Icehouse release 116 | 117 | * Mon Oct 28 2013 Luis Pabon - 1.10.1-0 118 | - Havana Release 119 | 120 | * Wed Aug 21 2013 Luis Pabon - 1.8.0-7 121 | - Update RPM spec file to support SRPMS 122 | -------------------------------------------------------------------------------- /makerpm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Simple script to create RPMs for G4S 4 | 5 | ## RPM NAME 6 | RPMNAME=glusterfs-openstack-swift 7 | 8 | cleanup() 9 | { 10 | rm -rf ${RPMBUILDDIR} > /dev/null 2>&1 11 | rm -f ${PKGCONFIG} > /dev/null 2>&1 12 | } 13 | 14 | fail() 15 | { 16 | cleanup 17 | echo $1 18 | exit $2 19 | } 20 | 21 | create_dir() 22 | { 23 | if [ ! -d "$1" ] ; then 24 | mkdir -p "$1" 25 | if [ $? -ne 0 ] ; then 26 | fail "Unable to create dir $1" $? 27 | fi 28 | fi 29 | } 30 | 31 | gittotar() 32 | { 33 | # Only archives committed changes 34 | gitarchive_dir="${RPMBUILDDIR}/gitarchive" 35 | specfile="${gitarchive_dir}/${SRCTAR_DIR}/${RPMNAME}.spec" 36 | create_dir "${gitarchive_dir}" 37 | 38 | # Export the current commited git changes to a directory 39 | git archive --format=tar --prefix=${SRCTAR_DIR}/ HEAD | (cd ${gitarchive_dir} && tar xf -) 40 | # Create a new spec file with the current package version information 41 | sed -e "s#__PKG_RELEASE__#${PKG_RELEASE}#" \ 42 | -e "s#__PKG_NAME__#${RPMNAME}#" \ 43 | -e "s#__PKG_VERSION__#${PKG_VERSION}#" \ 44 | ${specfile} > ${specfile}.new 45 | mv ${specfile}.new ${specfile} 46 | 47 | # Now create a tar file 48 | ( cd ${gitarchive_dir} && tar cf - ${SRCTAR_DIR} | gzip -c > ${SRCTAR} ) 49 | if [ $? -ne 0 -o \! -s ${SRCTAR} ] ; then 50 | fail "Unable to create git archive" $? 51 | fi 52 | } 53 | 54 | prep() 55 | { 56 | rm -rf ${RPMBUILDDIR} > /dev/null 2>&1 57 | create_dir ${RPMBUILDDIR} 58 | 59 | # Create a tar file out of the current committed changes 60 | gittotar 61 | 62 | } 63 | 64 | create_rpm() 65 | { 66 | # Create the rpm 67 | # _topdir Notifies rpmbuild the location of the root directory 68 | # containing the RPM information 69 | # _release Allows Jenkins to setup the version using the 70 | # build number 71 | rpmbuild --define "_topdir ${RPMBUILDDIR}" \ 72 | -ta ${SRCTAR} 73 | if [ $? -ne 0 ] ; then 74 | fail "Unable to create rpm" $? 75 | fi 76 | 77 | # Move the rpms to the root directory 78 | mv ${RPMBUILDDIR_RPMS}/noarch/*rpm ${BUILDDIR} 79 | mv ${RPMBUILDDIR_SRPMS}/*rpm ${BUILDDIR} 80 | if [ $? -ne 0 ] ; then 81 | fail "Unable to move rpm to ${BUILDDIR}" $? 82 | fi 83 | 84 | echo "RPMS are now available in ${BUILDDIR}" 85 | } 86 | 87 | create_src() 88 | { 89 | python setup.py sdist --format=gztar --dist-dir=${BUILDDIR} 90 | if [ $? -ne 0 ] ; then 91 | fail "Unable to create source archive" 92 | fi 93 | } 94 | 95 | ################## MAIN ##################### 96 | 97 | # Create a config file with the package information 98 | PKGCONFIG=${PWD}/pkgconfig.in 99 | env python pkgconfig.py 100 | if [ ! -f "${PKGCONFIG}" ] ; then 101 | fail "Unable to create package information file ${PKGCONFIG}" 1 102 | fi 103 | 104 | # Get package version information 105 | . ${PKGCONFIG} 106 | if [ -z "${NAME}" ] ; then 107 | fail "Unable to read the package name from the file created by pkgconfig.py" 1 108 | fi 109 | if [ -z "${VERSION}" ] ; then 110 | fail "Unable to read the package version from the file created by pkgconfig.py" 1 111 | fi 112 | if [ -z "${RELEASE}" ] ; then 113 | fail "Unable to read the package version from the file created by pkgconfig.py" 1 114 | fi 115 | 116 | PKG_NAME=$NAME 117 | PKG_VERSION=$VERSION 118 | 119 | # 120 | # This can be set by JENKINS builds 121 | # If the environment variable PKG_RELEASE 122 | # has not been set, then we set it locally to 123 | # a default value 124 | # 125 | if [ -z "$PKG_RELEASE" ] ; then 126 | PKG_RELEASE="${RELEASE}" 127 | else 128 | PKG_RELEASE="${RELEASE}.${PKG_RELEASE}" 129 | fi 130 | 131 | BUILDDIR=$PWD/build 132 | RPMBUILDDIR=${BUILDDIR}/rpmbuild 133 | RPMBUILDDIR_RPMS=${RPMBUILDDIR}/RPMS 134 | RPMBUILDDIR_SRPMS=${RPMBUILDDIR}/SRPMS 135 | SRCNAME=${RPMNAME}-${PKG_VERSION}-${PKG_RELEASE} 136 | SRCTAR_DIR=${PKG_NAME}-${PKG_VERSION} 137 | SRCTAR=${RPMBUILDDIR}/${SRCNAME}.tar.gz 138 | 139 | prep 140 | create_src 141 | create_rpm 142 | cleanup 143 | -------------------------------------------------------------------------------- /pkgconfig.py: -------------------------------------------------------------------------------- 1 | # Simple program to save all package information 2 | # into a file which can be sourced by a bash script 3 | 4 | from gluster.swift import _pkginfo as pkginfo 5 | 6 | PKGCONFIG = 'pkgconfig.in' 7 | 8 | pkginfo.save_config(PKGCONFIG) 9 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # The order of packages is significant, because pip processes them in the order 2 | # of appearance. Changing the order has an impact on the overall integration 3 | # process, which may cause wedges in the gate later. 4 | 5 | dnspython>=1.14.0 # http://www.dnspython.org/LICENSE 6 | eventlet>=0.17.4 # MIT 7 | greenlet>=0.3.1 8 | netifaces>=0.5,!=0.10.0,!=0.10.1 9 | pastedeploy>=1.3.3 10 | six>=1.9.0 11 | xattr>=0.4 12 | PyECLib>=1.3.1 # BSD 13 | 14 | # gluster-swift specific requirements 15 | prettytable # needed by gswauth 16 | scandir>=1.3 17 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2013 Red Hat, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from setuptools import setup, find_packages 17 | from gluster.swift import _pkginfo 18 | 19 | 20 | setup( 21 | name=_pkginfo.name, 22 | version=_pkginfo.full_version, 23 | description='Gluster For Swift', 24 | license='Apache License (2.0)', 25 | author='Red Hat, Inc.', 26 | author_email='gluster-users@gluster.org', 27 | url='http://launchpad.net/gluster-swift', 28 | packages=find_packages(exclude=['test', 'bin']), 29 | test_suite='nose.collector', 30 | classifiers=[ 31 | 'Development Status :: 5 - Production/Stable' 32 | 'Environment :: OpenStack' 33 | 'Intended Audience :: Information Technology' 34 | 'Intended Audience :: System Administrators' 35 | 'License :: OSI Approved :: Apache Software License' 36 | 'Operating System :: POSIX :: Linux' 37 | 'Programming Language :: Python' 38 | 'Programming Language :: Python :: 2' 39 | 'Programming Language :: Python :: 2.6' 40 | 'Programming Language :: Python :: 2.7' 41 | ], 42 | install_requires=[], 43 | scripts=[ 44 | 'bin/gluster-swift-gen-builders', 45 | 'bin/gluster-swift-print-metadata', 46 | 'bin/gluster-swift-migrate-metadata', 47 | 'bin/gluster-swift-object-expirer', 48 | 'gluster/swift/common/middleware/gswauth/bin/gswauth-add-account', 49 | 'gluster/swift/common/middleware/gswauth/bin/gswauth-add-user', 50 | 'gluster/swift/common/middleware/gswauth/bin/gswauth-cleanup-tokens', 51 | 'gluster/swift/common/middleware/gswauth/bin/gswauth-delete-account', 52 | 'gluster/swift/common/middleware/gswauth/bin/gswauth-delete-user', 53 | 'gluster/swift/common/middleware/gswauth/bin/gswauth-list', 54 | 'gluster/swift/common/middleware/gswauth/bin/gswauth-prep', 55 | 'gluster/swift/common/middleware/gswauth/bin/' 56 | 'gswauth-set-account-service', 57 | 58 | ], 59 | entry_points={ 60 | 'paste.app_factory': [ 61 | 'proxy=gluster.swift.proxy.server:app_factory', 62 | 'object=gluster.swift.obj.server:app_factory', 63 | 'container=gluster.swift.container.server:app_factory', 64 | 'account=gluster.swift.account.server:app_factory', 65 | ], 66 | 'paste.filter_factory': [ 67 | 'gswauth=gluster.swift.common.middleware.gswauth.swauth.' 68 | 'middleware:filter_factory', 69 | ], 70 | }, 71 | ) 72 | -------------------------------------------------------------------------------- /test-requirements.txt: -------------------------------------------------------------------------------- 1 | # The order of packages is significant, because pip processes them in the order 2 | # of appearance. Changing the order has an impact on the overall integration 3 | # process, which may cause wedges in the gate later. 4 | 5 | # Hacking already pins down pep8, pyflakes and flake8 6 | hacking>=0.11.0,<0.12 7 | coverage>=3.6 8 | nose 9 | nosexcover 10 | nosehtmloutput>=0.0.3 11 | os-api-ref>=1.0.0 # Apache-2.0 12 | os-testr>=0.8.0 13 | mock>=2.0 # BSD 14 | python-swiftclient 15 | python-keystoneclient!=2.1.0,>=2.0.0 16 | -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2010-2012 OpenStack Foundation 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # See http://code.google.com/p/python-nose/issues/detail?id=373 17 | # The code below enables nosetests to work with i18n _() blocks 18 | from __future__ import print_function 19 | import sys 20 | import os 21 | try: 22 | from unittest.util import safe_repr 23 | except ImportError: 24 | # Probably py26 25 | _MAX_LENGTH = 80 26 | 27 | def safe_repr(obj, short=False): 28 | try: 29 | result = repr(obj) 30 | except Exception: 31 | result = object.__repr__(obj) 32 | if not short or len(result) < _MAX_LENGTH: 33 | return result 34 | return result[:_MAX_LENGTH] + ' [truncated]...' 35 | 36 | from eventlet.green import socket 37 | 38 | # make unittests pass on all locale 39 | import swift 40 | setattr(swift, 'gettext_', lambda x: x) 41 | 42 | from swift.common.utils import readconf 43 | 44 | 45 | # Work around what seems to be a Python bug. 46 | # c.f. https://bugs.launchpad.net/swift/+bug/820185. 47 | import logging 48 | logging.raiseExceptions = False 49 | 50 | 51 | def get_config(section_name=None, defaults=None): 52 | """ 53 | Attempt to get a test config dictionary. 54 | 55 | :param section_name: the section to read (all sections if not defined) 56 | :param defaults: an optional dictionary namespace of defaults 57 | """ 58 | config = {} 59 | if defaults is not None: 60 | config.update(defaults) 61 | 62 | config_file = os.environ.get('SWIFT_TEST_CONFIG_FILE', 63 | '/etc/swift/test.conf') 64 | try: 65 | config = readconf(config_file, section_name) 66 | except IOError: 67 | if not os.path.exists(config_file): 68 | print('Unable to read test config %s - file not found' 69 | % config_file, file=sys.stderr) 70 | elif not os.access(config_file, os.R_OK): 71 | print('Unable to read test config %s - permission denied' 72 | % config_file, file=sys.stderr) 73 | except ValueError as e: 74 | print(e) 75 | return config 76 | 77 | 78 | def listen_zero(): 79 | """ 80 | The eventlet.listen() always sets SO_REUSEPORT, so when called with 81 | ("localhost",0), instead of returning unique ports it can return the 82 | same port twice. That causes our tests to fail, so open-code it here 83 | without SO_REUSEPORT. 84 | """ 85 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 86 | sock.bind(("127.0.0.1", 0)) 87 | sock.listen(50) 88 | return sock 89 | -------------------------------------------------------------------------------- /test/functional_auth/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gluster/gluster-swift/488744a005fb399af8d094ad7a62c1917410398c/test/functional_auth/__init__.py -------------------------------------------------------------------------------- /test/functional_auth/common_conf/account-server.conf: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | devices = /mnt/gluster-object 3 | # 4 | # Once you are confident that your startup processes will always have your 5 | # gluster volumes properly mounted *before* the account-server workers start, 6 | # you can *consider* setting this value to "false" to reduce the per-request 7 | # overhead it can incur. 8 | # 9 | # *** Keep false for Functional Tests *** 10 | mount_check = false 11 | bind_port = 6012 12 | # 13 | # Override swift's default behaviour for fallocate. 14 | disable_fallocate = true 15 | # 16 | # One or two workers should be sufficient for almost any installation of 17 | # Gluster. 18 | workers = 1 19 | 20 | [pipeline:main] 21 | pipeline = account-server 22 | 23 | [app:account-server] 24 | use = egg:gluster_swift#account 25 | user = root 26 | log_facility = LOG_LOCAL2 27 | log_level = WARN 28 | # 29 | # After ensuring things are running in a stable manner, you can turn off 30 | # normal request logging for the account server to unclutter the log 31 | # files. Warnings and errors will still be logged. 32 | log_requests = off 33 | 34 | # The following parameter is used by object-expirer and needs to be same 35 | # across all conf files! 36 | auto_create_account_prefix = gs 37 | -------------------------------------------------------------------------------- /test/functional_auth/common_conf/container-server.conf: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | devices = /mnt/gluster-object 3 | # 4 | # Once you are confident that your startup processes will always have your 5 | # gluster volumes properly mounted *before* the container-server workers 6 | # start, you can *consider* setting this value to "false" to reduce the 7 | # per-request overhead it can incur. 8 | # 9 | # *** Keep false for Functional Tests *** 10 | mount_check = false 11 | bind_port = 6011 12 | # 13 | # Override swift's default behaviour for fallocate. 14 | disable_fallocate = true 15 | # 16 | # One or two workers should be sufficient for almost any installation of 17 | # Gluster. 18 | workers = 1 19 | 20 | [pipeline:main] 21 | pipeline = container-server 22 | 23 | [app:container-server] 24 | use = egg:gluster_swift#container 25 | user = root 26 | log_facility = LOG_LOCAL2 27 | log_level = WARN 28 | # 29 | # After ensuring things are running in a stable manner, you can turn off 30 | # normal request logging for the container server to unclutter the log 31 | # files. Warnings and errors will still be logged. 32 | log_requests = off 33 | 34 | #enable object versioning for functional test 35 | allow_versions = on 36 | 37 | # The following parameter is used by object-expirer and needs to be same 38 | # across all conf files! 39 | auto_create_account_prefix = gs 40 | -------------------------------------------------------------------------------- /test/functional_auth/common_conf/fs.conf: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | # 3 | # IP address of a node in the GlusterFS server cluster hosting the 4 | # volumes to be served via Swift API. 5 | mount_ip = localhost 6 | 7 | # Performance optimization parameter. When turned off, the filesystem will 8 | # see a reduced number of stat calls, resulting in substantially faster 9 | # response time for GET and HEAD container requests on containers with large 10 | # numbers of objects, at the expense of an accurate count of combined bytes 11 | # used by all objects in the container. For most installations "off" works 12 | # fine. 13 | # 14 | # *** Keep on for Functional Tests *** 15 | accurate_size_in_listing = on 16 | 17 | # *** Keep on for Functional Tests *** 18 | container_update_object_count = on 19 | account_update_container_count = on 20 | -------------------------------------------------------------------------------- /test/functional_auth/common_conf/object-expirer.conf: -------------------------------------------------------------------------------- 1 | #TODO: Add documentation to explain various options 2 | #For now, refer: https://github.com/openstack/swift/blob/master/etc/object-expirer.conf-sample 3 | 4 | [DEFAULT] 5 | devices = /mnt/gluster-object 6 | 7 | [object-expirer] 8 | user = root 9 | log_facility = LOG_LOCAL2 10 | log_level = DEBUG 11 | # The following parameters are used by object-expirer and needs to be same 12 | # across all conf files! 13 | auto_create_account_prefix = gs 14 | expiring_objects_account_name = expiring 15 | 16 | interval = 30 17 | 18 | [pipeline:main] 19 | pipeline = catch_errors cache proxy-logging proxy-server 20 | 21 | [app:proxy-server] 22 | use = egg:gluster_swift#proxy 23 | log_facility = LOG_LOCAL1 24 | log_level = DEBUG 25 | 26 | [filter:cache] 27 | use = egg:swift#memcache 28 | 29 | [filter:catch_errors] 30 | use = egg:swift#catch_errors 31 | 32 | [filter:proxy-logging] 33 | use = egg:swift#proxy_logging 34 | -------------------------------------------------------------------------------- /test/functional_auth/common_conf/object-server.conf: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | devices = /mnt/gluster-object 3 | # 4 | # Once you are confident that your startup processes will always have your 5 | # gluster volumes properly mounted *before* the object-server workers start, 6 | # you can *consider* setting this value to "false" to reduce the per-request 7 | # overhead it can incur. 8 | # 9 | # *** Keep false for Functional Tests *** 10 | mount_check = false 11 | bind_port = 6010 12 | # 13 | # Maximum number of clients one worker can process simultaneously (it will 14 | # actually accept N + 1). Setting this to one (1) will only handle one request 15 | # at a time, without accepting another request concurrently. By increasing the 16 | # number of workers to a much higher value, one can prevent slow file system 17 | # operations for one request from starving other requests. 18 | max_clients = 1024 19 | # 20 | # If not doing the above, setting this value initially to match the number of 21 | # CPUs is a good starting point for determining the right value. 22 | workers = 1 23 | # Override swift's default behaviour for fallocate. 24 | disable_fallocate = true 25 | 26 | [pipeline:main] 27 | pipeline = object-server 28 | 29 | [app:object-server] 30 | use = egg:gluster_swift#object 31 | user = root 32 | log_facility = LOG_LOCAL2 33 | log_level = WARN 34 | # 35 | # For performance, after ensuring things are running in a stable manner, you 36 | # can turn off normal request logging for the object server to reduce the 37 | # per-request overhead and unclutter the log files. Warnings and errors will 38 | # still be logged. 39 | log_requests = off 40 | # 41 | # Adjust this value to match the stripe width of the underlying storage array 42 | # (not the stripe element size). This will provide a reasonable starting point 43 | # for tuning this value. 44 | disk_chunk_size = 65536 45 | # 46 | # Adjust this value match whatever is set for the disk_chunk_size initially. 47 | # This will provide a reasonable starting point for tuning this value. 48 | network_chunk_size = 65556 49 | 50 | # The following parameter is used by object-expirer and needs to be same 51 | # across all conf files! 52 | auto_create_account_prefix = gs 53 | expiring_objects_account_name = expiring 54 | -------------------------------------------------------------------------------- /test/functional_auth/common_conf/swift.conf: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | 3 | 4 | [swift-hash] 5 | # random unique string that can never change (DO NOT LOSE) 6 | swift_hash_path_suffix = gluster 7 | 8 | 9 | # The swift-constraints section sets the basic constraints on data 10 | # saved in the swift cluster. 11 | 12 | [swift-constraints] 13 | 14 | # max_file_size is the largest "normal" object that can be saved in 15 | # the cluster. This is also the limit on the size of each segment of 16 | # a "large" object when using the large object manifest support. 17 | # This value is set in bytes. Setting it to lower than 1MiB will cause 18 | # some tests to fail. 19 | # Default is 1 TiB = 2**30*1024 20 | max_file_size = 1099511627776 21 | 22 | 23 | # max_meta_name_length is the max number of bytes in the utf8 encoding 24 | # of the name portion of a metadata header. 25 | 26 | #max_meta_name_length = 128 27 | 28 | 29 | # max_meta_value_length is the max number of bytes in the utf8 encoding 30 | # of a metadata value 31 | 32 | #max_meta_value_length = 256 33 | 34 | 35 | # max_meta_count is the max number of metadata keys that can be stored 36 | # on a single account, container, or object 37 | 38 | #max_meta_count = 90 39 | 40 | 41 | # max_meta_overall_size is the max number of bytes in the utf8 encoding 42 | # of the metadata (keys + values) 43 | 44 | #max_meta_overall_size = 4096 45 | 46 | 47 | # max_object_name_length is the max number of bytes in the utf8 encoding of an 48 | # object name: Gluster FS can handle much longer file names, but the length 49 | # between the slashes of the URL is handled below. Remember that most web 50 | # clients can't handle anything greater than 2048, and those that do are 51 | # rather clumsy. 52 | 53 | max_object_name_length = 2048 54 | 55 | # max_object_name_component_length (GlusterFS) is the max number of bytes in 56 | # the utf8 encoding of an object name component (the part between the 57 | # slashes); this is a limit imposed by the underlying file system (for XFS it 58 | # is 255 bytes). 59 | 60 | max_object_name_component_length = 255 61 | 62 | # container_listing_limit is the default (and max) number of items 63 | # returned for a container listing request 64 | 65 | #container_listing_limit = 10000 66 | 67 | 68 | # account_listing_limit is the default (and max) number of items returned 69 | # for an account listing request 70 | 71 | #account_listing_limit = 10000 72 | 73 | 74 | # max_account_name_length is the max number of bytes in the utf8 encoding of 75 | # an account name: Gluster FS Filename limit (XFS limit?), must be the same 76 | # size as max_object_name_component_length above. 77 | 78 | max_account_name_length = 255 79 | 80 | 81 | # max_container_name_length is the max number of bytes in the utf8 encoding 82 | # of a container name: Gluster FS Filename limit (XFS limit?), must be the same 83 | # size as max_object_name_component_length above. 84 | 85 | max_container_name_length = 255 86 | -------------------------------------------------------------------------------- /test/functional_auth/common_conf/test.conf: -------------------------------------------------------------------------------- 1 | [func_test] 2 | # sample config 3 | auth_host = 127.0.0.1 4 | auth_port = 8080 5 | auth_ssl = no 6 | auth_prefix = /auth/ 7 | ## sample config for Swift with Keystone 8 | #auth_version = 2 9 | #auth_host = localhost 10 | #auth_port = 5000 11 | #auth_ssl = no 12 | #auth_prefix = /v2.0/ 13 | 14 | # GSWauth internal admin user configuration information 15 | admin_key = gswauthkey 16 | admin_user = .super_admin 17 | 18 | # Gluster setup information 19 | devices = /mnt/gluster-object 20 | gsmetadata_volume = gsmetadata 21 | 22 | # Primary functional test account (needs admin access to the account) 23 | account = test 24 | username = tester 25 | password = testing 26 | 27 | # User on a second account (needs admin access to the account) 28 | account2 = test2 29 | username2 = tester2 30 | password2 = testing2 31 | 32 | # User on same account as first, but without admin access 33 | username3 = tester3 34 | password3 = testing3 35 | 36 | # Default constraints if not defined here, the test runner will try 37 | # to set them from /etc/swift/swift.conf. If that file isn't found, 38 | # the test runner will skip tests that depend on these values. 39 | # Note that the cluster must have "sane" values for the test suite to pass. 40 | #max_file_size = 5368709122 41 | #max_meta_name_length = 128 42 | #max_meta_value_length = 256 43 | #max_meta_count = 90 44 | #max_meta_overall_size = 4096 45 | #max_object_name_length = 1024 46 | #container_listing_limit = 10000 47 | #account_listing_limit = 10000 48 | #max_account_name_length = 256 49 | #max_container_name_length = 256 50 | normalized_urls = True 51 | 52 | collate = C 53 | 54 | [unit_test] 55 | fake_syslog = False 56 | 57 | [probe_test] 58 | # check_server_timeout = 30 59 | -------------------------------------------------------------------------------- /test/functional_auth/gswauth/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gluster/gluster-swift/488744a005fb399af8d094ad7a62c1917410398c/test/functional_auth/gswauth/__init__.py -------------------------------------------------------------------------------- /test/functional_auth/gswauth/conf/proxy-server.conf: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | bind_port = 8080 3 | user = root 4 | # Consider using 1 worker per CPU 5 | workers = 1 6 | 7 | [pipeline:main] 8 | pipeline = catch_errors gatekeeper healthcheck proxy-logging cache bulk tempurl ratelimit crossdomain gswauth staticweb container-quotas account-quotas slo dlo proxy-logging proxy-server 9 | 10 | [app:proxy-server] 11 | use = egg:gluster_swift#proxy 12 | log_facility = LOG_LOCAL1 13 | log_level = WARN 14 | # The API allows for account creation and deletion, but since Gluster/Swift 15 | # automounts a Gluster volume for a given account, there is no way to create 16 | # or delete an account. So leave this off. 17 | allow_account_management = false 18 | account_autocreate = true 19 | # Ensure the proxy server uses fast-POSTs since we don't need to make a copy 20 | # of the entire object given that all metadata is stored in the object 21 | # extended attributes (no .meta file used after creation) and no container 22 | # sync feature to present. 23 | object_post_as_copy = false 24 | # Only need to recheck the account exists once a day 25 | recheck_account_existence = 86400 26 | # May want to consider bumping this up if containers are created and destroyed 27 | # infrequently. 28 | recheck_container_existence = 60 29 | # Timeout clients that don't read or write to the proxy server after 5 30 | # seconds. 31 | client_timeout = 5 32 | # Give more time to connect to the object, container or account servers in 33 | # cases of high load. 34 | conn_timeout = 5 35 | # For high load situations, once connected to an object, container or account 36 | # server, allow for delays communicating with them. 37 | node_timeout = 60 38 | # May want to consider bumping up this value to 1 - 4 MB depending on how much 39 | # traffic is for multi-megabyte or gigabyte requests; perhaps matching the 40 | # stripe width (not stripe element size) of your storage volume is a good 41 | # starting point. See below for sizing information. 42 | object_chunk_size = 65536 43 | # If you do decide to increase the object_chunk_size, then consider lowering 44 | # this value to one. Up to "put_queue_length" object_chunk_size'd buffers can 45 | # be queued to the object server for processing. Given one proxy server worker 46 | # can handle up to 1,024 connections, by default, it will consume 10 * 65,536 47 | # * 1,024 bytes of memory in the worse case (default values). Be sure the 48 | # amount of memory available on the system can accommodate increased values 49 | # for object_chunk_size. 50 | put_queue_depth = 10 51 | # The following parameter is used by object-expirer and needs to be same 52 | # across all conf files! 53 | auto_create_account_prefix = gs 54 | expiring_objects_account_name = expiring 55 | 56 | [filter:cache] 57 | use = egg:swift#memcache 58 | # Update this line to contain a comma separated list of memcache servers 59 | # shared by all nodes running the proxy-server service. 60 | memcache_servers = localhost:11211 61 | 62 | [filter:catch_errors] 63 | use = egg:swift#catch_errors 64 | 65 | [filter:healthcheck] 66 | use = egg:swift#healthcheck 67 | 68 | [filter:proxy-logging] 69 | use = egg:swift#proxy_logging 70 | 71 | [filter:bulk] 72 | use = egg:swift#bulk 73 | 74 | [filter:ratelimit] 75 | use = egg:swift#ratelimit 76 | 77 | [filter:crossdomain] 78 | use = egg:swift#crossdomain 79 | 80 | [filter:dlo] 81 | use = egg:swift#dlo 82 | 83 | [filter:slo] 84 | use = egg:swift#slo 85 | 86 | [filter:tempurl] 87 | use = egg:swift#tempurl 88 | 89 | [filter:staticweb] 90 | use = egg:swift#staticweb 91 | 92 | [filter:account-quotas] 93 | use = egg:swift#account_quotas 94 | 95 | [filter:container-quotas] 96 | use = egg:swift#container_quotas 97 | 98 | [filter:gatekeeper] 99 | use = egg:swift#gatekeeper 100 | 101 | [filter:gswauth] 102 | use = egg:gluster_swift#gswauth 103 | set log_name = gswauth 104 | super_admin_key = gswauthkey 105 | metadata_volume = gsmetadata 106 | -------------------------------------------------------------------------------- /test/functional_auth/gswauth/conf/test.conf: -------------------------------------------------------------------------------- 1 | [func_test] 2 | # sample config 3 | auth_host = 127.0.0.1 4 | auth_port = 8080 5 | auth_ssl = no 6 | auth_prefix = /auth/ 7 | ## sample config for Swift with Keystone 8 | #auth_version = 2 9 | #auth_host = localhost 10 | #auth_port = 5000 11 | #auth_ssl = no 12 | #auth_prefix = /v2.0/ 13 | 14 | # GSWauth internal admin user configuration information 15 | admin_key = gswauthkey 16 | admin_user = .super_admin 17 | 18 | # Gluster setup information 19 | devices = /mnt/gluster-object 20 | gsmetadata_volume = gsmetadata 21 | 22 | # Primary functional test account (needs admin access to the account) 23 | account = test 24 | username = tester 25 | password = testing 26 | 27 | # User on a second account (needs admin access to the account) 28 | account2 = test2 29 | username2 = tester2 30 | password2 = testing2 31 | 32 | # User on same account as first, but without admin access 33 | username3 = tester3 34 | password3 = testing3 35 | 36 | # Default constraints if not defined here, the test runner will try 37 | # to set them from /etc/swift/swift.conf. If that file isn't found, 38 | # the test runner will skip tests that depend on these values. 39 | # Note that the cluster must have "sane" values for the test suite to pass. 40 | #max_file_size = 5368709122 41 | #max_meta_name_length = 128 42 | #max_meta_value_length = 256 43 | #max_meta_count = 90 44 | #max_meta_overall_size = 4096 45 | #max_object_name_length = 1024 46 | #container_listing_limit = 10000 47 | #account_listing_limit = 10000 48 | #max_account_name_length = 256 49 | #max_container_name_length = 256 50 | normalized_urls = True 51 | 52 | collate = C 53 | 54 | [unit_test] 55 | fake_syslog = False 56 | 57 | [probe_test] 58 | # check_server_timeout = 30 59 | -------------------------------------------------------------------------------- /test/functional_auth/keystone/conf/proxy-server.conf: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | bind_port = 8080 3 | user = root 4 | # Consider using 1 worker per CPU 5 | workers = 1 6 | 7 | [pipeline:main] 8 | #pipeline = catch_errors healthcheck proxy-logging cache tempauth proxy-logging proxy-server 9 | pipeline = catch_errors healthcheck proxy-logging cache tempurl authtoken keystoneauth proxy-logging proxy-server 10 | 11 | [app:proxy-server] 12 | use = egg:gluster_swift#proxy 13 | log_facility = LOG_LOCAL1 14 | log_level = WARN 15 | # The API allows for account creation and deletion, but since Gluster/Swift 16 | # automounts a Gluster volume for a given account, there is no way to create 17 | # or delete an account. So leave this off. 18 | allow_account_management = false 19 | account_autocreate = true 20 | # Ensure the proxy server uses fast-POSTs since we don't need to make a copy 21 | # of the entire object given that all metadata is stored in the object 22 | # extended attributes (no .meta file used after creation) and no container 23 | # sync feature to present. 24 | object_post_as_copy = false 25 | # Only need to recheck the account exists once a day 26 | recheck_account_existence = 86400 27 | # May want to consider bumping this up if containers are created and destroyed 28 | # infrequently. 29 | recheck_container_existence = 60 30 | # Timeout clients that don't read or write to the proxy server after 5 31 | # seconds. 32 | client_timeout = 5 33 | # Give more time to connect to the object, container or account servers in 34 | # cases of high load. 35 | conn_timeout = 5 36 | # For high load situations, once connected to an object, container or account 37 | # server, allow for delays communicating with them. 38 | node_timeout = 60 39 | # May want to consider bumping up this value to 1 - 4 MB depending on how much 40 | # traffic is for multi-megabyte or gigabyte requests; perhaps matching the 41 | # stripe width (not stripe element size) of your storage volume is a good 42 | # starting point. See below for sizing information. 43 | object_chunk_size = 65536 44 | # If you do decide to increase the object_chunk_size, then consider lowering 45 | # this value to one. Up to "put_queue_length" object_chunk_size'd buffers can 46 | # be queued to the object server for processing. Given one proxy server worker 47 | # can handle up to 1,024 connections, by default, it will consume 10 * 65,536 48 | # * 1,024 bytes of memory in the worse case (default values). Be sure the 49 | # amount of memory available on the system can accommodate increased values 50 | # for object_chunk_size. 51 | put_queue_depth = 10 52 | # The following parameter is used by object-expirer and needs to be same 53 | # across all conf files! 54 | auto_create_account_prefix = gs 55 | expiring_objects_account_name = expiring 56 | 57 | [filter:catch_errors] 58 | use = egg:swift#catch_errors 59 | 60 | [filter:proxy-logging] 61 | use = egg:swift#proxy_logging 62 | 63 | [filter:healthcheck] 64 | use = egg:swift#healthcheck 65 | 66 | [filter:cache] 67 | use = egg:swift#memcache 68 | # Update this line to contain a comma separated list of memcache servers 69 | # shared by all nodes running the proxy-server service. 70 | memcache_servers = localhost:11211 71 | 72 | [filter:tempurl] 73 | use = egg:swift#tempurl 74 | 75 | [filter:authtoken] 76 | paste.filter_factory = keystoneclient.middleware.auth_token:filter_factory 77 | signing_dir = /etc/swift 78 | # Keystone server info 79 | auth_host = 127.0.0.1 80 | auth_port = 35357 81 | auth_protocol = http 82 | # Swift server info 83 | service_host = 127.0.0.1 84 | service_port = 8080 85 | admin_token = ADMIN 86 | # Needed to support Swift container ACL 87 | delay_auth_decision = true 88 | 89 | [filter:keystoneauth] 90 | use = egg:swift#keystoneauth 91 | operator_roles = admin 92 | is_admin = true 93 | cache = swift.cache 94 | 95 | 96 | -------------------------------------------------------------------------------- /test/functional_auth/keystone/conf/test.conf: -------------------------------------------------------------------------------- 1 | [func_test] 2 | # sample config 3 | #auth_host = 127.0.0.1 4 | #auth_port = 8080 5 | #auth_ssl = no 6 | #auth_prefix = /auth/ 7 | ## sample config for Swift with Keystone 8 | auth_version = 2 9 | auth_host = localhost 10 | auth_port = 5000 11 | auth_ssl = no 12 | auth_prefix = /v2.0/ 13 | 14 | # Primary functional test account (needs admin access to the account) 15 | account = test 16 | username = tester 17 | password = testing 18 | 19 | # User on a second account (needs admin access to the account) 20 | account2 = test2 21 | username2 = tester2 22 | password2 = testing2 23 | 24 | # User on same account as first, but without admin access 25 | username3 = tester3 26 | password3 = testing3 27 | 28 | # Default constraints if not defined here, the test runner will try 29 | # to set them from /etc/swift/swift.conf. If that file isn't found, 30 | # the test runner will skip tests that depend on these values. 31 | # Note that the cluster must have "sane" values for the test suite to pass. 32 | #max_file_size = 5368709122 33 | #max_meta_name_length = 128 34 | #max_meta_value_length = 256 35 | #max_meta_count = 90 36 | #max_meta_overall_size = 4096 37 | #max_object_name_length = 1024 38 | #container_listing_limit = 10000 39 | #account_listing_limit = 10000 40 | #max_account_name_length = 256 41 | #max_container_name_length = 256 42 | normalized_urls = True 43 | 44 | collate = C 45 | 46 | [unit_test] 47 | fake_syslog = False 48 | 49 | [probe_test] 50 | # check_server_timeout = 30 51 | -------------------------------------------------------------------------------- /test/functional_auth/swiftkerbauth/conf/proxy-server.conf: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | bind_port = 8080 3 | user = root 4 | # Consider using 1 worker per CPU 5 | workers = 1 6 | 7 | [pipeline:main] 8 | pipeline = catch_errors healthcheck proxy-logging cache tempurl proxy-logging kerbauth proxy-server 9 | 10 | [app:proxy-server] 11 | use = egg:gluster_swift#proxy 12 | log_facility = LOG_LOCAL1 13 | log_level = WARN 14 | # The API allows for account creation and deletion, but since Gluster/Swift 15 | # automounts a Gluster volume for a given account, there is no way to create 16 | # or delete an account. So leave this off. 17 | allow_account_management = false 18 | account_autocreate = true 19 | # Ensure the proxy server uses fast-POSTs since we don't need to make a copy 20 | # of the entire object given that all metadata is stored in the object 21 | # extended attributes (no .meta file used after creation) and no container 22 | # sync feature to present. 23 | object_post_as_copy = false 24 | # Only need to recheck the account exists once a day 25 | recheck_account_existence = 86400 26 | # May want to consider bumping this up if containers are created and destroyed 27 | # infrequently. 28 | recheck_container_existence = 60 29 | # Timeout clients that don't read or write to the proxy server after 5 30 | # seconds. 31 | client_timeout = 5 32 | # Give more time to connect to the object, container or account servers in 33 | # cases of high load. 34 | conn_timeout = 5 35 | # For high load situations, once connected to an object, container or account 36 | # server, allow for delays communicating with them. 37 | node_timeout = 60 38 | # May want to consider bumping up this value to 1 - 4 MB depending on how much 39 | # traffic is for multi-megabyte or gigabyte requests; perhaps matching the 40 | # stripe width (not stripe element size) of your storage volume is a good 41 | # starting point. See below for sizing information. 42 | object_chunk_size = 65536 43 | # If you do decide to increase the object_chunk_size, then consider lowering 44 | # this value to one. Up to "put_queue_length" object_chunk_size'd buffers can 45 | # be queued to the object server for processing. Given one proxy server worker 46 | # can handle up to 1,024 connections, by default, it will consume 10 * 65,536 47 | # * 1,024 bytes of memory in the worse case (default values). Be sure the 48 | # amount of memory available on the system can accommodate increased values 49 | # for object_chunk_size. 50 | put_queue_depth = 10 51 | # The following parameter is used by object-expirer and needs to be same 52 | # across all conf files! 53 | auto_create_account_prefix = gs 54 | expiring_objects_account_name = expiring 55 | 56 | [filter:catch_errors] 57 | use = egg:swift#catch_errors 58 | 59 | [filter:proxy-logging] 60 | use = egg:swift#proxy_logging 61 | access_log_level = WARN 62 | 63 | [filter:healthcheck] 64 | use = egg:swift#healthcheck 65 | 66 | [filter:cache] 67 | use = egg:swift#memcache 68 | # Update this line to contain a comma separated list of memcache servers 69 | # shared by all nodes running the proxy-server service. 70 | memcache_servers = localhost:11211 71 | 72 | [filter:tempurl] 73 | use = egg:swift#tempurl 74 | 75 | [filter:kerbauth] 76 | use = egg:gluster_swift#kerbauth 77 | ext_authentication_url = http://client.rhelbox.com/cgi-bin/swift-auth 78 | -------------------------------------------------------------------------------- /test/functional_auth/tempauth/conf/proxy-server.conf: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | bind_port = 8080 3 | user = root 4 | # Consider using 1 worker per CPU 5 | workers = 1 6 | 7 | [pipeline:main] 8 | pipeline = catch_errors gatekeeper healthcheck proxy-logging cache bulk tempurl ratelimit crossdomain tempauth staticweb container-quotas account-quotas slo dlo proxy-logging proxy-server 9 | 10 | [app:proxy-server] 11 | use = egg:gluster_swift#proxy 12 | log_facility = LOG_LOCAL1 13 | log_level = WARN 14 | # The API allows for account creation and deletion, but since Gluster/Swift 15 | # automounts a Gluster volume for a given account, there is no way to create 16 | # or delete an account. So leave this off. 17 | allow_account_management = false 18 | account_autocreate = true 19 | # Ensure the proxy server uses fast-POSTs since we don't need to make a copy 20 | # of the entire object given that all metadata is stored in the object 21 | # extended attributes (no .meta file used after creation) and no container 22 | # sync feature to present. 23 | object_post_as_copy = false 24 | # Only need to recheck the account exists once a day 25 | recheck_account_existence = 86400 26 | # May want to consider bumping this up if containers are created and destroyed 27 | # infrequently. 28 | recheck_container_existence = 60 29 | # Timeout clients that don't read or write to the proxy server after 5 30 | # seconds. 31 | client_timeout = 5 32 | # Give more time to connect to the object, container or account servers in 33 | # cases of high load. 34 | conn_timeout = 5 35 | # For high load situations, once connected to an object, container or account 36 | # server, allow for delays communicating with them. 37 | node_timeout = 60 38 | # May want to consider bumping up this value to 1 - 4 MB depending on how much 39 | # traffic is for multi-megabyte or gigabyte requests; perhaps matching the 40 | # stripe width (not stripe element size) of your storage volume is a good 41 | # starting point. See below for sizing information. 42 | object_chunk_size = 65536 43 | # If you do decide to increase the object_chunk_size, then consider lowering 44 | # this value to one. Up to "put_queue_length" object_chunk_size'd buffers can 45 | # be queued to the object server for processing. Given one proxy server worker 46 | # can handle up to 1,024 connections, by default, it will consume 10 * 65,536 47 | # * 1,024 bytes of memory in the worse case (default values). Be sure the 48 | # amount of memory available on the system can accommodate increased values 49 | # for object_chunk_size. 50 | put_queue_depth = 10 51 | # The following parameter is used by object-expirer and needs to be same 52 | # across all conf files! 53 | auto_create_account_prefix = gs 54 | expiring_objects_account_name = expiring 55 | 56 | [filter:cache] 57 | use = egg:swift#memcache 58 | # Update this line to contain a comma separated list of memcache servers 59 | # shared by all nodes running the proxy-server service. 60 | memcache_servers = localhost:11211 61 | 62 | [filter:catch_errors] 63 | use = egg:swift#catch_errors 64 | 65 | [filter:healthcheck] 66 | use = egg:swift#healthcheck 67 | 68 | [filter:proxy-logging] 69 | use = egg:swift#proxy_logging 70 | 71 | [filter:bulk] 72 | use = egg:swift#bulk 73 | 74 | [filter:ratelimit] 75 | use = egg:swift#ratelimit 76 | 77 | [filter:crossdomain] 78 | use = egg:swift#crossdomain 79 | 80 | [filter:dlo] 81 | use = egg:swift#dlo 82 | 83 | [filter:slo] 84 | use = egg:swift#slo 85 | 86 | [filter:tempurl] 87 | use = egg:swift#tempurl 88 | 89 | [filter:staticweb] 90 | use = egg:swift#staticweb 91 | 92 | [filter:account-quotas] 93 | use = egg:swift#account_quotas 94 | 95 | [filter:container-quotas] 96 | use = egg:swift#container_quotas 97 | 98 | [filter:gatekeeper] 99 | use = egg:swift#gatekeeper 100 | 101 | [filter:tempauth] 102 | use = egg:swift#tempauth 103 | user_admin_admin = admin .admin .reseller_admin 104 | user_test_tester = testing .admin 105 | user_test2_tester2 = testing2 .admin 106 | user_test_tester3 = testing3 107 | -------------------------------------------------------------------------------- /test/functional_auth/tempauth/conf/test.conf: -------------------------------------------------------------------------------- 1 | [func_test] 2 | # sample config 3 | auth_host = 127.0.0.1 4 | auth_port = 8080 5 | auth_ssl = no 6 | auth_prefix = /auth/ 7 | ## sample config for Swift with Keystone 8 | #auth_version = 2 9 | #auth_host = localhost 10 | #auth_port = 5000 11 | #auth_ssl = no 12 | #auth_prefix = /v2.0/ 13 | 14 | # Primary functional test account (needs admin access to the account) 15 | account = test 16 | username = tester 17 | password = testing 18 | 19 | # User on a second account (needs admin access to the account) 20 | account2 = test2 21 | username2 = tester2 22 | password2 = testing2 23 | 24 | # User on same account as first, but without admin access 25 | username3 = tester3 26 | password3 = testing3 27 | 28 | # Default constraints if not defined here, the test runner will try 29 | # to set them from /etc/swift/swift.conf. If that file isn't found, 30 | # the test runner will skip tests that depend on these values. 31 | # Note that the cluster must have "sane" values for the test suite to pass. 32 | #max_file_size = 5368709122 33 | #max_meta_name_length = 128 34 | #max_meta_value_length = 256 35 | #max_meta_count = 90 36 | #max_meta_overall_size = 4096 37 | #max_object_name_length = 1024 38 | #container_listing_limit = 10000 39 | #account_listing_limit = 10000 40 | #max_account_name_length = 256 41 | #max_container_name_length = 256 42 | normalized_urls = True 43 | 44 | collate = C 45 | 46 | [unit_test] 47 | fake_syslog = False 48 | 49 | [probe_test] 50 | # check_server_timeout = 30 51 | -------------------------------------------------------------------------------- /test/object_expirer_functional/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gluster/gluster-swift/488744a005fb399af8d094ad7a62c1917410398c/test/object_expirer_functional/__init__.py -------------------------------------------------------------------------------- /test/unit/account/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gluster/gluster-swift/488744a005fb399af8d094ad7a62c1917410398c/test/unit/account/__init__.py -------------------------------------------------------------------------------- /test/unit/account/test_server.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2013 Red Hat, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | """ Tests for gluster.swift.account.server subclass """ 17 | 18 | import os 19 | import errno 20 | import unittest 21 | from nose import SkipTest 22 | 23 | import gluster.swift.common.Glusterfs 24 | 25 | gluster.swift.common.Glusterfs.RUN_DIR = '/tmp/gluster_unit_tests/run' 26 | try: 27 | os.makedirs(gluster.swift.common.Glusterfs.RUN_DIR) 28 | except OSError as e: 29 | if e.errno != errno.EEXIST: 30 | raise 31 | 32 | import gluster.swift.account.server as server 33 | 34 | 35 | class TestAccountServer(unittest.TestCase): 36 | """ 37 | Tests for account server subclass. 38 | """ 39 | 40 | def test_constructor(self): 41 | raise SkipTest 42 | -------------------------------------------------------------------------------- /test/unit/common/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gluster/gluster-swift/488744a005fb399af8d094ad7a62c1917410398c/test/unit/common/__init__.py -------------------------------------------------------------------------------- /test/unit/common/data/README.rings: -------------------------------------------------------------------------------- 1 | The unit tests expect certain ring data built using the following command: 2 | 3 | ../../../../bin/gluster-swift-gen-builders test iops 4 | -------------------------------------------------------------------------------- /test/unit/common/data/account.builder: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gluster/gluster-swift/488744a005fb399af8d094ad7a62c1917410398c/test/unit/common/data/account.builder -------------------------------------------------------------------------------- /test/unit/common/data/account.ring.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gluster/gluster-swift/488744a005fb399af8d094ad7a62c1917410398c/test/unit/common/data/account.ring.gz -------------------------------------------------------------------------------- /test/unit/common/data/account_tree.tar.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gluster/gluster-swift/488744a005fb399af8d094ad7a62c1917410398c/test/unit/common/data/account_tree.tar.bz2 -------------------------------------------------------------------------------- /test/unit/common/data/container.builder: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gluster/gluster-swift/488744a005fb399af8d094ad7a62c1917410398c/test/unit/common/data/container.builder -------------------------------------------------------------------------------- /test/unit/common/data/container.ring.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gluster/gluster-swift/488744a005fb399af8d094ad7a62c1917410398c/test/unit/common/data/container.ring.gz -------------------------------------------------------------------------------- /test/unit/common/data/container_tree.tar.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gluster/gluster-swift/488744a005fb399af8d094ad7a62c1917410398c/test/unit/common/data/container_tree.tar.bz2 -------------------------------------------------------------------------------- /test/unit/common/data/object.builder: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gluster/gluster-swift/488744a005fb399af8d094ad7a62c1917410398c/test/unit/common/data/object.builder -------------------------------------------------------------------------------- /test/unit/common/data/object.ring.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gluster/gluster-swift/488744a005fb399af8d094ad7a62c1917410398c/test/unit/common/data/object.ring.gz -------------------------------------------------------------------------------- /test/unit/common/middleware/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gluster/gluster-swift/488744a005fb399af8d094ad7a62c1917410398c/test/unit/common/middleware/__init__.py -------------------------------------------------------------------------------- /test/unit/common/middleware/gswauth/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gluster/gluster-swift/488744a005fb399af8d094ad7a62c1917410398c/test/unit/common/middleware/gswauth/__init__.py -------------------------------------------------------------------------------- /test/unit/common/middleware/gswauth/swauth/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gluster/gluster-swift/488744a005fb399af8d094ad7a62c1917410398c/test/unit/common/middleware/gswauth/swauth/__init__.py -------------------------------------------------------------------------------- /test/unit/common/middleware/gswauth/swauth/test_authtypes.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 10 | # implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | # Pablo Llopis 2011 15 | 16 | import unittest 17 | import gluster.swift.common.middleware.gswauth.swauth.authtypes as authtypes 18 | 19 | 20 | class TestPlaintext(unittest.TestCase): 21 | 22 | def setUp(self): 23 | self.auth_encoder = authtypes.Plaintext() 24 | 25 | def test_plaintext_encode(self): 26 | enc_key = self.auth_encoder.encode('keystring') 27 | self.assertEquals('plaintext:keystring', enc_key) 28 | 29 | def test_plaintext_valid_match(self): 30 | creds = 'plaintext:keystring' 31 | match = self.auth_encoder.match('keystring', creds) 32 | self.assertEquals(match, True) 33 | 34 | def test_plaintext_invalid_match(self): 35 | creds = 'plaintext:other-keystring' 36 | match = self.auth_encoder.match('keystring', creds) 37 | self.assertEquals(match, False) 38 | 39 | 40 | class TestSha1(unittest.TestCase): 41 | 42 | def setUp(self): 43 | self.auth_encoder = authtypes.Sha1() 44 | self.auth_encoder.salt = 'salt' 45 | 46 | def test_sha1_encode(self): 47 | enc_key = self.auth_encoder.encode('keystring') 48 | self.assertEquals('sha1:salt$d50dc700c296e23ce5b41f7431a0e01f69010f06', 49 | enc_key) 50 | 51 | def test_sha1_valid_match(self): 52 | creds = 'sha1:salt$d50dc700c296e23ce5b41f7431a0e01f69010f06' 53 | match = self.auth_encoder.match('keystring', creds) 54 | self.assertEquals(match, True) 55 | 56 | def test_sha1_invalid_match(self): 57 | creds = 'sha1:salt$deadbabedeadbabedeadbabec0ffeebadc0ffeee' 58 | match = self.auth_encoder.match('keystring', creds) 59 | self.assertEquals(match, False) 60 | 61 | 62 | if __name__ == '__main__': 63 | unittest.main() 64 | -------------------------------------------------------------------------------- /test/unit/common/test_constraints.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2013 Red Hat, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import unittest 17 | import swift.common.constraints 18 | from nose import SkipTest 19 | from mock import Mock, patch 20 | from gluster.swift.common import constraints as cnt 21 | 22 | 23 | def mock_glusterfs_mount(*args, **kwargs): 24 | return True 25 | 26 | 27 | def mock_constraints_conf_int(*args, **kwargs): 28 | return 1000 29 | 30 | 31 | def mock_check_object_creation(*args, **kwargs): 32 | return None 33 | 34 | 35 | def mock_check_mount(*args, **kwargs): 36 | return True 37 | 38 | 39 | def mock_check_mount_err(*args, **kwargs): 40 | return False 41 | 42 | 43 | class TestConstraints(unittest.TestCase): 44 | """ Tests for common.constraints """ 45 | 46 | def tearDown(self): 47 | cnt.set_object_name_component_length() 48 | 49 | def test_set_object_name_component_length(self): 50 | len = cnt.get_object_name_component_length() 51 | cnt.set_object_name_component_length(len+1) 52 | self.assertEqual(len, cnt.get_object_name_component_length()-1) 53 | 54 | if hasattr(swift.common.constraints, 'constraints_conf_int'): 55 | len = swift.common.constraints.constraints_conf_int( 56 | 'max_object_name_component_length', 255) 57 | cnt.set_object_name_component_length() 58 | self.assertEqual(len, cnt.get_object_name_component_length()) 59 | 60 | with patch('swift.common.constraints.constraints_conf_int', 61 | mock_constraints_conf_int): 62 | cnt.set_object_name_component_length() 63 | self.assertEqual(cnt.get_object_name_component_length(), 1000) 64 | 65 | def test_validate_obj_name_component(self): 66 | max_obj_len = cnt.get_object_name_component_length() 67 | self.assertFalse(cnt.validate_obj_name_component('tests'*(max_obj_len/5))) 68 | cnt.set_object_name_component_length(300) 69 | self.assertFalse(cnt.validate_obj_name_component('tests'*60)) 70 | 71 | def test_validate_obj_name_component_err(self): 72 | max_obj_len = cnt.get_object_name_component_length() 73 | self.assertTrue(cnt.validate_obj_name_component('tests'*(max_obj_len/5+1))) 74 | self.assertTrue(cnt.validate_obj_name_component('.')) 75 | self.assertTrue(cnt.validate_obj_name_component('..')) 76 | self.assertTrue(cnt.validate_obj_name_component('')) 77 | 78 | def test_gluster_check_object_creation(self): 79 | with patch('gluster.swift.common.constraints.__check_object_creation', 80 | mock_check_object_creation): 81 | req = Mock() 82 | req.headers = [] 83 | self.assertFalse(cnt.gluster_check_object_creation(req, 'dir/z')) 84 | 85 | def test_gluster_check_object_creation_err(self): 86 | with patch('gluster.swift.common.constraints.__check_object_creation', 87 | mock_check_object_creation): 88 | req = Mock() 89 | req.headers = [] 90 | self.assertTrue(cnt.gluster_check_object_creation(req, 'dir/.')) 91 | -------------------------------------------------------------------------------- /test/unit/common/test_ring.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2013 Red Hat, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import os 17 | import errno 18 | import unittest 19 | import gluster.swift.common.constraints 20 | import swift.common.utils 21 | from gluster.swift.common.ring import Ring 22 | 23 | 24 | class TestRing(unittest.TestCase): 25 | """ Tests for common.ring """ 26 | 27 | def setUp(self): 28 | swift.common.utils.HASH_PATH_SUFFIX = 'endcap' 29 | swiftdir = os.path.join(os.getcwd(), "common", "data") 30 | self.ring = Ring(swiftdir, ring_name='object') 31 | 32 | def test_first_device(self): 33 | part, node = self.ring.get_nodes('test') 34 | assert node[0]['device'] == 'test' 35 | node = self.ring.get_part_nodes(0) 36 | assert node[0]['device'] == 'test' 37 | for node in self.ring.get_more_nodes(0): 38 | assert node['device'] == 'volume_not_in_ring' 39 | 40 | def test_invalid_device(self): 41 | part, node = self.ring.get_nodes('test2') 42 | assert node[0]['device'] == 'volume_not_in_ring' 43 | node = self.ring.get_part_nodes(0) 44 | assert node[0]['device'] == 'volume_not_in_ring' 45 | 46 | def test_second_device(self): 47 | part, node = self.ring.get_nodes('iops') 48 | assert node[0]['device'] == 'iops' 49 | node = self.ring.get_part_nodes(0) 50 | assert node[0]['device'] == 'iops' 51 | for node in self.ring.get_more_nodes(0): 52 | assert node['device'] == 'volume_not_in_ring' 53 | 54 | def test_second_device_part(self): 55 | part = self.ring.get_part('iops') 56 | assert part == 0 57 | 58 | def test_second_device_with_reseller_prefix(self): 59 | part, node = self.ring.get_nodes('AUTH_iops') 60 | assert node[0]['device'] == 'iops' 61 | 62 | def test_partition_id_for_multiple_accounts(self): 63 | test_part, test_node = self.ring.get_nodes('test') 64 | iops_part, iops_node = self.ring.get_nodes('iops') 65 | self.assertNotEqual(test_part, iops_part) 66 | self.assertEqual(test_node, self.ring.get_part_nodes(test_part)) 67 | self.assertEqual(iops_node, self.ring.get_part_nodes(iops_part)) 68 | self.assertNotEqual(test_node, self.ring.get_part_nodes(iops_part)) 69 | self.assertNotEqual(iops_node, self.ring.get_part_nodes(test_part)) 70 | 71 | def test_invalid_partition(self): 72 | nodes = self.ring.get_part_nodes(0) 73 | self.assertEqual(nodes[0]['device'], 'volume_not_in_ring') 74 | 75 | def test_ring_file_enoent(self): 76 | swiftdir = os.path.join(os.getcwd(), "common", "data") 77 | try: 78 | self.ring = Ring(swiftdir, ring_name='obj') 79 | except OSError as ose: 80 | if ose.errno == errno.ENOENT: 81 | pass 82 | else: 83 | self.fail('ENOENT expected, %s received.' %ose.errno) 84 | else: 85 | self.fail('OSError expected.') 86 | -------------------------------------------------------------------------------- /test/unit/container/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gluster/gluster-swift/488744a005fb399af8d094ad7a62c1917410398c/test/unit/container/__init__.py -------------------------------------------------------------------------------- /test/unit/container/test_server.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2013 Red Hat, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | """ Tests for gluster.swift.container.server subclass """ 17 | 18 | import unittest 19 | from nose import SkipTest 20 | 21 | import gluster.swift.container.server as server 22 | 23 | 24 | class TestContainerServer(unittest.TestCase): 25 | """ 26 | Tests for container server subclass. 27 | """ 28 | 29 | def test_constructor(self): 30 | raise SkipTest 31 | -------------------------------------------------------------------------------- /test/unit/obj/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gluster/gluster-swift/488744a005fb399af8d094ad7a62c1917410398c/test/unit/obj/__init__.py -------------------------------------------------------------------------------- /test/unit/obj/test_server.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2013 Red Hat, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | """ Tests for gluster.swift.obj.server subclass """ 17 | 18 | import unittest 19 | from nose import SkipTest 20 | 21 | import gluster.swift.obj.server as server 22 | 23 | 24 | class TestObjServer(unittest.TestCase): 25 | """ 26 | Tests for object server subclass. 27 | """ 28 | 29 | def test_constructor(self): 30 | raise SkipTest 31 | -------------------------------------------------------------------------------- /test/unit/proxy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gluster/gluster-swift/488744a005fb399af8d094ad7a62c1917410398c/test/unit/proxy/__init__.py -------------------------------------------------------------------------------- /test/unit/proxy/controllers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gluster/gluster-swift/488744a005fb399af8d094ad7a62c1917410398c/test/unit/proxy/controllers/__init__.py -------------------------------------------------------------------------------- /test/unit/test_swift.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2013 Red Hat, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | """ Tests for gluster.swift """ 17 | 18 | import os 19 | import unittest 20 | import shutil 21 | import tempfile 22 | 23 | import gluster.swift as gs 24 | 25 | 26 | class TestPkgInfo(unittest.TestCase): 27 | """ 28 | Tests for gluster.swift PkgInfo class. 29 | """ 30 | 31 | def test_constructor(self): 32 | pi = gs.PkgInfo('a', 'b', 'c', 'd') 33 | assert pi.canonical_version == 'a' 34 | assert pi.name == 'c' 35 | self.assertEqual(pi.release, 'b') 36 | assert pi.final == 'd' 37 | 38 | def test_pretty_version(self): 39 | pi = gs.PkgInfo('a', 'b', 'c', False) 40 | assert pi.pretty_version == 'a-dev' 41 | pi = gs.PkgInfo('a', 'b', 'c', True) 42 | assert pi.pretty_version == 'a' 43 | 44 | def test_save_config(self): 45 | pi = gs.PkgInfo('a', 'b', 'c', 'd') 46 | td = tempfile.mkdtemp() 47 | try: 48 | sc = os.path.join(td, 'saved_config.txt') 49 | pi.save_config(sc) 50 | exp = 'NAME=c\nVERSION=a\nRELEASE=b\n' 51 | contents = file(sc, 'r').read() 52 | assert contents == exp 53 | finally: 54 | shutil.rmtree(td) 55 | -------------------------------------------------------------------------------- /tools/gswauth_functional_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (c) 2013 Red Hat, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 14 | # implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | # This program expects to be run by tox in a virtual python environment 19 | # so that it does not pollute the host development system 20 | 21 | sudo_env() 22 | { 23 | sudo bash -c "PATH=$PATH $*" 24 | } 25 | 26 | cleanup() 27 | { 28 | sudo service memcached stop 29 | sudo_env swift-init main stop 30 | sudo rm -rf /etc/swift > /dev/null 2>&1 31 | sudo rm -rf /mnt/gluster-object/test{,2}/* > /dev/null 2>&1 32 | sudo setfattr -x user.swift.metadata /mnt/gluster-object/test{,2} > /dev/null 2>&1 33 | gswauth_cleanup 34 | } 35 | 36 | gswauth_cleanup() 37 | { 38 | sudo rm -rf /mnt/gluster-object/gsmetadata/.* > /dev/null 2>&1 39 | sudo rm -rf /mnt/gluster-object/gsmetadata/* > /dev/null 2>&1 40 | sudo setfattr -x user.swift.metadata /mnt/gluster-object/gsmetadata > /dev/null 2>&1 41 | } 42 | 43 | quit() 44 | { 45 | echo "$1" 46 | exit 1 47 | } 48 | 49 | 50 | fail() 51 | { 52 | cleanup 53 | quit "$1" 54 | } 55 | 56 | run_generic_tests() 57 | { 58 | # clean up gsmetadata dir 59 | gswauth_cleanup 60 | 61 | #swauth-prep 62 | sudo_env gswauth-prep -K gswauthkey || fail "Unable to prep gswauth" 63 | sudo_env gswauth-add-user -K gswauthkey -a test tester testing || fail "Unable to add user test" 64 | sudo_env gswauth-add-user -K gswauthkey -a test2 tester2 testing2 || fail "Unable to add user test2" 65 | sudo_env gswauth-add-user -K gswauthkey test tester3 testing3 || fail "Unable to add user test3" 66 | 67 | nosetests -v --exe \ 68 | --with-xunit \ 69 | --xunit-file functional_tests_result/gluster-swift-gswauth-generic-functional-TC-report.xml \ 70 | --with-html-output \ 71 | --html-out-file functional_tests_result/gluster-swift-gswauth-generic-functional-result.html \ 72 | test/functional || fail "Functional tests failed" 73 | } 74 | 75 | ### MAIN ### 76 | 77 | # Only run if there is no configuration in the system 78 | if [ -x /etc/swift ] ; then 79 | quit "/etc/swift exists, cannot run functional tests." 80 | fi 81 | 82 | # Check the directories exist 83 | DIRS="/mnt/gluster-object /mnt/gluster-object/test /mnt/gluster-object/test2 /mnt/gluster-object/gsmetadata" 84 | for d in $DIRS ; do 85 | if [ ! -x $d ] ; then 86 | quit "$d must exist on an XFS or GlusterFS volume" 87 | fi 88 | done 89 | 90 | export SWIFT_TEST_CONFIG_FILE=/etc/swift/test.conf 91 | 92 | # Install the configuration files 93 | sudo mkdir /etc/swift > /dev/null 2>&1 94 | sudo cp -r test/functional_auth/common_conf/* /etc/swift || fail "Unable to copy configuration files to /etc/swift" 95 | sudo cp -r test/functional_auth/gswauth/conf/* /etc/swift || fail "Unable to copy configuration files to /etc/swift" 96 | sudo_env gluster-swift-gen-builders test test2 gsmetadata || fail "Unable to create ring files" 97 | 98 | # Start the services 99 | sudo service memcached start || fail "Unable to start memcached" 100 | sudo_env swift-init main start || fail "Unable to start swift" 101 | 102 | #swauth-prep 103 | sudo_env gswauth-prep -K gswauthkey || fail "Unable to prep gswauth" 104 | 105 | mkdir functional_tests_result > /dev/null 2>&1 106 | nosetests -v --exe \ 107 | --with-xunit \ 108 | --xunit-file functional_tests_result/gluster-swift-gswauth-functional-TC-report.xml \ 109 | --with-html-output \ 110 | --html-out-file functional_tests_result/gluster-swift-gswauth-functional-result.html \ 111 | test/functional_auth/gswauth || fail "Functional gswauth test failed" 112 | 113 | run_generic_tests 114 | 115 | cleanup 116 | exit 0 117 | -------------------------------------------------------------------------------- /tools/keystone_functional_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (c) 2013 Red Hat, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 14 | # implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | # This program expects to be run by tox in a virtual python environment 19 | # so that it does not pollute the host development system 20 | 21 | sudo_env() 22 | { 23 | sudo bash -c "PATH=$PATH $*" 24 | } 25 | 26 | cleanup() 27 | { 28 | sudo service memcached stop 29 | sudo_env swift-init main stop 30 | sudo rm -rf /etc/swift > /dev/null 2>&1 31 | for acct in /mnt/gluster-object/* ; do 32 | sudo rm -rf /mnt/gluster-object/${acct}/* > /dev/null 2>&1 33 | sudo setfattr -x user.swift.metadata /mnt/gluster-object/${acct} > /dev/null 2>&1 34 | done 35 | } 36 | 37 | quit() 38 | { 39 | echo "$1" 40 | exit 1 41 | } 42 | 43 | 44 | fail() 45 | { 46 | cleanup 47 | quit "$1" 48 | } 49 | 50 | ### MAIN ### 51 | 52 | # Only run if there is no configuration in the system 53 | if [ -x /etc/swift ] ; then 54 | quit "/etc/swift exists, cannot run functional tests." 55 | fi 56 | 57 | # Check the directories exist 58 | DIRS="/mnt/gluster-object /mnt/gluster-object/test /mnt/gluster-object/test2 /mnt/gluster-object/gsmetadata" 59 | for d in $DIRS ; do 60 | if [ ! -x $d ] ; then 61 | quit "$d must exist on an XFS or GlusterFS volume" 62 | fi 63 | done 64 | 65 | # Check if keystone is running on this host 66 | if ! ps -ef | grep keystone | grep python > /dev/null 2>&1 ; then 67 | fail "Keystone is not running on localhost" 68 | fi 69 | 70 | export SWIFT_TEST_CONFIG_FILE=/etc/swift/test.conf 71 | 72 | # Install the configuration files 73 | sudo mkdir /etc/swift > /dev/null 2>&1 74 | sudo cp -r test/functional_auth/keystone/conf/* /etc/swift || fail "Unable to copy configuration files to /etc/swift" 75 | 76 | # Create the ring files according to any directories 77 | # in /mnt/gluster-object since the script can't 78 | # interrogate keystone to get the tenant-id's 79 | accounts="" 80 | for acct in /mnt/gluster-object/* ; do 81 | acct=`basename $acct` 82 | accounts="$acct $accounts" 83 | done 84 | sudo_env gluster-swift-gen-builders $accounts || fail "Unable to create ring files" 85 | 86 | # Start the services 87 | sudo service memcached start || fail "Unable to start memcached" 88 | sudo_env swift-init main start || fail "Unable to start swift" 89 | 90 | mkdir functional_tests_result > /dev/null 2>&1 91 | 92 | echo "== Keystone: Generic Functional Tests ==" 93 | 94 | nosetests -v --exe \ 95 | --with-xunit \ 96 | --xunit-file functional_tests_result/gluster-swift-keystone-generic-functional-TC-report.xml \ 97 | --with-html-output \ 98 | --html-out-file functional_tests_result/gluster-swift-keystone-generic-functional-result.html \ 99 | test/functional || fail "Functional tests failed" 100 | 101 | cleanup 102 | exit 0 103 | -------------------------------------------------------------------------------- /tools/object_expirer_functional.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (c) 2013 Red Hat, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 14 | # implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | # This program expects to be run by tox in a virtual python environment 19 | # so that it does not pollute the host development system 20 | 21 | sudo_env() 22 | { 23 | sudo bash -c "PATH=$PATH $*" 24 | } 25 | 26 | cleanup() 27 | { 28 | sudo service memcached stop 29 | sudo_env swift-init main stop 30 | sudo rm -rf /etc/swift > /dev/null 2>&1 31 | sudo rm -rf /mnt/gluster-object/test{,2}/* > /dev/null 2>&1 32 | sudo rm -rf /mnt/gluster-object/gsexpiring/* > /dev/null 2>&1 33 | sudo setfattr -x user.swift.metadata /mnt/gluster-object/test > /dev/null 2>&1 34 | sudo setfattr -x user.swift.metadata /mnt/gluster-object/gsexpiring > /dev/null 2>&1 35 | } 36 | 37 | quit() 38 | { 39 | echo "$1" 40 | exit 1 41 | } 42 | 43 | 44 | fail() 45 | { 46 | cleanup 47 | quit "$1" 48 | } 49 | 50 | ### MAIN ### 51 | 52 | # Only run if there is no configuration in the system 53 | if [ -x /etc/swift ] ; then 54 | quit "/etc/swift exists, cannot run functional tests." 55 | fi 56 | 57 | # Check the directories exist 58 | DIRS="/mnt/gluster-object /mnt/gluster-object/test /mnt/gluster-object/gsexpiring" 59 | for d in $DIRS ; do 60 | if [ ! -x $d ] ; then 61 | quit "$d must exist on an XFS or GlusterFS volume" 62 | fi 63 | done 64 | 65 | export SWIFT_TEST_CONFIG_FILE=/etc/swift/test.conf 66 | 67 | # Install the configuration files 68 | sudo mkdir /etc/swift > /dev/null 2>&1 69 | sudo cp -r test/functional_auth/common_conf/* /etc/swift || fail "Unable to copy configuration files to /etc/swift" 70 | sudo cp -r test/functional_auth/tempauth/conf/* /etc/swift || fail "Unable to copy configuration files to /etc/swift" 71 | sudo_env gluster-swift-gen-builders gsexpiring test || fail "Unable to create ring files" 72 | 73 | # Remove old files 74 | sudo rm -rf /mnt/gluster-object/test* > /dev/null 2>&1 75 | sudo rm -rf /mnt/gluster-object/gsexpiring/* > /dev/null 2>&1 76 | 77 | # Start the services 78 | sudo service memcached start || fail "Unable to start memcached" 79 | sudo_env swift-init main start || fail "Unable to start swift" 80 | 81 | echo "Running functional tests with tempauth" 82 | 83 | mkdir functional_tests > /dev/null 2>&1 84 | nosetests -v --exe \ 85 | test/object_expirer_functional || fail "Object expirer functional tests failed" 86 | 87 | cleanup 88 | exit 0 89 | -------------------------------------------------------------------------------- /tools/tempauth_functional_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (c) 2013 Red Hat, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 14 | # implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | # This program expects to be run by tox in a virtual python environment 19 | # so that it does not pollute the host development system 20 | 21 | sudo_env() 22 | { 23 | sudo bash -c "PATH=$PATH $*" 24 | } 25 | 26 | cleanup() 27 | { 28 | sudo service memcached stop 29 | sudo_env swift-init main stop 30 | sudo rm -rf /etc/swift > /dev/null 2>&1 31 | sudo rm -rf /mnt/gluster-object/test{,2}/* > /dev/null 2>&1 32 | sudo setfattr -x user.swift.metadata /mnt/gluster-object/test{,2} > /dev/null 2>&1 33 | } 34 | 35 | quit() 36 | { 37 | echo "$1" 38 | exit 1 39 | } 40 | 41 | 42 | fail() 43 | { 44 | cleanup 45 | quit "$1" 46 | } 47 | 48 | ### MAIN ### 49 | 50 | # Only run if there is no configuration in the system 51 | if [ -x /etc/swift ] ; then 52 | quit "/etc/swift exists, cannot run functional tests." 53 | fi 54 | 55 | # Check the directories exist 56 | DIRS="/mnt/gluster-object /mnt/gluster-object/test /mnt/gluster-object/test2" 57 | for d in $DIRS ; do 58 | if [ ! -x $d ] ; then 59 | quit "$d must exist on an XFS or GlusterFS volume" 60 | fi 61 | done 62 | 63 | export SWIFT_TEST_CONFIG_FILE=/etc/swift/test.conf 64 | 65 | # Install the configuration files 66 | sudo mkdir /etc/swift > /dev/null 2>&1 67 | sudo cp -r test/functional_auth/common_conf/* /etc/swift || fail "Unable to copy configuration files to /etc/swift" 68 | sudo cp -r test/functional_auth/tempauth/conf/* /etc/swift || fail "Unable to copy configuration files to /etc/swift" 69 | sudo_env gluster-swift-gen-builders test test2 || fail "Unable to create ring files" 70 | 71 | # Start the services 72 | sudo service memcached start || fail "Unable to start memcached" 73 | sudo_env swift-init main start || fail "Unable to start swift" 74 | 75 | echo "Running functional tests with tempauth" 76 | 77 | mkdir functional_tests > /dev/null 2>&1 78 | nosetests -v --exe \ 79 | --with-xunit \ 80 | --xunit-file functional_tests/gluster-swift-generic-functional-TC-report.xml \ 81 | --with-html-output \ 82 | --html-out-file functional_tests/gluster-swift-generic-functional-result.html \ 83 | test/functional || fail "Functional tests failed" 84 | 85 | cleanup 86 | exit 0 87 | -------------------------------------------------------------------------------- /tools/tox_run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (c) 2013 Red Hat, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 14 | # implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | # This program expects to be run by tox in a virtual python environment 19 | # so that it does not pollute the host development system 20 | 21 | sudo_env() 22 | { 23 | sudo bash -c "PATH=$PATH $*" 24 | } 25 | 26 | cleanup() 27 | { 28 | sudo service memcached stop 29 | sudo_env swift-init main stop 30 | sudo rm -rf /etc/swift > /dev/null 2>&1 31 | sudo rm -rf /mnt/gluster-object/test{,2}/* > /dev/null 2>&1 32 | sudo setfattr -x user.swift.metadata /mnt/gluster-object/test{,2} > /dev/null 2>&1 33 | gswauth_cleanup 34 | } 35 | 36 | gswauth_cleanup() 37 | { 38 | sudo rm -rf /mnt/gluster-object/gsmetadata/.* > /dev/null 2>&1 39 | sudo rm -rf /mnt/gluster-object/gsmetadata/* > /dev/null 2>&1 40 | sudo setfattr -x user.swift.metadata /mnt/gluster-object/gsmetadata > /dev/null 2>&1 41 | } 42 | 43 | quit() 44 | { 45 | echo "$1" 46 | exit 1 47 | } 48 | 49 | 50 | fail() 51 | { 52 | cleanup 53 | quit "$1" 54 | } 55 | 56 | ### MAIN ### 57 | 58 | # Only run if there is no configuration in the system 59 | if [ -x /etc/swift ] ; then 60 | quit "/etc/swift exists, cannot run functional tests." 61 | fi 62 | 63 | # Check the directories exist 64 | DIRS="/mnt/gluster-object /mnt/gluster-object/test /mnt/gluster-object/test2 /mnt/gluster-object/gsmetadata" 65 | for d in $DIRS ; do 66 | if [ ! -x $d ] ; then 67 | quit "$d must exist on an XFS or GlusterFS volume" 68 | fi 69 | done 70 | 71 | export SWIFT_TEST_CONFIG_FILE=/etc/swift/test.conf 72 | 73 | # Install the configuration files 74 | sudo mkdir /etc/swift > /dev/null 2>&1 75 | sudo cp -r test/functional_auth/gswauth/conf/* /etc/swift || fail "Unable to copy configuration files to /etc/swift" 76 | sudo_env gluster-swift-gen-builders test test2 gsmetadata || fail "Unable to create ring files" 77 | 78 | # Start the services 79 | sudo service memcached start || fail "Unable to start memcached" 80 | sudo_env swift-init main start || fail "Unable to start swift" 81 | 82 | #swauth-prep 83 | sudo_env gswauth-prep -K gswauthkey || fail "Unable to prep gswauth" 84 | sudo_env gswauth-add-user -K gswauthkey -a test tester testing || fail "Unable to add user test" 85 | sudo_env gswauth-add-user -K gswauthkey -a test2 tester2 testing2 || fail "Unable to add user test2" 86 | sudo_env gswauth-add-user -K gswauthkey test tester3 testing3 || fail "Unable to add user test3" 87 | 88 | # open the shell to the user 89 | bash --norc -i 90 | cleanup 91 | exit 0 92 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py26,py27,pep8,functest 3 | minversion = 1.6 4 | skipsdist = True 5 | 6 | [testenv] 7 | usedevelop = True 8 | install_command = pip install --allow-external netifaces --allow-insecure netifaces -U {opts} {packages} 9 | whitelist_externals=bash 10 | setenv = VIRTUAL_ENV={envdir} 11 | NOSE_WITH_COVERAGE=1 12 | NOSE_COVER_BRANCHES=1 13 | NOSE_COVER_PACKAGE=gluster 14 | deps = 15 | git+https://github.com/openstack/swift.git@2.15.1 16 | -r{toxinidir}/requirements.txt 17 | -r{toxinidir}/test-requirements.txt 18 | # Just having testtools package installed fixes some dependency issue 19 | # https://github.com/swiftstack/vagrant-swift-all-in-one/issues/19 20 | # pip needs a good dependency resolver :/ 21 | # testtools 22 | # flake8 23 | changedir = {toxinidir}/test/unit 24 | commands = nosetests -v {posargs} 25 | passenv = SWIFT_* *_proxy 26 | 27 | [testenv:cover] 28 | setenv = VIRTUAL_ENV={envdir} 29 | NOSE_WITH_COVERAGE=1 30 | NOSE_COVER_BRANCHES=1 31 | NOSE_COVER_HTML=1 32 | NOSE_COVER_HTML_DIR={toxinidir}/cover 33 | 34 | [tox:jenkins] 35 | downloadcache = ~/cache/pip 36 | 37 | [testenv:functest] 38 | changedir = {toxinidir} 39 | commands = bash ./.functests 40 | 41 | [testenv:ksfunctest] 42 | changedir = {toxinidir} 43 | commands = bash tools/keystone_functional_tests.sh 44 | 45 | [testenv:pep8] 46 | changedir = {toxinidir} 47 | commands = 48 | flake8 gluster test setup.py 49 | 50 | [testenv:venv] 51 | changedir = {toxinidir} 52 | commands = {posargs} 53 | 54 | [testenv:run] 55 | changedir = {toxinidir} 56 | commands = bash tools/tox_run.sh 57 | 58 | [flake8] 59 | # it's not a bug that we aren't using all of hacking 60 | # H102 -> apache2 license exists 61 | # H103 -> license is apache 62 | # H201 -> no bare excepts (unless marked with " # noqa") 63 | # H231 -> Check for except statements to be Python 3.x compatible 64 | # H501 -> don't use locals() for str formatting 65 | # H903 -> \n not \r\n 66 | ignore = H 67 | select = F,E,W,H102,H103,H201,H231,H501,H903 68 | exclude = .venv,.tox,dist,doc,*egg,test 69 | show-source = True 70 | --------------------------------------------------------------------------------