diff options
Diffstat (limited to 'test/functional')
| -rw-r--r-- | test/functional/__init__.py | 959 | ||||
| -rw-r--r-- | test/functional/gluster_swift_tests.py | 3 | ||||
| -rw-r--r-- | test/functional/swift_test_client.py | 246 | ||||
| -rwxr-xr-x | test/functional/test_account.py | 123 | ||||
| -rwxr-xr-x | test/functional/test_container.py | 549 | ||||
| -rwxr-xr-x | test/functional/test_object.py | 353 | ||||
| -rw-r--r-- | test/functional/tests.py | 1331 |
7 files changed, 3182 insertions, 382 deletions
diff --git a/test/functional/__init__.py b/test/functional/__init__.py index e69de29..580de56 100644 --- a/test/functional/__init__.py +++ b/test/functional/__init__.py @@ -0,0 +1,959 @@ +# Copyright (c) 2014 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import httplib +import mock +import os +import sys +import pickle +import socket +import locale +import eventlet +import eventlet.debug +import functools +import random +from ConfigParser import ConfigParser, NoSectionError +from time import time, sleep +from httplib import HTTPException +from urlparse import urlparse +from nose import SkipTest +from contextlib import closing +from gzip import GzipFile +from shutil import rmtree +from tempfile import mkdtemp +from swift.common.middleware.memcache import MemcacheMiddleware +from swift.common.storage_policy import parse_storage_policies, PolicyError + +from test import get_config +from test.functional.swift_test_client import Account, Connection, \ + ResponseError +# This has the side effect of mocking out the xattr module so that unit tests +# (and in this case, when in-process functional tests are called for) can run +# on file systems that don't support extended attributes. +from test.unit import debug_logger, FakeMemcache + +from swift.common import constraints, utils, ring, storage_policy +from swift.common.ring import Ring +from swift.common.wsgi import monkey_patch_mimetools, loadapp +from swift.common.utils import config_true_value +from swift.account import server as account_server +from swift.container import server as container_server +from swift.obj import server as object_server, mem_server as mem_object_server +import swift.proxy.controllers.obj + +httplib._MAXHEADERS = constraints.MAX_HEADER_COUNT +DEBUG = True + +# In order to get the proper blocking behavior of sockets without using +# threads, where we can set an arbitrary timeout for some piece of code under +# test, we use eventlet with the standard socket library patched. We have to +# perform this setup at module import time, since all the socket module +# bindings in the swiftclient code will have been made by the time nose +# invokes the package or class setup methods. +eventlet.hubs.use_hub(utils.get_hub()) +eventlet.patcher.monkey_patch(all=False, socket=True) +eventlet.debug.hub_exceptions(False) + +from swiftclient import get_auth, http_connection + +has_insecure = False +try: + from swiftclient import __version__ as client_version + # Prevent a ValueError in StrictVersion with '2.0.3.68.ga99c2ff' + client_version = '.'.join(client_version.split('.')[:3]) +except ImportError: + # Pre-PBR we had version, not __version__. Anyhow... + client_version = '1.2' +from distutils.version import StrictVersion +if StrictVersion(client_version) >= StrictVersion('2.0'): + has_insecure = True + + +config = {} +web_front_end = None +normalized_urls = None + +# If no config was read, we will fall back to old school env vars +swift_test_auth_version = None +swift_test_auth = os.environ.get('SWIFT_TEST_AUTH') +swift_test_user = [os.environ.get('SWIFT_TEST_USER'), None, None, '', ''] +swift_test_key = [os.environ.get('SWIFT_TEST_KEY'), None, None, '', ''] +swift_test_tenant = ['', '', '', '', ''] +swift_test_perm = ['', '', '', '', ''] +swift_test_domain = ['', '', '', '', ''] +swift_test_user_id = ['', '', '', '', ''] +swift_test_tenant_id = ['', '', '', '', ''] + +skip, skip2, skip3, skip_service_tokens = False, False, False, False + +orig_collate = '' +insecure = False + +orig_hash_path_suff_pref = ('', '') +orig_swift_conf_name = None + +in_process = False +_testdir = _test_servers = _test_coros = None + + +class FakeMemcacheMiddleware(MemcacheMiddleware): + """ + Caching middleware that fakes out caching in swift if memcached + does not appear to be running. + """ + + def __init__(self, app, conf): + super(FakeMemcacheMiddleware, self).__init__(app, conf) + self.memcache = FakeMemcache() + + +class InProcessException(BaseException): + pass + + +def _info(msg): + print >> sys.stderr, msg + + +def _debug(msg): + if DEBUG: + _info('DEBUG: ' + msg) + + +def _in_process_setup_swift_conf(swift_conf_src, testdir): + # override swift.conf contents for in-process functional test runs + conf = ConfigParser() + conf.read(swift_conf_src) + try: + section = 'swift-hash' + conf.set(section, 'swift_hash_path_suffix', 'inprocfunctests') + conf.set(section, 'swift_hash_path_prefix', 'inprocfunctests') + section = 'swift-constraints' + max_file_size = (8 * 1024 * 1024) + 2 # 8 MB + 2 + conf.set(section, 'max_file_size', max_file_size) + except NoSectionError: + msg = 'Conf file %s is missing section %s' % (swift_conf_src, section) + raise InProcessException(msg) + + test_conf_file = os.path.join(testdir, 'swift.conf') + with open(test_conf_file, 'w') as fp: + conf.write(fp) + + return test_conf_file + + +def _in_process_find_conf_file(conf_src_dir, conf_file_name, use_sample=True): + """ + Look for a file first in conf_src_dir, if it exists, otherwise optionally + look in the source tree sample 'etc' dir. + + :param conf_src_dir: Directory in which to search first for conf file. May + be None + :param conf_file_name: Name of conf file + :param use_sample: If True and the conf_file_name is not found, then return + any sample conf file found in the source tree sample + 'etc' dir by appending '-sample' to conf_file_name + :returns: Path to conf file + :raises InProcessException: If no conf file is found + """ + dflt_src_dir = os.path.normpath(os.path.join(os.path.abspath(__file__), + os.pardir, os.pardir, os.pardir, + 'etc')) + conf_src_dir = dflt_src_dir if conf_src_dir is None else conf_src_dir + conf_file_path = os.path.join(conf_src_dir, conf_file_name) + if os.path.exists(conf_file_path): + return conf_file_path + + if use_sample: + # fall back to using the corresponding sample conf file + conf_file_name += '-sample' + conf_file_path = os.path.join(dflt_src_dir, conf_file_name) + if os.path.exists(conf_file_path): + return conf_file_path + + msg = 'Failed to find config file %s' % conf_file_name + raise InProcessException(msg) + + +def _in_process_setup_ring(swift_conf, conf_src_dir, testdir): + """ + If SWIFT_TEST_POLICY is set: + - look in swift.conf file for specified policy + - move this to be policy-0 but preserving its options + - copy its ring file to test dir, changing its devices to suit + in process testing, and renaming it to suit policy-0 + Otherwise, create a default ring file. + """ + conf = ConfigParser() + conf.read(swift_conf) + sp_prefix = 'storage-policy:' + + try: + # policy index 0 will be created if no policy exists in conf + policies = parse_storage_policies(conf) + except PolicyError as e: + raise InProcessException(e) + + # clear all policies from test swift.conf before adding test policy back + for policy in policies: + conf.remove_section(sp_prefix + str(policy.idx)) + + policy_specified = os.environ.get('SWIFT_TEST_POLICY') + if policy_specified: + policy_to_test = policies.get_by_name(policy_specified) + if policy_to_test is None: + raise InProcessException('Failed to find policy name "%s"' + % policy_specified) + _info('Using specified policy %s' % policy_to_test.name) + else: + policy_to_test = policies.default + _info('Defaulting to policy %s' % policy_to_test.name) + + # make policy_to_test be policy index 0 and default for the test config + sp_zero_section = sp_prefix + '0' + conf.add_section(sp_zero_section) + for (k, v) in policy_to_test.get_info(config=True).items(): + conf.set(sp_zero_section, k, v) + conf.set(sp_zero_section, 'default', True) + + with open(swift_conf, 'w') as fp: + conf.write(fp) + + # look for a source ring file + ring_file_src = ring_file_test = 'object.ring.gz' + if policy_to_test.idx: + ring_file_src = 'object-%s.ring.gz' % policy_to_test.idx + try: + ring_file_src = _in_process_find_conf_file(conf_src_dir, ring_file_src, + use_sample=False) + except InProcessException as e: + if policy_specified: + raise InProcessException('Failed to find ring file %s' + % ring_file_src) + ring_file_src = None + + ring_file_test = os.path.join(testdir, ring_file_test) + if ring_file_src: + # copy source ring file to a policy-0 test ring file, re-homing servers + _info('Using source ring file %s' % ring_file_src) + ring_data = ring.RingData.load(ring_file_src) + obj_sockets = [] + for dev in ring_data.devs: + device = 'sd%c1' % chr(len(obj_sockets) + ord('a')) + utils.mkdirs(os.path.join(_testdir, 'sda1')) + utils.mkdirs(os.path.join(_testdir, 'sda1', 'tmp')) + obj_socket = eventlet.listen(('localhost', 0)) + obj_sockets.append(obj_socket) + dev['port'] = obj_socket.getsockname()[1] + dev['ip'] = '127.0.0.1' + dev['device'] = device + dev['replication_port'] = dev['port'] + dev['replication_ip'] = dev['ip'] + ring_data.save(ring_file_test) + else: + # make default test ring, 2 replicas, 4 partitions, 2 devices + _info('No source object ring file, creating 2rep/4part/2dev ring') + obj_sockets = [eventlet.listen(('localhost', 0)) for _ in (0, 1)] + ring_data = ring.RingData( + [[0, 1, 0, 1], [1, 0, 1, 0]], + [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', + 'port': obj_sockets[0].getsockname()[1]}, + {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', + 'port': obj_sockets[1].getsockname()[1]}], + 30) + with closing(GzipFile(ring_file_test, 'wb')) as f: + pickle.dump(ring_data, f) + + for dev in ring_data.devs: + _debug('Ring file dev: %s' % dev) + + return obj_sockets + + +def in_process_setup(the_object_server=object_server): + _info('IN-PROCESS SERVERS IN USE FOR FUNCTIONAL TESTS') + _info('Using object_server class: %s' % the_object_server.__name__) + conf_src_dir = os.environ.get('SWIFT_TEST_IN_PROCESS_CONF_DIR') + + if conf_src_dir is not None: + if not os.path.isdir(conf_src_dir): + msg = 'Config source %s is not a dir' % conf_src_dir + raise InProcessException(msg) + _info('Using config source dir: %s' % conf_src_dir) + + # If SWIFT_TEST_IN_PROCESS_CONF specifies a config source dir then + # prefer config files from there, otherwise read config from source tree + # sample files. A mixture of files from the two sources is allowed. + proxy_conf = _in_process_find_conf_file(conf_src_dir, 'proxy-server.conf') + _info('Using proxy config from %s' % proxy_conf) + swift_conf_src = _in_process_find_conf_file(conf_src_dir, 'swift.conf') + _info('Using swift config from %s' % swift_conf_src) + + monkey_patch_mimetools() + + global _testdir + _testdir = os.path.join(mkdtemp(), 'tmp_functional') + utils.mkdirs(_testdir) + rmtree(_testdir) + utils.mkdirs(os.path.join(_testdir, 'sda1')) + utils.mkdirs(os.path.join(_testdir, 'sda1', 'tmp')) + utils.mkdirs(os.path.join(_testdir, 'sdb1')) + utils.mkdirs(os.path.join(_testdir, 'sdb1', 'tmp')) + + swift_conf = _in_process_setup_swift_conf(swift_conf_src, _testdir) + obj_sockets = _in_process_setup_ring(swift_conf, conf_src_dir, _testdir) + + global orig_swift_conf_name + orig_swift_conf_name = utils.SWIFT_CONF_FILE + utils.SWIFT_CONF_FILE = swift_conf + constraints.reload_constraints() + storage_policy.SWIFT_CONF_FILE = swift_conf + storage_policy.reload_storage_policies() + global config + if constraints.SWIFT_CONSTRAINTS_LOADED: + # Use the swift constraints that are loaded for the test framework + # configuration + _c = dict((k, str(v)) + for k, v in constraints.EFFECTIVE_CONSTRAINTS.items()) + config.update(_c) + else: + # In-process swift constraints were not loaded, somethings wrong + raise SkipTest + global orig_hash_path_suff_pref + orig_hash_path_suff_pref = utils.HASH_PATH_PREFIX, utils.HASH_PATH_SUFFIX + utils.validate_hash_conf() + + # We create the proxy server listening socket to get its port number so + # that we can add it as the "auth_port" value for the functional test + # clients. + prolis = eventlet.listen(('localhost', 0)) + + # The following set of configuration values is used both for the + # functional test frame work and for the various proxy, account, container + # and object servers. + config.update({ + # Values needed by the various in-process swift servers + 'devices': _testdir, + 'swift_dir': _testdir, + 'mount_check': 'false', + 'client_timeout': '4', + 'allow_account_management': 'true', + 'account_autocreate': 'true', + 'allow_versions': 'True', + # Below are values used by the functional test framework, as well as + # by the various in-process swift servers + 'auth_host': '127.0.0.1', + 'auth_port': str(prolis.getsockname()[1]), + 'auth_ssl': 'no', + 'auth_prefix': '/auth/', + # Primary functional test account (needs admin access to the + # account) + 'account': 'test', + 'username': 'tester', + 'password': 'testing', + # User on a second account (needs admin access to the account) + 'account2': 'test2', + 'username2': 'tester2', + 'password2': 'testing2', + # User on same account as first, but without admin access + 'username3': 'tester3', + 'password3': 'testing3', + # Service user and prefix (emulates glance, cinder, etc. user) + 'account5': 'test5', + 'username5': 'tester5', + 'password5': 'testing5', + 'service_prefix': 'SERVICE', + # For tempauth middleware. Update reseller_prefix + 'reseller_prefix': 'AUTH, SERVICE', + 'SERVICE_require_group': 'service' + }) + + acc1lis = eventlet.listen(('localhost', 0)) + acc2lis = eventlet.listen(('localhost', 0)) + con1lis = eventlet.listen(('localhost', 0)) + con2lis = eventlet.listen(('localhost', 0)) + + account_ring_path = os.path.join(_testdir, 'account.ring.gz') + with closing(GzipFile(account_ring_path, 'wb')) as f: + pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], + [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', + 'port': acc1lis.getsockname()[1]}, + {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', + 'port': acc2lis.getsockname()[1]}], 30), + f) + container_ring_path = os.path.join(_testdir, 'container.ring.gz') + with closing(GzipFile(container_ring_path, 'wb')) as f: + pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], + [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', + 'port': con1lis.getsockname()[1]}, + {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', + 'port': con2lis.getsockname()[1]}], 30), + f) + + eventlet.wsgi.HttpProtocol.default_request_version = "HTTP/1.0" + # Turn off logging requests by the underlying WSGI software. + eventlet.wsgi.HttpProtocol.log_request = lambda *a: None + logger = utils.get_logger(config, 'wsgi-server', log_route='wsgi') + # Redirect logging other messages by the underlying WSGI software. + eventlet.wsgi.HttpProtocol.log_message = \ + lambda s, f, *a: logger.error('ERROR WSGI: ' + f % a) + # Default to only 4 seconds for in-process functional test runs + eventlet.wsgi.WRITE_TIMEOUT = 4 + + acc1srv = account_server.AccountController( + config, logger=debug_logger('acct1')) + acc2srv = account_server.AccountController( + config, logger=debug_logger('acct2')) + con1srv = container_server.ContainerController( + config, logger=debug_logger('cont1')) + con2srv = container_server.ContainerController( + config, logger=debug_logger('cont2')) + + objsrvs = [ + (obj_sockets[index], + the_object_server.ObjectController( + config, logger=debug_logger('obj%d' % (index + 1)))) + for index in range(len(obj_sockets)) + ] + + logger = debug_logger('proxy') + + def get_logger(name, *args, **kwargs): + return logger + + with mock.patch('swift.common.utils.get_logger', get_logger): + with mock.patch('swift.common.middleware.memcache.MemcacheMiddleware', + FakeMemcacheMiddleware): + try: + app = loadapp(proxy_conf, global_conf=config) + except Exception as e: + raise InProcessException(e) + + nl = utils.NullLogger() + prospa = eventlet.spawn(eventlet.wsgi.server, prolis, app, nl) + acc1spa = eventlet.spawn(eventlet.wsgi.server, acc1lis, acc1srv, nl) + acc2spa = eventlet.spawn(eventlet.wsgi.server, acc2lis, acc2srv, nl) + con1spa = eventlet.spawn(eventlet.wsgi.server, con1lis, con1srv, nl) + con2spa = eventlet.spawn(eventlet.wsgi.server, con2lis, con2srv, nl) + + objspa = [eventlet.spawn(eventlet.wsgi.server, objsrv[0], objsrv[1], nl) + for objsrv in objsrvs] + + global _test_coros + _test_coros = \ + (prospa, acc1spa, acc2spa, con1spa, con2spa) + tuple(objspa) + + # Create accounts "test" and "test2" + def create_account(act): + ts = utils.normalize_timestamp(time()) + account_ring = Ring(_testdir, ring_name='account') + partition, nodes = account_ring.get_nodes(act) + for node in nodes: + # Note: we are just using the http_connect method in the object + # controller here to talk to the account server nodes. + conn = swift.proxy.controllers.obj.http_connect( + node['ip'], node['port'], node['device'], partition, 'PUT', + '/' + act, {'X-Timestamp': ts, 'x-trans-id': act}) + resp = conn.getresponse() + assert(resp.status == 201) + + create_account('AUTH_test') + create_account('AUTH_test2') + +cluster_info = {} + + +def get_cluster_info(): + # The fallback constraints used for testing will come from the current + # effective constraints. + eff_constraints = dict(constraints.EFFECTIVE_CONSTRAINTS) + + # We'll update those constraints based on what the /info API provides, if + # anything. + global cluster_info + try: + conn = Connection(config) + conn.authenticate() + cluster_info.update(conn.cluster_info()) + except (ResponseError, socket.error): + # Failed to get cluster_information via /info API, so fall back on + # test.conf data + pass + else: + try: + eff_constraints.update(cluster_info['swift']) + except KeyError: + # Most likely the swift cluster has "expose_info = false" set + # in its proxy-server.conf file, so we'll just do the best we + # can. + print >>sys.stderr, "** Swift Cluster not exposing /info **" + + # Finally, we'll allow any constraint present in the swift-constraints + # section of test.conf to override everything. Note that only those + # constraints defined in the constraints module are converted to integers. + test_constraints = get_config('swift-constraints') + for k in constraints.DEFAULT_CONSTRAINTS: + try: + test_constraints[k] = int(test_constraints[k]) + except KeyError: + pass + except ValueError: + print >>sys.stderr, "Invalid constraint value: %s = %s" % ( + k, test_constraints[k]) + eff_constraints.update(test_constraints) + + # Just make it look like these constraints were loaded from a /info call, + # even if the /info call failed, or when they are overridden by values + # from the swift-constraints section of test.conf + cluster_info['swift'] = eff_constraints + + +def setup_package(): + in_process_env = os.environ.get('SWIFT_TEST_IN_PROCESS') + if in_process_env is not None: + use_in_process = utils.config_true_value(in_process_env) + else: + use_in_process = None + + global in_process + + if use_in_process: + # Explicitly set to True, so barrel on ahead with in-process + # functional test setup. + in_process = True + # NOTE: No attempt is made to a read local test.conf file. + else: + if use_in_process is None: + # Not explicitly set, default to using in-process functional tests + # if the test.conf file is not found, or does not provide a usable + # configuration. + config.update(get_config('func_test')) + if config: + in_process = False + else: + in_process = True + else: + # Explicitly set to False, do not attempt to use in-process + # functional tests, be sure we attempt to read from local + # test.conf file. + in_process = False + config.update(get_config('func_test')) + + if in_process: + in_mem_obj_env = os.environ.get('SWIFT_TEST_IN_MEMORY_OBJ') + in_mem_obj = utils.config_true_value(in_mem_obj_env) + try: + in_process_setup(the_object_server=( + mem_object_server if in_mem_obj else object_server)) + except InProcessException as exc: + print >> sys.stderr, ('Exception during in-process setup: %s' + % str(exc)) + raise + + global web_front_end + web_front_end = config.get('web_front_end', 'integral') + global normalized_urls + normalized_urls = config.get('normalized_urls', False) + + global orig_collate + orig_collate = locale.setlocale(locale.LC_COLLATE) + locale.setlocale(locale.LC_COLLATE, config.get('collate', 'C')) + + global insecure + insecure = config_true_value(config.get('insecure', False)) + + global swift_test_auth_version + global swift_test_auth + global swift_test_user + global swift_test_key + global swift_test_tenant + global swift_test_perm + global swift_test_domain + global swift_test_service_prefix + + swift_test_service_prefix = None + + if config: + swift_test_auth_version = str(config.get('auth_version', '1')) + + swift_test_auth = 'http' + if config_true_value(config.get('auth_ssl', 'no')): + swift_test_auth = 'https' + if 'auth_prefix' not in config: + config['auth_prefix'] = '/' + try: + suffix = '://%(auth_host)s:%(auth_port)s%(auth_prefix)s' % config + swift_test_auth += suffix + except KeyError: + pass # skip + + if 'service_prefix' in config: + swift_test_service_prefix = utils.append_underscore( + config['service_prefix']) + + if swift_test_auth_version == "1": + swift_test_auth += 'v1.0' + + try: + if 'account' in config: + swift_test_user[0] = '%(account)s:%(username)s' % config + else: + swift_test_user[0] = '%(username)s' % config + swift_test_key[0] = config['password'] + except KeyError: + # bad config, no account/username configured, tests cannot be + # run + pass + try: + swift_test_user[1] = '%s%s' % ( + '%s:' % config['account2'] if 'account2' in config else '', + config['username2']) + swift_test_key[1] = config['password2'] + except KeyError: + pass # old config, no second account tests can be run + try: + swift_test_user[2] = '%s%s' % ( + '%s:' % config['account'] if 'account' + in config else '', config['username3']) + swift_test_key[2] = config['password3'] + except KeyError: + pass # old config, no third account tests can be run + try: + swift_test_user[4] = '%s%s' % ( + '%s:' % config['account5'], config['username5']) + swift_test_key[4] = config['password5'] + swift_test_tenant[4] = config['account5'] + except KeyError: + pass # no service token tests can be run + + for _ in range(3): + swift_test_perm[_] = swift_test_user[_] + + else: + swift_test_user[0] = config['username'] + swift_test_tenant[0] = config['account'] + swift_test_key[0] = config['password'] + swift_test_user[1] = config['username2'] + swift_test_tenant[1] = config['account2'] + swift_test_key[1] = config['password2'] + swift_test_user[2] = config['username3'] + swift_test_tenant[2] = config['account'] + swift_test_key[2] = config['password3'] + if 'username4' in config: + swift_test_user[3] = config['username4'] + swift_test_tenant[3] = config['account4'] + swift_test_key[3] = config['password4'] + swift_test_domain[3] = config['domain4'] + if 'username5' in config: + swift_test_user[4] = config['username5'] + swift_test_tenant[4] = config['account5'] + swift_test_key[4] = config['password5'] + + for _ in range(5): + swift_test_perm[_] = swift_test_tenant[_] + ':' \ + + swift_test_user[_] + + global skip + skip = not all([swift_test_auth, swift_test_user[0], swift_test_key[0]]) + if skip: + print >>sys.stderr, 'SKIPPING FUNCTIONAL TESTS DUE TO NO CONFIG' + + global skip2 + skip2 = not all([not skip, swift_test_user[1], swift_test_key[1]]) + if not skip and skip2: + print >>sys.stderr, \ + 'SKIPPING SECOND ACCOUNT FUNCTIONAL TESTS' \ + ' DUE TO NO CONFIG FOR THEM' + + global skip3 + skip3 = not all([not skip, swift_test_user[2], swift_test_key[2]]) + if not skip and skip3: + print >>sys.stderr, \ + 'SKIPPING THIRD ACCOUNT FUNCTIONAL TESTS DUE TO NO CONFIG FOR THEM' + + global skip_if_not_v3 + skip_if_not_v3 = (swift_test_auth_version != '3' + or not all([not skip, + swift_test_user[3], + swift_test_key[3]])) + if not skip and skip_if_not_v3: + print >>sys.stderr, \ + 'SKIPPING FUNCTIONAL TESTS SPECIFIC TO AUTH VERSION 3' + + global skip_service_tokens + skip_service_tokens = not all([not skip, swift_test_user[4], + swift_test_key[4], swift_test_tenant[4], + swift_test_service_prefix]) + if not skip and skip_service_tokens: + print >>sys.stderr, \ + 'SKIPPING FUNCTIONAL TESTS SPECIFIC TO SERVICE TOKENS' + + get_cluster_info() + + +def teardown_package(): + global orig_collate + locale.setlocale(locale.LC_COLLATE, orig_collate) + + # clean up containers and objects left behind after running tests + conn = Connection(config) + conn.authenticate() + account = Account(conn, config.get('account', config['username'])) + account.delete_containers() + + global in_process + if in_process: + try: + for server in _test_coros: + server.kill() + except Exception: + pass + try: + rmtree(os.path.dirname(_testdir)) + except Exception: + pass + utils.HASH_PATH_PREFIX, utils.HASH_PATH_SUFFIX = \ + orig_hash_path_suff_pref + utils.SWIFT_CONF_FILE = orig_swift_conf_name + constraints.reload_constraints() + + +class AuthError(Exception): + pass + + +class InternalServerError(Exception): + pass + + +url = [None, None, None, None, None] +token = [None, None, None, None, None] +service_token = [None, None, None, None, None] +parsed = [None, None, None, None, None] +conn = [None, None, None, None, None] + + +def connection(url): + if has_insecure: + return http_connection(url, insecure=insecure) + return http_connection(url) + + +def get_url_token(user_index, os_options): + authargs = dict(snet=False, + tenant_name=swift_test_tenant[user_index], + auth_version=swift_test_auth_version, + os_options=os_options, + insecure=insecure) + return get_auth(swift_test_auth, + swift_test_user[user_index], + swift_test_key[user_index], + **authargs) + + +def retry(func, *args, **kwargs): + """ + You can use the kwargs to override: + 'retries' (default: 5) + 'use_account' (default: 1) - which user's token to pass + 'url_account' (default: matches 'use_account') - which user's storage URL + 'resource' (default: url[url_account] - URL to connect to; retry() + will interpolate the variable :storage_url: if present + 'service_user' - add a service token from this user (1 indexed) + """ + global url, token, service_token, parsed, conn + retries = kwargs.get('retries', 5) + attempts, backoff = 0, 1 + + # use account #1 by default; turn user's 1-indexed account into 0-indexed + use_account = kwargs.pop('use_account', 1) - 1 + service_user = kwargs.pop('service_user', None) + if service_user: + service_user -= 1 # 0-index + + # access our own account by default + url_account = kwargs.pop('url_account', use_account + 1) - 1 + os_options = {'user_domain_name': swift_test_domain[use_account], + 'project_domain_name': swift_test_domain[use_account]} + while attempts <= retries: + auth_failure = False + attempts += 1 + try: + if not url[use_account] or not token[use_account]: + url[use_account], token[use_account] = get_url_token( + use_account, os_options) + parsed[use_account] = conn[use_account] = None + if not parsed[use_account] or not conn[use_account]: + parsed[use_account], conn[use_account] = \ + connection(url[use_account]) + + # default resource is the account url[url_account] + resource = kwargs.pop('resource', '%(storage_url)s') + template_vars = {'storage_url': url[url_account]} + parsed_result = urlparse(resource % template_vars) + if isinstance(service_user, int): + if not service_token[service_user]: + dummy, service_token[service_user] = get_url_token( + service_user, os_options) + kwargs['service_token'] = service_token[service_user] + return func(url[url_account], token[use_account], + parsed_result, conn[url_account], + *args, **kwargs) + except (socket.error, HTTPException): + if attempts > retries: + raise + parsed[use_account] = conn[use_account] = None + if service_user: + service_token[service_user] = None + except AuthError: + auth_failure = True + url[use_account] = token[use_account] = None + if service_user: + service_token[service_user] = None + except InternalServerError: + pass + if attempts <= retries: + if not auth_failure: + sleep(backoff) + backoff *= 2 + raise Exception('No result after %s retries.' % retries) + + +def check_response(conn): + resp = conn.getresponse() + if resp.status == 401: + resp.read() + raise AuthError() + elif resp.status // 100 == 5: + resp.read() + raise InternalServerError() + return resp + + +def load_constraint(name): + global cluster_info + try: + c = cluster_info['swift'][name] + except KeyError: + raise SkipTest("Missing constraint: %s" % name) + if not isinstance(c, int): + raise SkipTest("Bad value, %r, for constraint: %s" % (c, name)) + return c + + +def get_storage_policy_from_cluster_info(info): + policies = info['swift'].get('policies', {}) + default_policy = [] + non_default_policies = [] + for p in policies: + if p.get('default', {}): + default_policy.append(p) + else: + non_default_policies.append(p) + return default_policy, non_default_policies + + +def reset_acl(): + def post(url, token, parsed, conn): + conn.request('POST', parsed.path, '', { + 'X-Auth-Token': token, + 'X-Account-Access-Control': '{}' + }) + return check_response(conn) + resp = retry(post, use_account=1) + resp.read() + + +def requires_acls(f): + @functools.wraps(f) + def wrapper(*args, **kwargs): + global skip, cluster_info + if skip or not cluster_info: + raise SkipTest('Requires account ACLs') + # Determine whether this cluster has account ACLs; if not, skip test + if not cluster_info.get('tempauth', {}).get('account_acls'): + raise SkipTest('Requires account ACLs') + if swift_test_auth_version != '1': + # remove when keystoneauth supports account acls + raise SkipTest('Requires account ACLs') + reset_acl() + try: + rv = f(*args, **kwargs) + finally: + reset_acl() + return rv + return wrapper + + +class FunctionalStoragePolicyCollection(object): + + def __init__(self, policies): + self._all = policies + self.default = None + for p in self: + if p.get('default', False): + assert self.default is None, 'Found multiple default ' \ + 'policies %r and %r' % (self.default, p) + self.default = p + + @classmethod + def from_info(cls, info=None): + if not (info or cluster_info): + get_cluster_info() + info = info or cluster_info + try: + policy_info = info['swift']['policies'] + except KeyError: + raise AssertionError('Did not find any policy info in %r' % info) + policies = cls(policy_info) + assert policies.default, \ + 'Did not find default policy in %r' % policy_info + return policies + + def __len__(self): + return len(self._all) + + def __iter__(self): + return iter(self._all) + + def __getitem__(self, index): + return self._all[index] + + def filter(self, **kwargs): + return self.__class__([p for p in self if all( + p.get(k) == v for k, v in kwargs.items())]) + + def exclude(self, **kwargs): + return self.__class__([p for p in self if all( + p.get(k) != v for k, v in kwargs.items())]) + + def select(self): + return random.choice(self) + + +def requires_policies(f): + @functools.wraps(f) + def wrapper(self, *args, **kwargs): + if skip: + raise SkipTest + try: + self.policies = FunctionalStoragePolicyCollection.from_info() + except AssertionError: + raise SkipTest("Unable to determine available policies") + if len(self.policies) < 2: + raise SkipTest("Multiple policies not enabled") + return f(self, *args, **kwargs) + + return wrapper diff --git a/test/functional/gluster_swift_tests.py b/test/functional/gluster_swift_tests.py index b4514c9..2ffb841 100644 --- a/test/functional/gluster_swift_tests.py +++ b/test/functional/gluster_swift_tests.py @@ -19,8 +19,9 @@ import random import os,sys,re,hashlib from nose import SkipTest -from test.functional.tests import config, locale, Base, Base2, Utils, \ +from test.functional.tests import Base, Base2, Utils, \ TestFileEnv +from test.functional import config, locale from test.functional.swift_test_client import Account, Connection, File, \ ResponseError diff --git a/test/functional/swift_test_client.py b/test/functional/swift_test_client.py index 27e025b..5c0ab87 100644 --- a/test/functional/swift_test_client.py +++ b/test/functional/swift_test_client.py @@ -26,10 +26,16 @@ import simplejson as json from nose import SkipTest from xml.dom import minidom + from swiftclient import get_auth +from swift.common import constraints +from swift.common.utils import config_true_value + from test import safe_repr +httplib._MAXHEADERS = constraints.MAX_HEADER_COUNT + class AuthenticationFailed(Exception): pass @@ -103,11 +109,13 @@ class Connection(object): def __init__(self, config): for key in 'auth_host auth_port auth_ssl username password'.split(): if key not in config: - raise SkipTest + raise SkipTest( + "Missing required configuration parameter: %s" % key) self.auth_host = config['auth_host'] self.auth_port = int(config['auth_port']) self.auth_ssl = config['auth_ssl'] in ('on', 'true', 'yes', '1') + self.insecure = config_true_value(config.get('insecure', 'false')) self.auth_prefix = config.get('auth_prefix', '/') self.auth_version = str(config.get('auth_version', '1')) @@ -117,6 +125,7 @@ class Connection(object): self.storage_host = None self.storage_port = None + self.storage_url = None self.conn_class = None @@ -145,10 +154,11 @@ class Connection(object): auth_netloc = "%s:%d" % (self.auth_host, self.auth_port) auth_url = auth_scheme + auth_netloc + auth_path + authargs = dict(snet=False, tenant_name=self.account, + auth_version=self.auth_version, os_options={}, + insecure=self.insecure) (storage_url, storage_token) = get_auth( - auth_url, auth_user, self.password, snet=False, - tenant_name=self.account, auth_version=self.auth_version, - os_options={}) + auth_url, auth_user, self.password, **authargs) if not (storage_url and storage_token): raise AuthenticationFailed() @@ -172,8 +182,14 @@ class Connection(object): # unicode and this would cause troubles when doing # no_safe_quote query. self.storage_url = str('/%s/%s' % (x[3], x[4])) - - self.storage_token = storage_token + self.account_name = str(x[4]) + self.auth_user = auth_user + # With v2 keystone, storage_token is unicode. + # We want it to be string otherwise this would cause + # troubles when doing query with already encoded + # non ascii characters in its headers. + self.storage_token = str(storage_token) + self.user_acl = '%s:%s' % (self.account, self.username) self.http_connect() return self.storage_url, self.storage_token @@ -184,7 +200,7 @@ class Connection(object): """ status = self.make_request('GET', '/info', cfg={'absolute_path': True}) - if status == 404: + if status // 100 == 4: return {} if not 200 <= status <= 299: raise ResponseError(self.response, 'GET', '/info') @@ -195,7 +211,12 @@ class Connection(object): port=self.storage_port) #self.connection.set_debuglevel(3) - def make_path(self, path=[], cfg={}): + def make_path(self, path=None, cfg=None): + if path is None: + path = [] + if cfg is None: + cfg = {} + if cfg.get('version_only_path'): return '/' + self.storage_url.split('/')[1] @@ -208,7 +229,9 @@ class Connection(object): else: return self.storage_url - def make_headers(self, hdrs, cfg={}): + def make_headers(self, hdrs, cfg=None): + if cfg is None: + cfg = {} headers = {} if not cfg.get('no_auth_token'): @@ -218,8 +241,16 @@ class Connection(object): headers.update(hdrs) return headers - def make_request(self, method, path=[], data='', hdrs={}, parms={}, - cfg={}): + def make_request(self, method, path=None, data='', hdrs=None, parms=None, + cfg=None): + if path is None: + path = [] + if hdrs is None: + hdrs = {} + if parms is None: + parms = {} + if cfg is None: + cfg = {} if not cfg.get('absolute_path'): # Set absolute_path=True to make a request to exactly the given # path, not storage path + given path. Useful for @@ -277,7 +308,14 @@ class Connection(object): 'Attempts: %s, Failures: %s' % (request, len(fail_messages), fail_messages)) - def put_start(self, path, hdrs={}, parms={}, cfg={}, chunked=False): + def put_start(self, path, hdrs=None, parms=None, cfg=None, chunked=False): + if hdrs is None: + hdrs = {} + if parms is None: + parms = {} + if cfg is None: + cfg = {} + self.http_connect() path = self.make_path(path, cfg) @@ -322,7 +360,10 @@ class Base(object): def __str__(self): return self.name - def header_fields(self, required_fields, optional_fields=()): + def header_fields(self, required_fields, optional_fields=None): + if optional_fields is None: + optional_fields = () + headers = dict(self.conn.response.getheaders()) ret = {} @@ -352,7 +393,11 @@ class Account(Base): self.conn = conn self.name = str(name) - def update_metadata(self, metadata={}, cfg={}): + def update_metadata(self, metadata=None, cfg=None): + if metadata is None: + metadata = {} + if cfg is None: + cfg = {} headers = dict(("X-Account-Meta-%s" % k, v) for k, v in metadata.items()) @@ -365,7 +410,14 @@ class Account(Base): def container(self, container_name): return Container(self.conn, self.name, container_name) - def containers(self, hdrs={}, parms={}, cfg={}): + def containers(self, hdrs=None, parms=None, cfg=None): + if hdrs is None: + hdrs = {} + if parms is None: + parms = {} + if cfg is None: + cfg = {} + format_type = parms.get('format', None) if format_type not in [None, 'json', 'xml']: raise RequestError('Invalid format: %s' % format_type) @@ -411,7 +463,13 @@ class Account(Base): return listing_empty(self.containers) - def info(self, hdrs={}, parms={}, cfg={}): + def info(self, hdrs=None, parms=None, cfg=None): + if hdrs is None: + hdrs = {} + if parms is None: + parms = {} + if cfg is None: + cfg = {} if self.conn.make_request('HEAD', self.path, hdrs=hdrs, parms=parms, cfg=cfg) != 204: @@ -435,11 +493,21 @@ class Container(Base): self.account = str(account) self.name = str(name) - def create(self, hdrs={}, parms={}, cfg={}): + def create(self, hdrs=None, parms=None, cfg=None): + if hdrs is None: + hdrs = {} + if parms is None: + parms = {} + if cfg is None: + cfg = {} return self.conn.make_request('PUT', self.path, hdrs=hdrs, parms=parms, cfg=cfg) in (201, 202) - def delete(self, hdrs={}, parms={}): + def delete(self, hdrs=None, parms=None): + if hdrs is None: + hdrs = {} + if parms is None: + parms = {} return self.conn.make_request('DELETE', self.path, hdrs=hdrs, parms=parms) == 204 @@ -457,7 +525,13 @@ class Container(Base): def file(self, file_name): return File(self.conn, self.account, self.name, file_name) - def files(self, hdrs={}, parms={}, cfg={}): + def files(self, hdrs=None, parms=None, cfg=None): + if hdrs is None: + hdrs = {} + if parms is None: + parms = {} + if cfg is None: + cfg = {} format_type = parms.get('format', None) if format_type not in [None, 'json', 'xml']: raise RequestError('Invalid format: %s' % format_type) @@ -507,14 +581,23 @@ class Container(Base): raise ResponseError(self.conn.response, 'GET', self.conn.make_path(self.path)) - def info(self, hdrs={}, parms={}, cfg={}): + def info(self, hdrs=None, parms=None, cfg=None): + if hdrs is None: + hdrs = {} + if parms is None: + parms = {} + if cfg is None: + cfg = {} self.conn.make_request('HEAD', self.path, hdrs=hdrs, parms=parms, cfg=cfg) if self.conn.response.status == 204: required_fields = [['bytes_used', 'x-container-bytes-used'], ['object_count', 'x-container-object-count']] - optional_fields = [['versions', 'x-versions-location']] + optional_fields = [ + ['versions', 'x-versions-location'], + ['tempurl_key', 'x-container-meta-temp-url-key'], + ['tempurl_key2', 'x-container-meta-temp-url-key-2']] return self.header_fields(required_fields, optional_fields) @@ -538,7 +621,9 @@ class File(Base): self.size = None self.metadata = {} - def make_headers(self, cfg={}): + def make_headers(self, cfg=None): + if cfg is None: + cfg = {} headers = {} if not cfg.get('no_content_length'): if cfg.get('set_content_length'): @@ -580,7 +665,13 @@ class File(Base): data.seek(0) return checksum.hexdigest() - def copy(self, dest_cont, dest_file, hdrs={}, parms={}, cfg={}): + def copy(self, dest_cont, dest_file, hdrs=None, parms=None, cfg=None): + if hdrs is None: + hdrs = {} + if parms is None: + parms = {} + if cfg is None: + cfg = {} if 'destination' in cfg: headers = {'Destination': cfg['destination']} elif cfg.get('no_destination'): @@ -595,7 +686,37 @@ class File(Base): return self.conn.make_request('COPY', self.path, hdrs=headers, parms=parms) == 201 - def delete(self, hdrs={}, parms={}): + def copy_account(self, dest_account, dest_cont, dest_file, + hdrs=None, parms=None, cfg=None): + if hdrs is None: + hdrs = {} + if parms is None: + parms = {} + if cfg is None: + cfg = {} + if 'destination' in cfg: + headers = {'Destination': cfg['destination']} + elif cfg.get('no_destination'): + headers = {} + else: + headers = {'Destination-Account': dest_account, + 'Destination': '%s/%s' % (dest_cont, dest_file)} + headers.update(hdrs) + + if 'Destination-Account' in headers: + headers['Destination-Account'] = \ + urllib.quote(headers['Destination-Account']) + if 'Destination' in headers: + headers['Destination'] = urllib.quote(headers['Destination']) + + return self.conn.make_request('COPY', self.path, hdrs=headers, + parms=parms) == 201 + + def delete(self, hdrs=None, parms=None): + if hdrs is None: + hdrs = {} + if parms is None: + parms = {} if self.conn.make_request('DELETE', self.path, hdrs=hdrs, parms=parms) != 204: @@ -604,7 +725,13 @@ class File(Base): return True - def info(self, hdrs={}, parms={}, cfg={}): + def info(self, hdrs=None, parms=None, cfg=None): + if hdrs is None: + hdrs = {} + if parms is None: + parms = {} + if cfg is None: + cfg = {} if self.conn.make_request('HEAD', self.path, hdrs=hdrs, parms=parms, cfg=cfg) != 200: @@ -615,8 +742,8 @@ class File(Base): ['content_type', 'content-type'], ['last_modified', 'last-modified'], ['etag', 'etag']] - - optional_fields = [['x_delete_at', 'x-delete-at'], + optional_fields = [['x_object_manifest', 'x-object-manifest'], + ['x_delete_at', 'x-delete-at'], ['x_delete_after', 'x-delete-after']] header_fields = self.header_fields(fields, @@ -624,7 +751,11 @@ class File(Base): header_fields['etag'] = header_fields['etag'].strip('"') return header_fields - def initialize(self, hdrs={}, parms={}): + def initialize(self, hdrs=None, parms=None): + if hdrs is None: + hdrs = {} + if parms is None: + parms = {} if not self.name: return False @@ -669,7 +800,11 @@ class File(Base): return data def read(self, size=-1, offset=0, hdrs=None, buffer=None, - callback=None, cfg={}, parms={}): + callback=None, cfg=None, parms=None): + if cfg is None: + cfg = {} + if parms is None: + parms = {} if size > 0: range_string = 'bytes=%d-%d' % (offset, (offset + size) - 1) @@ -726,7 +861,12 @@ class File(Base): finally: fobj.close() - def sync_metadata(self, metadata={}, cfg={}): + def sync_metadata(self, metadata=None, cfg=None): + if metadata is None: + metadata = {} + if cfg is None: + cfg = {} + self.metadata.update(metadata) if self.metadata: @@ -737,6 +877,7 @@ class File(Base): cfg.get('set_content_length') else: headers['Content-Length'] = 0 + self.conn.make_request('POST', self.path, hdrs=headers, cfg=cfg) if self.conn.response.status not in (201, 202): @@ -745,7 +886,14 @@ class File(Base): return True - def chunked_write(self, data=None, hdrs={}, parms={}, cfg={}): + def chunked_write(self, data=None, hdrs=None, parms=None, cfg=None): + if hdrs is None: + hdrs = {} + if parms is None: + parms = {} + if cfg is None: + cfg = {} + if data is not None and self.chunked_write_in_progress: self.conn.put_data(data, True) elif data is not None: @@ -764,8 +912,15 @@ class File(Base): else: raise RuntimeError - def write(self, data='', hdrs={}, parms={}, callback=None, cfg={}, + def write(self, data='', hdrs=None, parms=None, callback=None, cfg=None, return_resp=False): + if hdrs is None: + hdrs = {} + if parms is None: + parms = {} + if cfg is None: + cfg = {} + block_size = 2 ** 20 if isinstance(data, file): @@ -786,13 +941,15 @@ class File(Base): transferred = 0 buff = data.read(block_size) + buff_len = len(buff) try: - while len(buff) > 0: + while buff_len > 0: self.conn.put_data(buff) - buff = data.read(block_size) - transferred += len(buff) + transferred += buff_len if callable(callback): callback(transferred, self.size) + buff = data.read(block_size) + buff_len = len(buff) self.conn.put_end() except socket.timeout as err: @@ -814,7 +971,14 @@ class File(Base): return True - def write_random(self, size=None, hdrs={}, parms={}, cfg={}): + def write_random(self, size=None, hdrs=None, parms=None, cfg=None): + if hdrs is None: + hdrs = {} + if parms is None: + parms = {} + if cfg is None: + cfg = {} + data = self.random_data(size) if not self.write(data, hdrs=hdrs, parms=parms, cfg=cfg): raise ResponseError(self.conn.response, 'PUT', @@ -822,7 +986,15 @@ class File(Base): self.md5 = self.compute_md5sum(StringIO.StringIO(data)) return data - def write_random_return_resp(self, size=None, hdrs={}, parms={}, cfg={}): + def write_random_return_resp(self, size=None, hdrs=None, parms=None, + cfg=None): + if hdrs is None: + hdrs = {} + if parms is None: + parms = {} + if cfg is None: + cfg = {} + data = self.random_data(size) resp = self.write(data, hdrs=hdrs, parms=parms, cfg=cfg, return_resp=True) diff --git a/test/functional/test_account.py b/test/functional/test_account.py index 1cc61bc..30a8e74 100755 --- a/test/functional/test_account.py +++ b/test/functional/test_account.py @@ -21,13 +21,11 @@ from uuid import uuid4 from nose import SkipTest from string import letters -from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \ - MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH from swift.common.middleware.acl import format_acl -from swift_testing import (check_response, retry, skip, skip2, skip3, - web_front_end, requires_acls) -import swift_testing -from test.functional.tests import load_constraint + +from test.functional import check_response, retry, requires_acls, \ + load_constraint +import test.functional as tf class TestAccount(unittest.TestCase): @@ -69,7 +67,7 @@ class TestAccount(unittest.TestCase): self.assertEqual(resp.status // 100, 2) def test_metadata(self): - if skip: + if tf.skip: raise SkipTest def post(url, token, parsed, conn, value): @@ -109,6 +107,9 @@ class TestAccount(unittest.TestCase): self.assertEqual(resp.getheader('x-account-meta-test'), 'Value') def test_invalid_acls(self): + if tf.skip: + raise SkipTest + def post(url, token, parsed, conn, headers): new_headers = dict({'X-Auth-Token': token}, **headers) conn.request('POST', parsed.path, '', new_headers) @@ -145,7 +146,7 @@ class TestAccount(unittest.TestCase): resp.read() self.assertEqual(resp.status, 400) - acl_user = swift_testing.swift_test_user[1] + acl_user = tf.swift_test_user[1] acl = {'admin': [acl_user], 'invalid_key': 'invalid_value'} headers = {'x-account-access-control': format_acl( version=2, acl_dict=acl)} @@ -173,7 +174,7 @@ class TestAccount(unittest.TestCase): @requires_acls def test_read_only_acl(self): - if skip3: + if tf.skip3: raise SkipTest def get(url, token, parsed, conn): @@ -191,7 +192,7 @@ class TestAccount(unittest.TestCase): self.assertEquals(resp.status, 403) # grant read access - acl_user = swift_testing.swift_test_user[2] + acl_user = tf.swift_test_user[2] acl = {'read-only': [acl_user]} headers = {'x-account-access-control': format_acl( version=2, acl_dict=acl)} @@ -224,7 +225,7 @@ class TestAccount(unittest.TestCase): @requires_acls def test_read_write_acl(self): - if skip3: + if tf.skip3: raise SkipTest def get(url, token, parsed, conn): @@ -242,7 +243,7 @@ class TestAccount(unittest.TestCase): self.assertEquals(resp.status, 403) # grant read-write access - acl_user = swift_testing.swift_test_user[2] + acl_user = tf.swift_test_user[2] acl = {'read-write': [acl_user]} headers = {'x-account-access-control': format_acl( version=2, acl_dict=acl)} @@ -265,7 +266,7 @@ class TestAccount(unittest.TestCase): @requires_acls def test_admin_acl(self): - if skip3: + if tf.skip3: raise SkipTest def get(url, token, parsed, conn): @@ -283,7 +284,7 @@ class TestAccount(unittest.TestCase): self.assertEquals(resp.status, 403) # grant admin access - acl_user = swift_testing.swift_test_user[2] + acl_user = tf.swift_test_user[2] acl = {'admin': [acl_user]} acl_json_str = format_acl(version=2, acl_dict=acl) headers = {'x-account-access-control': acl_json_str} @@ -323,7 +324,7 @@ class TestAccount(unittest.TestCase): @requires_acls def test_protected_tempurl(self): - if skip3: + if tf.skip3: raise SkipTest def get(url, token, parsed, conn): @@ -335,7 +336,7 @@ class TestAccount(unittest.TestCase): conn.request('POST', parsed.path, '', new_headers) return check_response(conn) - # add a account metadata, and temp-url-key to account + # add an account metadata, and temp-url-key to account value = str(uuid4()) headers = { 'x-account-meta-temp-url-key': 'secret', @@ -346,7 +347,7 @@ class TestAccount(unittest.TestCase): self.assertEqual(resp.status, 204) # grant read-only access to tester3 - acl_user = swift_testing.swift_test_user[2] + acl_user = tf.swift_test_user[2] acl = {'read-only': [acl_user]} acl_json_str = format_acl(version=2, acl_dict=acl) headers = {'x-account-access-control': acl_json_str} @@ -364,7 +365,7 @@ class TestAccount(unittest.TestCase): self.assertEqual(resp.getheader('X-Account-Meta-Temp-Url-Key'), None) # grant read-write access to tester3 - acl_user = swift_testing.swift_test_user[2] + acl_user = tf.swift_test_user[2] acl = {'read-write': [acl_user]} acl_json_str = format_acl(version=2, acl_dict=acl) headers = {'x-account-access-control': acl_json_str} @@ -382,7 +383,7 @@ class TestAccount(unittest.TestCase): self.assertEqual(resp.getheader('X-Account-Meta-Temp-Url-Key'), None) # grant admin access to tester3 - acl_user = swift_testing.swift_test_user[2] + acl_user = tf.swift_test_user[2] acl = {'admin': [acl_user]} acl_json_str = format_acl(version=2, acl_dict=acl) headers = {'x-account-access-control': acl_json_str} @@ -417,7 +418,7 @@ class TestAccount(unittest.TestCase): @requires_acls def test_account_acls(self): - if skip2: + if tf.skip2: raise SkipTest def post(url, token, parsed, conn, headers): @@ -464,7 +465,7 @@ class TestAccount(unittest.TestCase): # User1 is swift_owner of their own account, so they can POST an # ACL -- let's do this and make User2 (test_user[1]) an admin - acl_user = swift_testing.swift_test_user[1] + acl_user = tf.swift_test_user[1] acl = {'admin': [acl_user]} headers = {'x-account-access-control': format_acl( version=2, acl_dict=acl)} @@ -541,7 +542,7 @@ class TestAccount(unittest.TestCase): @requires_acls def test_swift_account_acls(self): - if skip: + if tf.skip: raise SkipTest def post(url, token, parsed, conn, headers): @@ -604,7 +605,7 @@ class TestAccount(unittest.TestCase): resp.read() def test_swift_prohibits_garbage_account_acls(self): - if skip: + if tf.skip: raise SkipTest def post(url, token, parsed, conn, headers): @@ -671,7 +672,7 @@ class TestAccount(unittest.TestCase): resp.read() def test_unicode_metadata(self): - if skip: + if tf.skip: raise SkipTest def post(url, token, parsed, conn, name, value): @@ -684,7 +685,7 @@ class TestAccount(unittest.TestCase): return check_response(conn) uni_key = u'X-Account-Meta-uni\u0E12' uni_value = u'uni\u0E12' - if (web_front_end == 'integral'): + if (tf.web_front_end == 'integral'): resp = retry(post, uni_key, '1') resp.read() self.assertTrue(resp.status in (201, 204)) @@ -700,7 +701,7 @@ class TestAccount(unittest.TestCase): self.assert_(resp.status in (200, 204), resp.status) self.assertEqual(resp.getheader('X-Account-Meta-uni'), uni_value.encode('utf-8')) - if (web_front_end == 'integral'): + if (tf.web_front_end == 'integral'): resp = retry(post, uni_key, uni_value) resp.read() self.assertEqual(resp.status, 204) @@ -711,7 +712,7 @@ class TestAccount(unittest.TestCase): uni_value.encode('utf-8')) def test_multi_metadata(self): - if skip: + if tf.skip: raise SkipTest def post(url, token, parsed, conn, name, value): @@ -740,7 +741,7 @@ class TestAccount(unittest.TestCase): self.assertEqual(resp.getheader('x-account-meta-two'), '2') def test_bad_metadata(self): - if skip: + if tf.skip: raise SkipTest def post(url, token, parsed, conn, extra_headers): @@ -750,27 +751,31 @@ class TestAccount(unittest.TestCase): return check_response(conn) resp = retry(post, - {'X-Account-Meta-' + ('k' * MAX_META_NAME_LENGTH): 'v'}) + {'X-Account-Meta-' + ( + 'k' * self.max_meta_name_length): 'v'}) resp.read() self.assertEqual(resp.status, 204) resp = retry( post, - {'X-Account-Meta-' + ('k' * (MAX_META_NAME_LENGTH + 1)): 'v'}) + {'X-Account-Meta-' + ('k' * ( + self.max_meta_name_length + 1)): 'v'}) resp.read() self.assertEqual(resp.status, 400) resp = retry(post, - {'X-Account-Meta-Too-Long': 'k' * MAX_META_VALUE_LENGTH}) + {'X-Account-Meta-Too-Long': ( + 'k' * self.max_meta_value_length)}) resp.read() self.assertEqual(resp.status, 204) resp = retry( post, - {'X-Account-Meta-Too-Long': 'k' * (MAX_META_VALUE_LENGTH + 1)}) + {'X-Account-Meta-Too-Long': 'k' * ( + self.max_meta_value_length + 1)}) resp.read() self.assertEqual(resp.status, 400) def test_bad_metadata2(self): - if skip: + if tf.skip: raise SkipTest def post(url, token, parsed, conn, extra_headers): @@ -785,20 +790,20 @@ class TestAccount(unittest.TestCase): resp = retry(post, headers) headers = {} - for x in xrange(MAX_META_COUNT): + for x in xrange(self.max_meta_count): headers['X-Account-Meta-%d' % x] = 'v' resp = retry(post, headers) resp.read() self.assertEqual(resp.status, 204) headers = {} - for x in xrange(MAX_META_COUNT + 1): + for x in xrange(self.max_meta_count + 1): headers['X-Account-Meta-%d' % x] = 'v' resp = retry(post, headers) resp.read() self.assertEqual(resp.status, 400) def test_bad_metadata3(self): - if skip: + if tf.skip: raise SkipTest def post(url, token, parsed, conn, extra_headers): @@ -807,31 +812,55 @@ class TestAccount(unittest.TestCase): conn.request('POST', parsed.path, '', headers) return check_response(conn) - # TODO: Find the test that adds these and remove them. - headers = {'x-remove-account-meta-temp-url-key': 'remove', - 'x-remove-account-meta-temp-url-key-2': 'remove'} - resp = retry(post, headers) - headers = {} - header_value = 'k' * MAX_META_VALUE_LENGTH + header_value = 'k' * self.max_meta_value_length size = 0 x = 0 - while size < MAX_META_OVERALL_SIZE - 4 - MAX_META_VALUE_LENGTH: - size += 4 + MAX_META_VALUE_LENGTH + while size < (self.max_meta_overall_size - 4 + - self.max_meta_value_length): + size += 4 + self.max_meta_value_length headers['X-Account-Meta-%04d' % x] = header_value x += 1 - if MAX_META_OVERALL_SIZE - size > 1: + if self.max_meta_overall_size - size > 1: headers['X-Account-Meta-k'] = \ - 'v' * (MAX_META_OVERALL_SIZE - size - 1) + 'v' * (self.max_meta_overall_size - size - 1) resp = retry(post, headers) resp.read() self.assertEqual(resp.status, 204) headers['X-Account-Meta-k'] = \ - 'v' * (MAX_META_OVERALL_SIZE - size) + 'v' * (self.max_meta_overall_size - size) resp = retry(post, headers) resp.read() self.assertEqual(resp.status, 400) +class TestAccountInNonDefaultDomain(unittest.TestCase): + def setUp(self): + if tf.skip or tf.skip2 or tf.skip_if_not_v3: + raise SkipTest('AUTH VERSION 3 SPECIFIC TEST') + + def test_project_domain_id_header(self): + # make sure account exists (assumes account auto create) + def post(url, token, parsed, conn): + conn.request('POST', parsed.path, '', + {'X-Auth-Token': token}) + return check_response(conn) + + resp = retry(post, use_account=4) + resp.read() + self.assertEqual(resp.status, 204) + + # account in non-default domain should have a project domain id + def head(url, token, parsed, conn): + conn.request('HEAD', parsed.path, '', + {'X-Auth-Token': token}) + return check_response(conn) + + resp = retry(head, use_account=4) + resp.read() + self.assertEqual(resp.status, 204) + self.assertTrue('X-Account-Project-Domain-Id' in resp.headers) + + if __name__ == '__main__': unittest.main() diff --git a/test/functional/test_container.py b/test/functional/test_container.py index 91702e9..d7896a4 100755 --- a/test/functional/test_container.py +++ b/test/functional/test_container.py @@ -20,19 +20,19 @@ import unittest from nose import SkipTest from uuid import uuid4 -from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \ - MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH - -from swift_testing import check_response, retry, skip, skip2, skip3, \ - swift_test_perm, web_front_end, requires_acls, swift_test_user +from test.functional import check_response, retry, requires_acls, \ + load_constraint, requires_policies +import test.functional as tf class TestContainer(unittest.TestCase): def setUp(self): - if skip: + if tf.skip: raise SkipTest self.name = uuid4().hex + # this container isn't created by default, but will be cleaned up + self.container = uuid4().hex def put(url, token, parsed, conn): conn.request('PUT', parsed.path + '/' + self.name, '', @@ -43,44 +43,58 @@ class TestContainer(unittest.TestCase): resp.read() self.assertEqual(resp.status, 201) + self.max_meta_count = load_constraint('max_meta_count') + self.max_meta_name_length = load_constraint('max_meta_name_length') + self.max_meta_overall_size = load_constraint('max_meta_overall_size') + self.max_meta_value_length = load_constraint('max_meta_value_length') + def tearDown(self): - if skip: + if tf.skip: raise SkipTest - def get(url, token, parsed, conn): - conn.request('GET', parsed.path + '/' + self.name + '?format=json', - '', {'X-Auth-Token': token}) - return check_response(conn) - - def delete(url, token, parsed, conn, obj): - conn.request('DELETE', - '/'.join([parsed.path, self.name, obj['name']]), '', - {'X-Auth-Token': token}) - return check_response(conn) - - while True: - resp = retry(get) - body = resp.read() - self.assert_(resp.status // 100 == 2, resp.status) - objs = json.loads(body) - if not objs: - break - for obj in objs: - resp = retry(delete, obj) - resp.read() - self.assertEqual(resp.status, 204) - - def delete(url, token, parsed, conn): - conn.request('DELETE', parsed.path + '/' + self.name, '', + def get(url, token, parsed, conn, container): + conn.request( + 'GET', parsed.path + '/' + container + '?format=json', '', + {'X-Auth-Token': token}) + return check_response(conn) + + def delete(url, token, parsed, conn, container, obj): + conn.request( + 'DELETE', '/'.join([parsed.path, container, obj['name']]), '', + {'X-Auth-Token': token}) + return check_response(conn) + + for container in (self.name, self.container): + while True: + resp = retry(get, container) + body = resp.read() + if resp.status == 404: + break + self.assert_(resp.status // 100 == 2, resp.status) + objs = json.loads(body) + if not objs: + break + for obj in objs: + resp = retry(delete, container, obj) + resp.read() + self.assertEqual(resp.status, 204) + + def delete(url, token, parsed, conn, container): + conn.request('DELETE', parsed.path + '/' + container, '', {'X-Auth-Token': token}) return check_response(conn) - resp = retry(delete) + resp = retry(delete, self.name) resp.read() self.assertEqual(resp.status, 204) + # container may have not been created + resp = retry(delete, self.container) + resp.read() + self.assert_(resp.status in (204, 404)) + def test_multi_metadata(self): - if skip: + if tf.skip: raise SkipTest def post(url, token, parsed, conn, name, value): @@ -110,7 +124,7 @@ class TestContainer(unittest.TestCase): self.assertEqual(resp.getheader('x-container-meta-two'), '2') def test_unicode_metadata(self): - if skip: + if tf.skip: raise SkipTest def post(url, token, parsed, conn, name, value): @@ -125,7 +139,7 @@ class TestContainer(unittest.TestCase): uni_key = u'X-Container-Meta-uni\u0E12' uni_value = u'uni\u0E12' - if (web_front_end == 'integral'): + if (tf.web_front_end == 'integral'): resp = retry(post, uni_key, '1') resp.read() self.assertEqual(resp.status, 204) @@ -141,7 +155,7 @@ class TestContainer(unittest.TestCase): self.assert_(resp.status in (200, 204), resp.status) self.assertEqual(resp.getheader('X-Container-Meta-uni'), uni_value.encode('utf-8')) - if (web_front_end == 'integral'): + if (tf.web_front_end == 'integral'): resp = retry(post, uni_key, uni_value) resp.read() self.assertEqual(resp.status, 204) @@ -152,7 +166,7 @@ class TestContainer(unittest.TestCase): uni_value.encode('utf-8')) def test_PUT_metadata(self): - if skip: + if tf.skip: raise SkipTest def put(url, token, parsed, conn, name, value): @@ -209,7 +223,7 @@ class TestContainer(unittest.TestCase): self.assertEqual(resp.status, 204) def test_POST_metadata(self): - if skip: + if tf.skip: raise SkipTest def post(url, token, parsed, conn, value): @@ -249,7 +263,7 @@ class TestContainer(unittest.TestCase): self.assertEqual(resp.getheader('x-container-meta-test'), 'Value') def test_PUT_bad_metadata(self): - if skip: + if tf.skip: raise SkipTest def put(url, token, parsed, conn, name, extra_headers): @@ -266,7 +280,7 @@ class TestContainer(unittest.TestCase): name = uuid4().hex resp = retry( put, name, - {'X-Container-Meta-' + ('k' * MAX_META_NAME_LENGTH): 'v'}) + {'X-Container-Meta-' + ('k' * self.max_meta_name_length): 'v'}) resp.read() self.assertEqual(resp.status, 201) resp = retry(delete, name) @@ -275,7 +289,8 @@ class TestContainer(unittest.TestCase): name = uuid4().hex resp = retry( put, name, - {'X-Container-Meta-' + ('k' * (MAX_META_NAME_LENGTH + 1)): 'v'}) + {'X-Container-Meta-' + ( + 'k' * (self.max_meta_name_length + 1)): 'v'}) resp.read() self.assertEqual(resp.status, 400) resp = retry(delete, name) @@ -285,7 +300,7 @@ class TestContainer(unittest.TestCase): name = uuid4().hex resp = retry( put, name, - {'X-Container-Meta-Too-Long': 'k' * MAX_META_VALUE_LENGTH}) + {'X-Container-Meta-Too-Long': 'k' * self.max_meta_value_length}) resp.read() self.assertEqual(resp.status, 201) resp = retry(delete, name) @@ -294,7 +309,8 @@ class TestContainer(unittest.TestCase): name = uuid4().hex resp = retry( put, name, - {'X-Container-Meta-Too-Long': 'k' * (MAX_META_VALUE_LENGTH + 1)}) + {'X-Container-Meta-Too-Long': 'k' * ( + self.max_meta_value_length + 1)}) resp.read() self.assertEqual(resp.status, 400) resp = retry(delete, name) @@ -303,7 +319,7 @@ class TestContainer(unittest.TestCase): name = uuid4().hex headers = {} - for x in xrange(MAX_META_COUNT): + for x in xrange(self.max_meta_count): headers['X-Container-Meta-%d' % x] = 'v' resp = retry(put, name, headers) resp.read() @@ -313,7 +329,7 @@ class TestContainer(unittest.TestCase): self.assertEqual(resp.status, 204) name = uuid4().hex headers = {} - for x in xrange(MAX_META_COUNT + 1): + for x in xrange(self.max_meta_count + 1): headers['X-Container-Meta-%d' % x] = 'v' resp = retry(put, name, headers) resp.read() @@ -324,16 +340,17 @@ class TestContainer(unittest.TestCase): name = uuid4().hex headers = {} - header_value = 'k' * MAX_META_VALUE_LENGTH + header_value = 'k' * self.max_meta_value_length size = 0 x = 0 - while size < MAX_META_OVERALL_SIZE - 4 - MAX_META_VALUE_LENGTH: - size += 4 + MAX_META_VALUE_LENGTH + while size < (self.max_meta_overall_size - 4 + - self.max_meta_value_length): + size += 4 + self.max_meta_value_length headers['X-Container-Meta-%04d' % x] = header_value x += 1 - if MAX_META_OVERALL_SIZE - size > 1: + if self.max_meta_overall_size - size > 1: headers['X-Container-Meta-k'] = \ - 'v' * (MAX_META_OVERALL_SIZE - size - 1) + 'v' * (self.max_meta_overall_size - size - 1) resp = retry(put, name, headers) resp.read() self.assertEqual(resp.status, 201) @@ -342,7 +359,7 @@ class TestContainer(unittest.TestCase): self.assertEqual(resp.status, 204) name = uuid4().hex headers['X-Container-Meta-k'] = \ - 'v' * (MAX_META_OVERALL_SIZE - size) + 'v' * (self.max_meta_overall_size - size) resp = retry(put, name, headers) resp.read() self.assertEqual(resp.status, 400) @@ -351,7 +368,7 @@ class TestContainer(unittest.TestCase): self.assertEqual(resp.status, 404) def test_POST_bad_metadata(self): - if skip: + if tf.skip: raise SkipTest def post(url, token, parsed, conn, extra_headers): @@ -362,28 +379,30 @@ class TestContainer(unittest.TestCase): resp = retry( post, - {'X-Container-Meta-' + ('k' * MAX_META_NAME_LENGTH): 'v'}) + {'X-Container-Meta-' + ('k' * self.max_meta_name_length): 'v'}) resp.read() self.assertEqual(resp.status, 204) resp = retry( post, - {'X-Container-Meta-' + ('k' * (MAX_META_NAME_LENGTH + 1)): 'v'}) + {'X-Container-Meta-' + ( + 'k' * (self.max_meta_name_length + 1)): 'v'}) resp.read() self.assertEqual(resp.status, 400) resp = retry( post, - {'X-Container-Meta-Too-Long': 'k' * MAX_META_VALUE_LENGTH}) + {'X-Container-Meta-Too-Long': 'k' * self.max_meta_value_length}) resp.read() self.assertEqual(resp.status, 204) resp = retry( post, - {'X-Container-Meta-Too-Long': 'k' * (MAX_META_VALUE_LENGTH + 1)}) + {'X-Container-Meta-Too-Long': 'k' * ( + self.max_meta_value_length + 1)}) resp.read() self.assertEqual(resp.status, 400) def test_POST_bad_metadata2(self): - if skip: + if tf.skip: raise SkipTest def post(url, token, parsed, conn, extra_headers): @@ -393,20 +412,20 @@ class TestContainer(unittest.TestCase): return check_response(conn) headers = {} - for x in xrange(MAX_META_COUNT): + for x in xrange(self.max_meta_count): headers['X-Container-Meta-%d' % x] = 'v' resp = retry(post, headers) resp.read() self.assertEqual(resp.status, 204) headers = {} - for x in xrange(MAX_META_COUNT + 1): + for x in xrange(self.max_meta_count + 1): headers['X-Container-Meta-%d' % x] = 'v' resp = retry(post, headers) resp.read() self.assertEqual(resp.status, 400) def test_POST_bad_metadata3(self): - if skip: + if tf.skip: raise SkipTest def post(url, token, parsed, conn, extra_headers): @@ -416,27 +435,28 @@ class TestContainer(unittest.TestCase): return check_response(conn) headers = {} - header_value = 'k' * MAX_META_VALUE_LENGTH + header_value = 'k' * self.max_meta_value_length size = 0 x = 0 - while size < MAX_META_OVERALL_SIZE - 4 - MAX_META_VALUE_LENGTH: - size += 4 + MAX_META_VALUE_LENGTH + while size < (self.max_meta_overall_size - 4 + - self.max_meta_value_length): + size += 4 + self.max_meta_value_length headers['X-Container-Meta-%04d' % x] = header_value x += 1 - if MAX_META_OVERALL_SIZE - size > 1: + if self.max_meta_overall_size - size > 1: headers['X-Container-Meta-k'] = \ - 'v' * (MAX_META_OVERALL_SIZE - size - 1) + 'v' * (self.max_meta_overall_size - size - 1) resp = retry(post, headers) resp.read() self.assertEqual(resp.status, 204) headers['X-Container-Meta-k'] = \ - 'v' * (MAX_META_OVERALL_SIZE - size) + 'v' * (self.max_meta_overall_size - size) resp = retry(post, headers) resp.read() self.assertEqual(resp.status, 400) def test_public_container(self): - if skip: + if tf.skip: raise SkipTest def get(url, token, parsed, conn): @@ -477,7 +497,7 @@ class TestContainer(unittest.TestCase): self.assert_(str(err).startswith('No result after '), err) def test_cross_account_container(self): - if skip or skip2: + if tf.skip or tf.skip2: raise SkipTest # Obtain the first account's string first_account = ['unknown'] @@ -505,8 +525,8 @@ class TestContainer(unittest.TestCase): def post(url, token, parsed, conn): conn.request('POST', parsed.path + '/' + self.name, '', {'X-Auth-Token': token, - 'X-Container-Read': swift_test_perm[1], - 'X-Container-Write': swift_test_perm[1]}) + 'X-Container-Read': tf.swift_test_perm[1], + 'X-Container-Write': tf.swift_test_perm[1]}) return check_response(conn) resp = retry(post) @@ -533,7 +553,7 @@ class TestContainer(unittest.TestCase): self.assertEqual(resp.status, 403) def test_cross_account_public_container(self): - if skip or skip2: + if tf.skip or tf.skip2: raise SkipTest # Obtain the first account's string first_account = ['unknown'] @@ -586,7 +606,7 @@ class TestContainer(unittest.TestCase): def post(url, token, parsed, conn): conn.request('POST', parsed.path + '/' + self.name, '', {'X-Auth-Token': token, - 'X-Container-Write': swift_test_perm[1]}) + 'X-Container-Write': tf.swift_test_perm[1]}) return check_response(conn) resp = retry(post) @@ -602,7 +622,7 @@ class TestContainer(unittest.TestCase): self.assertEqual(resp.status, 201) def test_nonadmin_user(self): - if skip or skip3: + if tf.skip or tf.skip3: raise SkipTest # Obtain the first account's string first_account = ['unknown'] @@ -630,7 +650,7 @@ class TestContainer(unittest.TestCase): def post(url, token, parsed, conn): conn.request('POST', parsed.path + '/' + self.name, '', {'X-Auth-Token': token, - 'X-Container-Read': swift_test_perm[2]}) + 'X-Container-Read': tf.swift_test_perm[2]}) return check_response(conn) resp = retry(post) @@ -655,7 +675,7 @@ class TestContainer(unittest.TestCase): def post(url, token, parsed, conn): conn.request('POST', parsed.path + '/' + self.name, '', {'X-Auth-Token': token, - 'X-Container-Write': swift_test_perm[2]}) + 'X-Container-Write': tf.swift_test_perm[2]}) return check_response(conn) resp = retry(post) @@ -672,7 +692,7 @@ class TestContainer(unittest.TestCase): @requires_acls def test_read_only_acl_listings(self): - if skip3: + if tf.skip3: raise SkipTest def get(url, token, parsed, conn): @@ -695,7 +715,7 @@ class TestContainer(unittest.TestCase): self.assertEquals(resp.status, 403) # grant read-only access - acl_user = swift_test_user[2] + acl_user = tf.swift_test_user[2] acl = {'read-only': [acl_user]} headers = {'x-account-access-control': json.dumps(acl)} resp = retry(post_account, headers=headers, use_account=1) @@ -725,7 +745,7 @@ class TestContainer(unittest.TestCase): @requires_acls def test_read_only_acl_metadata(self): - if skip3: + if tf.skip3: raise SkipTest def get(url, token, parsed, conn, name): @@ -760,7 +780,7 @@ class TestContainer(unittest.TestCase): self.assertEquals(resp.status, 403) # grant read-only access - acl_user = swift_test_user[2] + acl_user = tf.swift_test_user[2] acl = {'read-only': [acl_user]} headers = {'x-account-access-control': json.dumps(acl)} resp = retry(post_account, headers=headers, use_account=1) @@ -782,7 +802,7 @@ class TestContainer(unittest.TestCase): @requires_acls def test_read_write_acl_listings(self): - if skip3: + if tf.skip3: raise SkipTest def get(url, token, parsed, conn): @@ -810,7 +830,7 @@ class TestContainer(unittest.TestCase): self.assertEquals(resp.status, 403) # grant read-write access - acl_user = swift_test_user[2] + acl_user = tf.swift_test_user[2] acl = {'read-write': [acl_user]} headers = {'x-account-access-control': json.dumps(acl)} resp = retry(post, headers=headers, use_account=1) @@ -853,7 +873,7 @@ class TestContainer(unittest.TestCase): @requires_acls def test_read_write_acl_metadata(self): - if skip3: + if tf.skip3: raise SkipTest def get(url, token, parsed, conn, name): @@ -888,7 +908,7 @@ class TestContainer(unittest.TestCase): self.assertEquals(resp.status, 403) # grant read-write access - acl_user = swift_test_user[2] + acl_user = tf.swift_test_user[2] acl = {'read-write': [acl_user]} headers = {'x-account-access-control': json.dumps(acl)} resp = retry(post_account, headers=headers, use_account=1) @@ -924,7 +944,7 @@ class TestContainer(unittest.TestCase): @requires_acls def test_admin_acl_listing(self): - if skip3: + if tf.skip3: raise SkipTest def get(url, token, parsed, conn): @@ -952,7 +972,7 @@ class TestContainer(unittest.TestCase): self.assertEquals(resp.status, 403) # grant admin access - acl_user = swift_test_user[2] + acl_user = tf.swift_test_user[2] acl = {'admin': [acl_user]} headers = {'x-account-access-control': json.dumps(acl)} resp = retry(post, headers=headers, use_account=1) @@ -995,7 +1015,7 @@ class TestContainer(unittest.TestCase): @requires_acls def test_admin_acl_metadata(self): - if skip3: + if tf.skip3: raise SkipTest def get(url, token, parsed, conn, name): @@ -1030,7 +1050,7 @@ class TestContainer(unittest.TestCase): self.assertEquals(resp.status, 403) # grant access - acl_user = swift_test_user[2] + acl_user = tf.swift_test_user[2] acl = {'admin': [acl_user]} headers = {'x-account-access-control': json.dumps(acl)} resp = retry(post_account, headers=headers, use_account=1) @@ -1066,7 +1086,7 @@ class TestContainer(unittest.TestCase): @requires_acls def test_protected_container_sync(self): - if skip3: + if tf.skip3: raise SkipTest def get(url, token, parsed, conn, name): @@ -1100,7 +1120,7 @@ class TestContainer(unittest.TestCase): self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) # grant read-only access - acl_user = swift_test_user[2] + acl_user = tf.swift_test_user[2] acl = {'read-only': [acl_user]} headers = {'x-account-access-control': json.dumps(acl)} resp = retry(post_account, headers=headers, use_account=1) @@ -1122,7 +1142,7 @@ class TestContainer(unittest.TestCase): self.assertEqual(resp.status, 403) # grant read-write access - acl_user = swift_test_user[2] + acl_user = tf.swift_test_user[2] acl = {'read-write': [acl_user]} headers = {'x-account-access-control': json.dumps(acl)} resp = retry(post_account, headers=headers, use_account=1) @@ -1160,7 +1180,7 @@ class TestContainer(unittest.TestCase): self.assertEqual(resp.getheader('X-Container-Sync-Key'), 'secret') # grant admin access - acl_user = swift_test_user[2] + acl_user = tf.swift_test_user[2] acl = {'admin': [acl_user]} headers = {'x-account-access-control': json.dumps(acl)} resp = retry(post_account, headers=headers, use_account=1) @@ -1188,7 +1208,7 @@ class TestContainer(unittest.TestCase): @requires_acls def test_protected_container_acl(self): - if skip3: + if tf.skip3: raise SkipTest def get(url, token, parsed, conn, name): @@ -1224,7 +1244,7 @@ class TestContainer(unittest.TestCase): self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) # grant read-only access - acl_user = swift_test_user[2] + acl_user = tf.swift_test_user[2] acl = {'read-only': [acl_user]} headers = {'x-account-access-control': json.dumps(acl)} resp = retry(post_account, headers=headers, use_account=1) @@ -1250,7 +1270,7 @@ class TestContainer(unittest.TestCase): self.assertEqual(resp.status, 403) # grant read-write access - acl_user = swift_test_user[2] + acl_user = tf.swift_test_user[2] acl = {'read-write': [acl_user]} headers = {'x-account-access-control': json.dumps(acl)} resp = retry(post_account, headers=headers, use_account=1) @@ -1292,7 +1312,7 @@ class TestContainer(unittest.TestCase): self.assertEqual(resp.getheader('X-Container-Write'), 'jdoe') # grant admin access - acl_user = swift_test_user[2] + acl_user = tf.swift_test_user[2] acl = {'admin': [acl_user]} headers = {'x-account-access-control': json.dumps(acl)} resp = retry(post_account, headers=headers, use_account=1) @@ -1322,7 +1342,7 @@ class TestContainer(unittest.TestCase): self.assertEqual(resp.getheader('X-Container-Read'), '.r:*') def test_long_name_content_type(self): - if skip: + if tf.skip: raise SkipTest def put(url, token, parsed, conn): @@ -1338,7 +1358,7 @@ class TestContainer(unittest.TestCase): 'text/html; charset=UTF-8') def test_null_name(self): - if skip: + if tf.skip: raise SkipTest def put(url, token, parsed, conn): @@ -1347,12 +1367,343 @@ class TestContainer(unittest.TestCase): return check_response(conn) resp = retry(put) - if (web_front_end == 'apache2'): + if (tf.web_front_end == 'apache2'): self.assertEqual(resp.status, 404) else: self.assertEqual(resp.read(), 'Invalid UTF8 or contains NULL') self.assertEqual(resp.status, 412) + def test_create_container_gets_default_policy_by_default(self): + try: + default_policy = \ + tf.FunctionalStoragePolicyCollection.from_info().default + except AssertionError: + raise SkipTest() + + def put(url, token, parsed, conn): + conn.request('PUT', parsed.path + '/' + self.container, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEqual(resp.status // 100, 2) + + def head(url, token, parsed, conn): + conn.request('HEAD', parsed.path + '/' + self.container, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(head) + resp.read() + headers = dict((k.lower(), v) for k, v in resp.getheaders()) + self.assertEquals(headers.get('x-storage-policy'), + default_policy['name']) + + def test_error_invalid_storage_policy_name(self): + def put(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('PUT', parsed.path + '/' + self.container, '', + new_headers) + return check_response(conn) + + # create + resp = retry(put, {'X-Storage-Policy': uuid4().hex}) + resp.read() + self.assertEqual(resp.status, 400) + + @requires_policies + def test_create_non_default_storage_policy_container(self): + policy = self.policies.exclude(default=True).select() + + def put(url, token, parsed, conn, headers=None): + base_headers = {'X-Auth-Token': token} + if headers: + base_headers.update(headers) + conn.request('PUT', parsed.path + '/' + self.container, '', + base_headers) + return check_response(conn) + headers = {'X-Storage-Policy': policy['name']} + resp = retry(put, headers=headers) + resp.read() + self.assertEqual(resp.status, 201) + + def head(url, token, parsed, conn): + conn.request('HEAD', parsed.path + '/' + self.container, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(head) + resp.read() + headers = dict((k.lower(), v) for k, v in resp.getheaders()) + self.assertEquals(headers.get('x-storage-policy'), + policy['name']) + + # and test recreate with-out specifying Storage Policy + resp = retry(put) + resp.read() + self.assertEqual(resp.status, 202) + # should still be original storage policy + resp = retry(head) + resp.read() + headers = dict((k.lower(), v) for k, v in resp.getheaders()) + self.assertEquals(headers.get('x-storage-policy'), + policy['name']) + + # delete it + def delete(url, token, parsed, conn): + conn.request('DELETE', parsed.path + '/' + self.container, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(delete) + resp.read() + self.assertEqual(resp.status, 204) + + # verify no policy header + resp = retry(head) + resp.read() + headers = dict((k.lower(), v) for k, v in resp.getheaders()) + self.assertEquals(headers.get('x-storage-policy'), None) + + @requires_policies + def test_conflict_change_storage_policy_with_put(self): + def put(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('PUT', parsed.path + '/' + self.container, '', + new_headers) + return check_response(conn) + + # create + policy = self.policies.select() + resp = retry(put, {'X-Storage-Policy': policy['name']}) + resp.read() + self.assertEqual(resp.status, 201) + + # can't change it + other_policy = self.policies.exclude(name=policy['name']).select() + resp = retry(put, {'X-Storage-Policy': other_policy['name']}) + resp.read() + self.assertEqual(resp.status, 409) + + def head(url, token, parsed, conn): + conn.request('HEAD', parsed.path + '/' + self.container, '', + {'X-Auth-Token': token}) + return check_response(conn) + # still original policy + resp = retry(head) + resp.read() + headers = dict((k.lower(), v) for k, v in resp.getheaders()) + self.assertEquals(headers.get('x-storage-policy'), + policy['name']) + + @requires_policies + def test_noop_change_storage_policy_with_post(self): + def put(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('PUT', parsed.path + '/' + self.container, '', + new_headers) + return check_response(conn) + + # create + policy = self.policies.select() + resp = retry(put, {'X-Storage-Policy': policy['name']}) + resp.read() + self.assertEqual(resp.status, 201) + + def post(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path + '/' + self.container, '', + new_headers) + return check_response(conn) + # attempt update + for header in ('X-Storage-Policy', 'X-Storage-Policy-Index'): + other_policy = self.policies.exclude(name=policy['name']).select() + resp = retry(post, {header: other_policy['name']}) + resp.read() + self.assertEqual(resp.status, 204) + + def head(url, token, parsed, conn): + conn.request('HEAD', parsed.path + '/' + self.container, '', + {'X-Auth-Token': token}) + return check_response(conn) + # still original policy + resp = retry(head) + resp.read() + headers = dict((k.lower(), v) for k, v in resp.getheaders()) + self.assertEquals(headers.get('x-storage-policy'), + policy['name']) + + +class BaseTestContainerACLs(unittest.TestCase): + # subclasses can change the account in which container + # is created/deleted by setUp/tearDown + account = 1 + + def _get_account(self, url, token, parsed, conn): + return parsed.path + + def _get_tenant_id(self, url, token, parsed, conn): + account = parsed.path + return account.replace('/v1/AUTH_', '', 1) + + def setUp(self): + if tf.skip or tf.skip2 or tf.skip_if_not_v3: + raise SkipTest('AUTH VERSION 3 SPECIFIC TEST') + self.name = uuid4().hex + + def put(url, token, parsed, conn): + conn.request('PUT', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + resp = retry(put, use_account=self.account) + resp.read() + self.assertEqual(resp.status, 201) + + def tearDown(self): + if tf.skip or tf.skip2 or tf.skip_if_not_v3: + raise SkipTest + + def get(url, token, parsed, conn): + conn.request('GET', parsed.path + '/' + self.name + '?format=json', + '', {'X-Auth-Token': token}) + return check_response(conn) + + def delete(url, token, parsed, conn, obj): + conn.request('DELETE', + '/'.join([parsed.path, self.name, obj['name']]), '', + {'X-Auth-Token': token}) + return check_response(conn) + + while True: + resp = retry(get, use_account=self.account) + body = resp.read() + self.assert_(resp.status // 100 == 2, resp.status) + objs = json.loads(body) + if not objs: + break + for obj in objs: + resp = retry(delete, obj, use_account=self.account) + resp.read() + self.assertEqual(resp.status, 204) + + def delete(url, token, parsed, conn): + conn.request('DELETE', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + resp = retry(delete, use_account=self.account) + resp.read() + self.assertEqual(resp.status, 204) + + def _assert_cross_account_acl_granted(self, granted, grantee_account, acl): + ''' + Check whether a given container ACL is granted when a user specified + by account_b attempts to access a container. + ''' + # Obtain the first account's string + first_account = retry(self._get_account, use_account=self.account) + + # Ensure we can't access the container with the grantee account + def get2(url, token, parsed, conn): + conn.request('GET', first_account + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + resp = retry(get2, use_account=grantee_account) + resp.read() + self.assertEqual(resp.status, 403) + + def put2(url, token, parsed, conn): + conn.request('PUT', first_account + '/' + self.name + '/object', + 'test object', {'X-Auth-Token': token}) + return check_response(conn) + + resp = retry(put2, use_account=grantee_account) + resp.read() + self.assertEqual(resp.status, 403) + + # Post ACL to the container + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, + 'X-Container-Read': acl, + 'X-Container-Write': acl}) + return check_response(conn) + + resp = retry(post, use_account=self.account) + resp.read() + self.assertEqual(resp.status, 204) + + # Check access to container from grantee account with ACL in place + resp = retry(get2, use_account=grantee_account) + resp.read() + expected = 204 if granted else 403 + self.assertEqual(resp.status, expected) + + resp = retry(put2, use_account=grantee_account) + resp.read() + expected = 201 if granted else 403 + self.assertEqual(resp.status, expected) + + # Make the container private again + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, 'X-Container-Read': '', + 'X-Container-Write': ''}) + return check_response(conn) + + resp = retry(post, use_account=self.account) + resp.read() + self.assertEqual(resp.status, 204) + + # Ensure we can't access the container with the grantee account again + resp = retry(get2, use_account=grantee_account) + resp.read() + self.assertEqual(resp.status, 403) + + resp = retry(put2, use_account=grantee_account) + resp.read() + self.assertEqual(resp.status, 403) + + +class TestContainerACLsAccount1(BaseTestContainerACLs): + def test_cross_account_acl_names_with_user_in_non_default_domain(self): + # names in acls are disallowed when grantee is in a non-default domain + acl = '%s:%s' % (tf.swift_test_tenant[3], tf.swift_test_user[3]) + self._assert_cross_account_acl_granted(False, 4, acl) + + def test_cross_account_acl_ids_with_user_in_non_default_domain(self): + # ids are allowed in acls when grantee is in a non-default domain + tenant_id = retry(self._get_tenant_id, use_account=4) + acl = '%s:%s' % (tenant_id, '*') + self._assert_cross_account_acl_granted(True, 4, acl) + + def test_cross_account_acl_names_in_default_domain(self): + # names are allowed in acls when grantee and project are in + # the default domain + acl = '%s:%s' % (tf.swift_test_tenant[1], tf.swift_test_user[1]) + self._assert_cross_account_acl_granted(True, 2, acl) + + def test_cross_account_acl_ids_in_default_domain(self): + # ids are allowed in acls when grantee and project are in + # the default domain + tenant_id = retry(self._get_tenant_id, use_account=2) + acl = '%s:%s' % (tenant_id, '*') + self._assert_cross_account_acl_granted(True, 2, acl) + + +class TestContainerACLsAccount4(BaseTestContainerACLs): + account = 4 + + def test_cross_account_acl_names_with_project_in_non_default_domain(self): + # names in acls are disallowed when project is in a non-default domain + acl = '%s:%s' % (tf.swift_test_tenant[0], tf.swift_test_user[0]) + self._assert_cross_account_acl_granted(False, 1, acl) + + def test_cross_account_acl_ids_with_project_in_non_default_domain(self): + # ids are allowed in acls when project is in a non-default domain + tenant_id = retry(self._get_tenant_id, use_account=1) + acl = '%s:%s' % (tenant_id, '*') + self._assert_cross_account_acl_granted(True, 1, acl) + if __name__ == '__main__': unittest.main() diff --git a/test/functional/test_object.py b/test/functional/test_object.py index 675de30..e74a7f6 100755 --- a/test/functional/test_object.py +++ b/test/functional/test_object.py @@ -21,24 +21,22 @@ from uuid import uuid4 from swift.common.utils import json -from swift_testing import check_response, retry, skip, skip3, \ - swift_test_perm, web_front_end, requires_acls, swift_test_user +from test.functional import check_response, retry, requires_acls, \ + requires_policies +import test.functional as tf class TestObject(unittest.TestCase): def setUp(self): - if skip: + if tf.skip: raise SkipTest self.container = uuid4().hex - def put(url, token, parsed, conn): - conn.request('PUT', parsed.path + '/' + self.container, '', - {'X-Auth-Token': token}) - return check_response(conn) - resp = retry(put) - resp.read() - self.assertEqual(resp.status, 201) + self.containers = [] + self._create_container(self.container) + self._create_container(self.container, use_account=2) + self.obj = uuid4().hex def put(url, token, parsed, conn): @@ -50,40 +48,65 @@ class TestObject(unittest.TestCase): resp.read() self.assertEqual(resp.status, 201) + def _create_container(self, name=None, headers=None, use_account=1): + if not name: + name = uuid4().hex + self.containers.append(name) + headers = headers or {} + + def put(url, token, parsed, conn, name): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('PUT', parsed.path + '/' + name, '', + new_headers) + return check_response(conn) + resp = retry(put, name, use_account=use_account) + resp.read() + self.assertEqual(resp.status, 201) + return name + def tearDown(self): - if skip: + if tf.skip: raise SkipTest - def delete(url, token, parsed, conn, obj): - conn.request('DELETE', - '%s/%s/%s' % (parsed.path, self.container, obj), - '', {'X-Auth-Token': token}) + # get list of objects in container + def get(url, token, parsed, conn, container): + conn.request( + 'GET', parsed.path + '/' + container + '?format=json', '', + {'X-Auth-Token': token}) return check_response(conn) - # get list of objects in container - def list(url, token, parsed, conn): - conn.request('GET', - '%s/%s' % (parsed.path, self.container), - '', {'X-Auth-Token': token}) + # delete an object + def delete(url, token, parsed, conn, container, obj): + conn.request( + 'DELETE', '/'.join([parsed.path, container, obj['name']]), '', + {'X-Auth-Token': token}) return check_response(conn) - resp = retry(list) - object_listing = resp.read() - self.assertEqual(resp.status, 200) - # iterate over object listing and delete all objects - for obj in object_listing.splitlines(): - resp = retry(delete, obj) - resp.read() - self.assertEqual(resp.status, 204) + for container in self.containers: + while True: + resp = retry(get, container) + body = resp.read() + if resp.status == 404: + break + self.assert_(resp.status // 100 == 2, resp.status) + objs = json.loads(body) + if not objs: + break + for obj in objs: + resp = retry(delete, container, obj) + resp.read() + self.assertEqual(resp.status, 204) # delete the container - def delete(url, token, parsed, conn): - conn.request('DELETE', parsed.path + '/' + self.container, '', + def delete(url, token, parsed, conn, name): + conn.request('DELETE', parsed.path + '/' + name, '', {'X-Auth-Token': token}) return check_response(conn) - resp = retry(delete) - resp.read() - self.assertEqual(resp.status, 204) + + for container in self.containers: + resp = retry(delete, container) + resp.read() + self.assert_(resp.status in (204, 404)) def test_if_none_match(self): def put(url, token, parsed, conn): @@ -111,8 +134,47 @@ class TestObject(unittest.TestCase): resp.read() self.assertEquals(resp.status, 400) + def test_non_integer_x_delete_after(self): + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s/%s' % (parsed.path, self.container, + 'non_integer_x_delete_after'), + '', {'X-Auth-Token': token, + 'Content-Length': '0', + 'X-Delete-After': '*'}) + return check_response(conn) + resp = retry(put) + body = resp.read() + self.assertEquals(resp.status, 400) + self.assertEqual(body, 'Non-integer X-Delete-After') + + def test_non_integer_x_delete_at(self): + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s/%s' % (parsed.path, self.container, + 'non_integer_x_delete_at'), + '', {'X-Auth-Token': token, + 'Content-Length': '0', + 'X-Delete-At': '*'}) + return check_response(conn) + resp = retry(put) + body = resp.read() + self.assertEquals(resp.status, 400) + self.assertEqual(body, 'Non-integer X-Delete-At') + + def test_x_delete_at_in_the_past(self): + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s/%s' % (parsed.path, self.container, + 'x_delete_at_in_the_past'), + '', {'X-Auth-Token': token, + 'Content-Length': '0', + 'X-Delete-At': '0'}) + return check_response(conn) + resp = retry(put) + body = resp.read() + self.assertEquals(resp.status, 400) + self.assertEqual(body, 'X-Delete-At in past') + def test_copy_object(self): - if skip: + if tf.skip: raise SkipTest source = '%s/%s' % (self.container, self.obj) @@ -185,8 +247,118 @@ class TestObject(unittest.TestCase): resp.read() self.assertEqual(resp.status, 204) + def test_copy_between_accounts(self): + if tf.skip: + raise SkipTest + + source = '%s/%s' % (self.container, self.obj) + dest = '%s/%s' % (self.container, 'test_copy') + + # get contents of source + def get_source(url, token, parsed, conn): + conn.request('GET', + '%s/%s' % (parsed.path, source), + '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get_source) + source_contents = resp.read() + self.assertEqual(resp.status, 200) + self.assertEqual(source_contents, 'test') + + acct = tf.parsed[0].path.split('/', 2)[2] + + # copy source to dest with X-Copy-From-Account + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s' % (parsed.path, dest), '', + {'X-Auth-Token': token, + 'Content-Length': '0', + 'X-Copy-From-Account': acct, + 'X-Copy-From': source}) + return check_response(conn) + # try to put, will not succeed + # user does not have permissions to read from source + resp = retry(put, use_account=2) + self.assertEqual(resp.status, 403) + + # add acl to allow reading from source + def post(url, token, parsed, conn): + conn.request('POST', '%s/%s' % (parsed.path, self.container), '', + {'X-Auth-Token': token, + 'X-Container-Read': tf.swift_test_perm[1]}) + return check_response(conn) + resp = retry(post) + self.assertEqual(resp.status, 204) + + # retry previous put, now should succeed + resp = retry(put, use_account=2) + self.assertEqual(resp.status, 201) + + # contents of dest should be the same as source + def get_dest(url, token, parsed, conn): + conn.request('GET', + '%s/%s' % (parsed.path, dest), + '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get_dest, use_account=2) + dest_contents = resp.read() + self.assertEqual(resp.status, 200) + self.assertEqual(dest_contents, source_contents) + + # delete the copy + def delete(url, token, parsed, conn): + conn.request('DELETE', '%s/%s' % (parsed.path, dest), '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(delete, use_account=2) + resp.read() + self.assertEqual(resp.status, 204) + # verify dest does not exist + resp = retry(get_dest, use_account=2) + resp.read() + self.assertEqual(resp.status, 404) + + acct_dest = tf.parsed[1].path.split('/', 2)[2] + + # copy source to dest with COPY + def copy(url, token, parsed, conn): + conn.request('COPY', '%s/%s' % (parsed.path, source), '', + {'X-Auth-Token': token, + 'Destination-Account': acct_dest, + 'Destination': dest}) + return check_response(conn) + # try to copy, will not succeed + # user does not have permissions to write to destination + resp = retry(copy) + resp.read() + self.assertEqual(resp.status, 403) + + # add acl to allow write to destination + def post(url, token, parsed, conn): + conn.request('POST', '%s/%s' % (parsed.path, self.container), '', + {'X-Auth-Token': token, + 'X-Container-Write': tf.swift_test_perm[0]}) + return check_response(conn) + resp = retry(post, use_account=2) + self.assertEqual(resp.status, 204) + + # now copy will succeed + resp = retry(copy) + resp.read() + self.assertEqual(resp.status, 201) + + # contents of dest should be the same as source + resp = retry(get_dest, use_account=2) + dest_contents = resp.read() + self.assertEqual(resp.status, 200) + self.assertEqual(dest_contents, source_contents) + + # delete the copy + resp = retry(delete, use_account=2) + resp.read() + self.assertEqual(resp.status, 204) + def test_public_object(self): - if skip: + if tf.skip: raise SkipTest def get(url, token, parsed, conn): @@ -225,7 +397,7 @@ class TestObject(unittest.TestCase): self.assert_(str(err).startswith('No result after ')) def test_private_object(self): - if skip or skip3: + if tf.skip or tf.skip3: raise SkipTest # Ensure we can't access the object with the third account @@ -245,8 +417,8 @@ class TestObject(unittest.TestCase): conn.request('PUT', '%s/%s' % ( parsed.path, shared_container), '', {'X-Auth-Token': token, - 'X-Container-Read': swift_test_perm[2], - 'X-Container-Write': swift_test_perm[2]}) + 'X-Container-Read': tf.swift_test_perm[2], + 'X-Container-Write': tf.swift_test_perm[2]}) return check_response(conn) resp = retry(put) resp.read() @@ -319,8 +491,8 @@ class TestObject(unittest.TestCase): @requires_acls def test_read_only(self): - if skip3: - raise SkipTest + if tf.skip3: + raise tf.SkipTest def get_listing(url, token, parsed, conn): conn.request('GET', '%s/%s' % (parsed.path, self.container), '', @@ -361,7 +533,7 @@ class TestObject(unittest.TestCase): self.assertEquals(resp.status, 403) # grant read-only access - acl_user = swift_test_user[2] + acl_user = tf.swift_test_user[2] acl = {'read-only': [acl_user]} headers = {'x-account-access-control': json.dumps(acl)} resp = retry(post_account, headers=headers, use_account=1) @@ -400,7 +572,7 @@ class TestObject(unittest.TestCase): @requires_acls def test_read_write(self): - if skip3: + if tf.skip3: raise SkipTest def get_listing(url, token, parsed, conn): @@ -442,7 +614,7 @@ class TestObject(unittest.TestCase): self.assertEquals(resp.status, 403) # grant read-write access - acl_user = swift_test_user[2] + acl_user = tf.swift_test_user[2] acl = {'read-write': [acl_user]} headers = {'x-account-access-control': json.dumps(acl)} resp = retry(post_account, headers=headers, use_account=1) @@ -481,7 +653,7 @@ class TestObject(unittest.TestCase): @requires_acls def test_admin(self): - if skip3: + if tf.skip3: raise SkipTest def get_listing(url, token, parsed, conn): @@ -523,7 +695,7 @@ class TestObject(unittest.TestCase): self.assertEquals(resp.status, 403) # grant admin access - acl_user = swift_test_user[2] + acl_user = tf.swift_test_user[2] acl = {'admin': [acl_user]} headers = {'x-account-access-control': json.dumps(acl)} resp = retry(post_account, headers=headers, use_account=1) @@ -561,7 +733,7 @@ class TestObject(unittest.TestCase): self.assert_(self.obj not in listing) def test_manifest(self): - if skip: + if tf.skip: raise SkipTest # Data for the object segments segments1 = ['one', 'two', 'three', 'four', 'five'] @@ -672,7 +844,7 @@ class TestObject(unittest.TestCase): self.assertEqual(resp.read(), ''.join(segments2)) self.assertEqual(resp.status, 200) - if not skip3: + if not tf.skip3: # Ensure we can't access the manifest with the third account def get(url, token, parsed, conn): @@ -687,7 +859,7 @@ class TestObject(unittest.TestCase): def post(url, token, parsed, conn): conn.request('POST', '%s/%s' % (parsed.path, self.container), '', {'X-Auth-Token': token, - 'X-Container-Read': swift_test_perm[2]}) + 'X-Container-Read': tf.swift_test_perm[2]}) return check_response(conn) resp = retry(post) resp.read() @@ -745,7 +917,7 @@ class TestObject(unittest.TestCase): self.assertEqual(resp.read(), ''.join(segments3)) self.assertEqual(resp.status, 200) - if not skip3: + if not tf.skip3: # Ensure we can't access the manifest with the third account # (because the segments are in a protected container even if the @@ -763,7 +935,7 @@ class TestObject(unittest.TestCase): def post(url, token, parsed, conn): conn.request('POST', '%s/%s' % (parsed.path, acontainer), '', {'X-Auth-Token': token, - 'X-Container-Read': swift_test_perm[2]}) + 'X-Container-Read': tf.swift_test_perm[2]}) return check_response(conn) resp = retry(post) resp.read() @@ -831,7 +1003,7 @@ class TestObject(unittest.TestCase): self.assertEqual(resp.status, 204) def test_delete_content_type(self): - if skip: + if tf.skip: raise SkipTest def put(url, token, parsed, conn): @@ -853,7 +1025,7 @@ class TestObject(unittest.TestCase): 'text/html; charset=UTF-8') def test_delete_if_delete_at_bad(self): - if skip: + if tf.skip: raise SkipTest def put(url, token, parsed, conn): @@ -875,7 +1047,7 @@ class TestObject(unittest.TestCase): self.assertEqual(resp.status, 400) def test_null_name(self): - if skip: + if tf.skip: raise SkipTest def put(url, token, parsed, conn): @@ -884,23 +1056,20 @@ class TestObject(unittest.TestCase): self.container), 'test', {'X-Auth-Token': token}) return check_response(conn) resp = retry(put) - if (web_front_end == 'apache2'): + if (tf.web_front_end == 'apache2'): self.assertEqual(resp.status, 404) else: self.assertEqual(resp.read(), 'Invalid UTF8 or contains NULL') self.assertEqual(resp.status, 412) def test_cors(self): - if skip: + if tf.skip: raise SkipTest - def is_strict_mode(url, token, parsed, conn): - conn.request('GET', '/info') - resp = conn.getresponse() - if resp.status // 100 == 2: - info = json.loads(resp.read()) - return info.get('swift', {}).get('strict_cors_mode', False) - return False + try: + strict_cors = tf.cluster_info['swift']['strict_cors_mode'] + except KeyError: + raise SkipTest("cors mode is unknown") def put_cors_cont(url, token, parsed, conn, orig): conn.request( @@ -924,8 +1093,6 @@ class TestObject(unittest.TestCase): '', headers) return conn.getresponse() - strict_cors = retry(is_strict_mode) - resp = retry(put_cors_cont, '*') resp.read() self.assertEquals(resp.status // 100, 2) @@ -1001,6 +1168,64 @@ class TestObject(unittest.TestCase): self.assertEquals(headers.get('access-control-allow-origin'), 'http://m.com') + @requires_policies + def test_cross_policy_copy(self): + # create container in first policy + policy = self.policies.select() + container = self._create_container( + headers={'X-Storage-Policy': policy['name']}) + obj = uuid4().hex + + # create a container in second policy + other_policy = self.policies.exclude(name=policy['name']).select() + other_container = self._create_container( + headers={'X-Storage-Policy': other_policy['name']}) + other_obj = uuid4().hex + + def put_obj(url, token, parsed, conn, container, obj): + # to keep track of things, use the original path as the body + content = '%s/%s' % (container, obj) + path = '%s/%s' % (parsed.path, content) + conn.request('PUT', path, content, {'X-Auth-Token': token}) + return check_response(conn) + + # create objects + for c, o in zip((container, other_container), (obj, other_obj)): + resp = retry(put_obj, c, o) + resp.read() + self.assertEqual(resp.status, 201) + + def put_copy_from(url, token, parsed, conn, container, obj, source): + dest_path = '%s/%s/%s' % (parsed.path, container, obj) + conn.request('PUT', dest_path, '', + {'X-Auth-Token': token, + 'Content-Length': '0', + 'X-Copy-From': source}) + return check_response(conn) + + copy_requests = ( + (container, other_obj, '%s/%s' % (other_container, other_obj)), + (other_container, obj, '%s/%s' % (container, obj)), + ) + + # copy objects + for c, o, source in copy_requests: + resp = retry(put_copy_from, c, o, source) + resp.read() + self.assertEqual(resp.status, 201) + + def get_obj(url, token, parsed, conn, container, obj): + path = '%s/%s/%s' % (parsed.path, container, obj) + conn.request('GET', path, '', {'X-Auth-Token': token}) + return check_response(conn) + + # validate contents, contents should be source + validate_requests = copy_requests + for c, o, body in validate_requests: + resp = retry(get_obj, c, o) + self.assertEqual(resp.status, 200) + self.assertEqual(body, resp.read()) + if __name__ == '__main__': unittest.main() diff --git a/test/functional/tests.py b/test/functional/tests.py index b8633b0..daa8897 100644 --- a/test/functional/tests.py +++ b/test/functional/tests.py @@ -14,10 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Modifications by Red Hat, Inc. - from datetime import datetime -import os import hashlib import hmac import json @@ -25,72 +22,25 @@ import locale import random import StringIO import time -import threading +import os import unittest import urllib import uuid +from copy import deepcopy +import eventlet from nose import SkipTest -from ConfigParser import ConfigParser +from swift.common.http import is_success, is_client_error -from test import get_config +from test.functional import normalized_urls, load_constraint, cluster_info +from test.functional import check_response, retry +import test.functional as tf from test.functional.swift_test_client import Account, Connection, File, \ ResponseError -from swift.common.constraints import MAX_FILE_SIZE, MAX_META_NAME_LENGTH, \ - MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE, \ - MAX_OBJECT_NAME_LENGTH, CONTAINER_LISTING_LIMIT, ACCOUNT_LISTING_LIMIT, \ - MAX_ACCOUNT_NAME_LENGTH, MAX_CONTAINER_NAME_LENGTH, MAX_HEADER_SIZE from gluster.swift.common.constraints import \ set_object_name_component_length, get_object_name_component_length -default_constraints = dict(( - ('max_file_size', MAX_FILE_SIZE), - ('max_meta_name_length', MAX_META_NAME_LENGTH), - ('max_meta_value_length', MAX_META_VALUE_LENGTH), - ('max_meta_count', MAX_META_COUNT), - ('max_meta_overall_size', MAX_META_OVERALL_SIZE), - ('max_object_name_length', MAX_OBJECT_NAME_LENGTH), - ('container_listing_limit', CONTAINER_LISTING_LIMIT), - ('account_listing_limit', ACCOUNT_LISTING_LIMIT), - ('max_account_name_length', MAX_ACCOUNT_NAME_LENGTH), - ('max_container_name_length', MAX_CONTAINER_NAME_LENGTH), - ('max_header_size', MAX_HEADER_SIZE))) -constraints_conf = ConfigParser() -conf_exists = constraints_conf.read('/etc/swift/swift.conf') -# Constraints are set first from the test config, then from -# /etc/swift/swift.conf if it exists. If swift.conf doesn't exist, -# then limit test coverage. This allows SAIO tests to work fine but -# requires remote functional testing to know something about the cluster -# that is being tested. -config = get_config('func_test') -for k in default_constraints: - if k in config: - # prefer what's in test.conf - config[k] = int(config[k]) - elif conf_exists: - # swift.conf exists, so use what's defined there (or swift defaults) - # This normally happens when the test is running locally to the cluster - # as in a SAIO. - config[k] = default_constraints[k] - else: - # .functests don't know what the constraints of the tested cluster are, - # so the tests can't reliably pass or fail. Therefore, skip those - # tests. - config[k] = '%s constraint is not defined' % k - -web_front_end = config.get('web_front_end', 'integral') -normalized_urls = config.get('normalized_urls', False) set_object_name_component_length() - -def load_constraint(name): - c = config[name] - if not isinstance(c, int): - raise SkipTest(c) - return c - -locale.setlocale(locale.LC_COLLATE, config.get('collate', 'C')) - - def create_limit_filename(name_limit): """ Convert a split a large object name with @@ -116,42 +66,6 @@ def create_limit_filename(name_limit): return "".join(filename_list) -def chunks(s, length=3): - i, j = 0, length - while i < len(s): - yield s[i:j] - i, j = j, j + length - - -def timeout(seconds, method, *args, **kwargs): - class TimeoutThread(threading.Thread): - def __init__(self, method, *args, **kwargs): - threading.Thread.__init__(self) - - self.method = method - self.args = args - self.kwargs = kwargs - self.exception = None - - def run(self): - try: - self.method(*self.args, **self.kwargs) - except Exception as e: - self.exception = e - - t = TimeoutThread(method, *args, **kwargs) - t.start() - t.join(seconds) - - if t.exception: - raise t.exception - - if t.isAlive(): - t._Thread__stop() - return True - return False - - class Utils(object): @classmethod def create_ascii_name(cls, length=None): @@ -207,10 +121,10 @@ class Base2(object): class TestAccountEnv(object): @classmethod def setUp(cls): - cls.conn = Connection(config) + cls.conn = Connection(tf.config) cls.conn.authenticate() - cls.account = Account(cls.conn, config.get('account', - config['username'])) + cls.account = Account(cls.conn, tf.config.get('account', + tf.config['username'])) cls.account.delete_containers() cls.containers = [] @@ -386,6 +300,28 @@ class TestAccount(Base): self.assertEqual(sorted(containers, cmp=locale.strcoll), containers) + def testQuotedWWWAuthenticateHeader(self): + # check that the www-authenticate header value with the swift realm + # is correctly quoted. + conn = Connection(tf.config) + conn.authenticate() + inserted_html = '<b>Hello World' + hax = 'AUTH_haxx"\nContent-Length: %d\n\n%s' % (len(inserted_html), + inserted_html) + quoted_hax = urllib.quote(hax) + conn.connection.request('GET', '/v1/' + quoted_hax, None, {}) + resp = conn.connection.getresponse() + resp_headers = dict(resp.getheaders()) + self.assertTrue('www-authenticate' in resp_headers, + 'www-authenticate not found in %s' % resp_headers) + actual = resp_headers['www-authenticate'] + expected = 'Swift realm="%s"' % quoted_hax + # other middleware e.g. auth_token may also set www-authenticate + # headers in which case actual values will be a comma separated list. + # check that expected value is among the actual values + self.assertTrue(expected in actual, + '%s not found in %s' % (expected, actual)) + class TestAccountUTF8(Base2, TestAccount): set_up = False @@ -394,10 +330,10 @@ class TestAccountUTF8(Base2, TestAccount): class TestAccountNoContainersEnv(object): @classmethod def setUp(cls): - cls.conn = Connection(config) + cls.conn = Connection(tf.config) cls.conn.authenticate() - cls.account = Account(cls.conn, config.get('account', - config['username'])) + cls.account = Account(cls.conn, tf.config.get('account', + tf.config['username'])) cls.account.delete_containers() @@ -423,10 +359,10 @@ class TestAccountNoContainersUTF8(Base2, TestAccountNoContainers): class TestContainerEnv(object): @classmethod def setUp(cls): - cls.conn = Connection(config) + cls.conn = Connection(tf.config) cls.conn.authenticate() - cls.account = Account(cls.conn, config.get('account', - config['username'])) + cls.account = Account(cls.conn, tf.config.get('account', + tf.config['username'])) cls.account.delete_containers() cls.container = cls.account.container(Utils.create_name()) @@ -714,11 +650,11 @@ class TestContainerUTF8(Base2, TestContainer): class TestContainerPathsEnv(object): @classmethod def setUp(cls): - raise SkipTest('Objects ending in / are not supported') - cls.conn = Connection(config) + raise SkipTest('Objects ending in / are not supported') + cls.conn = Connection(tf.config) cls.conn.authenticate() - cls.account = Account(cls.conn, config.get('account', - config['username'])) + cls.account = Account(cls.conn, tf.config.get('account', + tf.config['username'])) cls.account.delete_containers() cls.file_size = 8 @@ -894,11 +830,24 @@ class TestContainerPaths(Base): class TestFileEnv(object): @classmethod def setUp(cls): - cls.conn = Connection(config) + cls.conn = Connection(tf.config) cls.conn.authenticate() - cls.account = Account(cls.conn, config.get('account', - config['username'])) + cls.account = Account(cls.conn, tf.config.get('account', + tf.config['username'])) + # creating another account and connection + # for account to account copy tests + config2 = deepcopy(tf.config) + config2['account'] = tf.config['account2'] + config2['username'] = tf.config['username2'] + config2['password'] = tf.config['password2'] + cls.conn2 = Connection(config2) + cls.conn2.authenticate() + + cls.account = Account(cls.conn, tf.config.get('account', + tf.config['username'])) cls.account.delete_containers() + cls.account2 = cls.conn2.get_account() + cls.account2.delete_containers() cls.container = cls.account.container(Utils.create_name()) if not cls.container.create(): @@ -952,6 +901,62 @@ class TestFile(Base): self.assert_(file_item.initialize()) self.assert_(metadata == file_item.metadata) + def testCopyAccount(self): + # makes sure to test encoded characters + source_filename = 'dealde%2Fl04 011e%204c8df/flash.png' + file_item = self.env.container.file(source_filename) + + metadata = {Utils.create_ascii_name(): Utils.create_name()} + + data = file_item.write_random() + file_item.sync_metadata(metadata) + + dest_cont = self.env.account.container(Utils.create_name()) + self.assert_(dest_cont.create()) + + acct = self.env.conn.account_name + # copy both from within and across containers + for cont in (self.env.container, dest_cont): + # copy both with and without initial slash + for prefix in ('', '/'): + dest_filename = Utils.create_name() + + file_item = self.env.container.file(source_filename) + file_item.copy_account(acct, + '%s%s' % (prefix, cont), + dest_filename) + + self.assert_(dest_filename in cont.files()) + + file_item = cont.file(dest_filename) + + self.assert_(data == file_item.read()) + self.assert_(file_item.initialize()) + self.assert_(metadata == file_item.metadata) + + dest_cont = self.env.account2.container(Utils.create_name()) + self.assert_(dest_cont.create(hdrs={ + 'X-Container-Write': self.env.conn.user_acl + })) + + acct = self.env.conn2.account_name + # copy both with and without initial slash + for prefix in ('', '/'): + dest_filename = Utils.create_name() + + file_item = self.env.container.file(source_filename) + file_item.copy_account(acct, + '%s%s' % (prefix, dest_cont), + dest_filename) + + self.assert_(dest_filename in dest_cont.files()) + + file_item = dest_cont.file(dest_filename) + + self.assert_(data == file_item.read()) + self.assert_(file_item.initialize()) + self.assert_(metadata == file_item.metadata) + def testCopy404s(self): source_filename = Utils.create_name() file_item = self.env.container.file(source_filename) @@ -990,6 +995,77 @@ class TestFile(Base): '%s%s' % (prefix, Utils.create_name()), Utils.create_name())) + def testCopyAccount404s(self): + acct = self.env.conn.account_name + acct2 = self.env.conn2.account_name + source_filename = Utils.create_name() + file_item = self.env.container.file(source_filename) + file_item.write_random() + + dest_cont = self.env.account.container(Utils.create_name()) + self.assert_(dest_cont.create(hdrs={ + 'X-Container-Read': self.env.conn2.user_acl + })) + dest_cont2 = self.env.account2.container(Utils.create_name()) + self.assert_(dest_cont2.create(hdrs={ + 'X-Container-Write': self.env.conn.user_acl, + 'X-Container-Read': self.env.conn.user_acl + })) + + for acct, cont in ((acct, dest_cont), (acct2, dest_cont2)): + for prefix in ('', '/'): + # invalid source container + source_cont = self.env.account.container(Utils.create_name()) + file_item = source_cont.file(source_filename) + self.assert_(not file_item.copy_account( + acct, + '%s%s' % (prefix, self.env.container), + Utils.create_name())) + if acct == acct2: + # there is no such source container + # and foreign user can have no permission to read it + self.assert_status(403) + else: + self.assert_status(404) + + self.assert_(not file_item.copy_account( + acct, + '%s%s' % (prefix, cont), + Utils.create_name())) + self.assert_status(404) + + # invalid source object + file_item = self.env.container.file(Utils.create_name()) + self.assert_(not file_item.copy_account( + acct, + '%s%s' % (prefix, self.env.container), + Utils.create_name())) + if acct == acct2: + # there is no such object + # and foreign user can have no permission to read it + self.assert_status(403) + else: + self.assert_status(404) + + self.assert_(not file_item.copy_account( + acct, + '%s%s' % (prefix, cont), + Utils.create_name())) + self.assert_status(404) + + # invalid destination container + file_item = self.env.container.file(source_filename) + self.assert_(not file_item.copy_account( + acct, + '%s%s' % (prefix, Utils.create_name()), + Utils.create_name())) + if acct == acct2: + # there is no such destination container + # and foreign user can have no permission to write there + self.assert_status(403) + else: + self.assert_status(404) + def testCopyNoDestinationHeader(self): source_filename = Utils.create_name() file_item = self.env.container.file(source_filename) @@ -1044,6 +1120,49 @@ class TestFile(Base): self.assert_(file_item.initialize()) self.assert_(metadata == file_item.metadata) + def testCopyFromAccountHeader(self): + acct = self.env.conn.account_name + src_cont = self.env.account.container(Utils.create_name()) + self.assert_(src_cont.create(hdrs={ + 'X-Container-Read': self.env.conn2.user_acl + })) + source_filename = Utils.create_name() + file_item = src_cont.file(source_filename) + + metadata = {} + for i in range(1): + metadata[Utils.create_ascii_name()] = Utils.create_name() + file_item.metadata = metadata + + data = file_item.write_random() + + dest_cont = self.env.account.container(Utils.create_name()) + self.assert_(dest_cont.create()) + dest_cont2 = self.env.account2.container(Utils.create_name()) + self.assert_(dest_cont2.create(hdrs={ + 'X-Container-Write': self.env.conn.user_acl + })) + + for cont in (src_cont, dest_cont, dest_cont2): + # copy both with and without initial slash + for prefix in ('', '/'): + dest_filename = Utils.create_name() + + file_item = cont.file(dest_filename) + file_item.write(hdrs={'X-Copy-From-Account': acct, + 'X-Copy-From': '%s%s/%s' % ( + prefix, + src_cont.name, + source_filename)}) + + self.assert_(dest_filename in cont.files()) + + file_item = cont.file(dest_filename) + + self.assert_(data == file_item.read()) + self.assert_(file_item.initialize()) + self.assert_(metadata == file_item.metadata) + def testCopyFromHeader404s(self): source_filename = Utils.create_name() file_item = self.env.container.file(source_filename) @@ -1075,6 +1194,52 @@ class TestFile(Base): self.env.container.name, source_filename)}) self.assert_status(404) + def testCopyFromAccountHeader404s(self): + acct = self.env.conn2.account_name + src_cont = self.env.account2.container(Utils.create_name()) + self.assert_(src_cont.create(hdrs={ + 'X-Container-Read': self.env.conn.user_acl + })) + source_filename = Utils.create_name() + file_item = src_cont.file(source_filename) + file_item.write_random() + dest_cont = self.env.account.container(Utils.create_name()) + self.assert_(dest_cont.create()) + + for prefix in ('', '/'): + # invalid source container + file_item = dest_cont.file(Utils.create_name()) + self.assertRaises(ResponseError, file_item.write, + hdrs={'X-Copy-From-Account': acct, + 'X-Copy-From': '%s%s/%s' % + (prefix, + Utils.create_name(), + source_filename)}) + # looks like cached responses leak "not found" + # to un-authorized users, not going to fix it now, but... + self.assert_status([403, 404]) + + # invalid source object + file_item = self.env.container.file(Utils.create_name()) + self.assertRaises(ResponseError, file_item.write, + hdrs={'X-Copy-From-Account': acct, + 'X-Copy-From': '%s%s/%s' % + (prefix, + src_cont, + Utils.create_name())}) + self.assert_status(404) + + # invalid destination container + dest_cont = self.env.account.container(Utils.create_name()) + file_item = dest_cont.file(Utils.create_name()) + self.assertRaises(ResponseError, file_item.write, + hdrs={'X-Copy-From-Account': acct, + 'X-Copy-From': '%s%s/%s' % + (prefix, + src_cont, + source_filename)}) + self.assert_status(404) + def testNameLimit(self): limit = load_constraint('max_object_name_length') @@ -1191,7 +1356,12 @@ class TestFile(Base): self.assertEqual(file_types, file_types_read) def testRangedGets(self): - file_length = 10000 + # We set the file_length to a strange multiple here. This is to check + # that ranges still work in the EC case when the requested range + # spans EC segment boundaries. The 1 MiB base value is chosen because + # that's a common EC segment size. The 1.33 multiple is to ensure we + # aren't aligned on segment boundaries + file_length = int(1048576 * 1.33) range_size = file_length / 10 file_item = self.env.container.file(Utils.create_name()) data = file_item.write_random(file_length) @@ -1254,6 +1424,15 @@ class TestFile(Base): limit = load_constraint('max_file_size') tsecs = 3 + def timeout(seconds, method, *args, **kwargs): + try: + with eventlet.Timeout(seconds): + method(*args, **kwargs) + except eventlet.Timeout: + return True + else: + return False + for i in (limit - 100, limit - 10, limit - 1, limit, limit + 1, limit + 10, limit + 100): @@ -1295,6 +1474,16 @@ class TestFile(Base): cfg={'no_content_length': True}) self.assert_status(400) + # no content-length + self.assertRaises(ResponseError, file_item.write_random, file_length, + cfg={'no_content_length': True}) + self.assert_status(411) + + self.assertRaises(ResponseError, file_item.write_random, file_length, + hdrs={'transfer-encoding': 'gzip,chunked'}, + cfg={'no_content_length': True}) + self.assert_status(501) + # bad request types #for req in ('LICK', 'GETorHEAD_base', 'container_info', # 'best_response'): @@ -1565,8 +1754,16 @@ class TestFile(Base): self.assertEqual(etag, header_etag) def testChunkedPut(self): - if (web_front_end == 'apache2'): - raise SkipTest() + if (tf.web_front_end == 'apache2'): + raise SkipTest("Chunked PUT can only be tested with apache2 web" + " front end") + + def chunks(s, length=3): + i, j = 0, length + while i < len(s): + yield s[i:j] + i, j = j, j + length + data = File.random_data(10000) etag = File.compute_md5sum(data) @@ -1590,10 +1787,10 @@ class TestFileUTF8(Base2, TestFile): class TestDloEnv(object): @classmethod def setUp(cls): - cls.conn = Connection(config) + cls.conn = Connection(tf.config) cls.conn.authenticate() - cls.account = Account(cls.conn, config.get('account', - config['username'])) + cls.account = Account(cls.conn, tf.config.get('account', + tf.config['username'])) cls.account.delete_containers() cls.container = cls.account.container(Utils.create_name()) @@ -1657,6 +1854,9 @@ class TestDlo(Base): file_item = self.env.container.file('man1') file_contents = file_item.read(parms={'multipart-manifest': 'get'}) self.assertEqual(file_contents, "man1-contents") + self.assertEqual(file_item.info()['x_object_manifest'], + "%s/%s/seg_lower" % + (self.env.container.name, self.env.segment_prefix)) def test_get_range(self): file_item = self.env.container.file('man1') @@ -1691,9 +1891,38 @@ class TestDlo(Base): self.assertEqual( file_contents, "aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffff") + # The copied object must not have X-Object-Manifest + self.assertTrue("x_object_manifest" not in file_item.info()) + + def test_copy_account(self): + # dlo use same account and same container only + acct = self.env.conn.account_name + # Adding a new segment, copying the manifest, and then deleting the + # segment proves that the new object is really the concatenated + # segments and not just a manifest. + f_segment = self.env.container.file("%s/seg_lowerf" % + (self.env.segment_prefix)) + f_segment.write('ffffffffff') + try: + man1_item = self.env.container.file('man1') + man1_item.copy_account(acct, + self.env.container.name, + "copied-man1") + finally: + # try not to leave this around for other tests to stumble over + f_segment.delete() + + file_item = self.env.container.file('copied-man1') + file_contents = file_item.read() + self.assertEqual( + file_contents, + "aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffff") + # The copied object must not have X-Object-Manifest + self.assertTrue("x_object_manifest" not in file_item.info()) def test_copy_manifest(self): - # Copying the manifest should result in another manifest + # Copying the manifest with multipart-manifest=get query string + # should result in another manifest try: man1_item = self.env.container.file('man1') man1_item.copy(self.env.container.name, "copied-man1", @@ -1707,10 +1936,57 @@ class TestDlo(Base): self.assertEqual( copied_contents, "aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeee") + self.assertEqual(man1_item.info()['x_object_manifest'], + copied.info()['x_object_manifest']) finally: # try not to leave this around for other tests to stumble over self.env.container.file("copied-man1").delete() + def test_dlo_if_match_get(self): + manifest = self.env.container.file("man1") + etag = manifest.info()['etag'] + + self.assertRaises(ResponseError, manifest.read, + hdrs={'If-Match': 'not-%s' % etag}) + self.assert_status(412) + + manifest.read(hdrs={'If-Match': etag}) + self.assert_status(200) + + def test_dlo_if_none_match_get(self): + manifest = self.env.container.file("man1") + etag = manifest.info()['etag'] + + self.assertRaises(ResponseError, manifest.read, + hdrs={'If-None-Match': etag}) + self.assert_status(304) + + manifest.read(hdrs={'If-None-Match': "not-%s" % etag}) + self.assert_status(200) + + def test_dlo_if_match_head(self): + manifest = self.env.container.file("man1") + etag = manifest.info()['etag'] + + self.assertRaises(ResponseError, manifest.info, + hdrs={'If-Match': 'not-%s' % etag}) + self.assert_status(412) + + manifest.info(hdrs={'If-Match': etag}) + self.assert_status(200) + + def test_dlo_if_none_match_head(self): + manifest = self.env.container.file("man1") + etag = manifest.info()['etag'] + + self.assertRaises(ResponseError, manifest.info, + hdrs={'If-None-Match': etag}) + self.assert_status(304) + + manifest.info(hdrs={'If-None-Match': "not-%s" % etag}) + self.assert_status(200) + + class TestDloUTF8(Base2, TestDlo): set_up = False @@ -1718,10 +1994,10 @@ class TestDloUTF8(Base2, TestDlo): class TestFileComparisonEnv(object): @classmethod def setUp(cls): - cls.conn = Connection(config) + cls.conn = Connection(tf.config) cls.conn.authenticate() - cls.account = Account(cls.conn, config.get('account', - config['username'])) + cls.account = Account(cls.conn, tf.config.get('account', + tf.config['username'])) cls.account.delete_containers() cls.container = cls.account.container(Utils.create_name()) @@ -1773,19 +2049,25 @@ class TestFileComparison(Base): for file_item in self.env.files: hdrs = {'If-Modified-Since': self.env.time_old_f1} self.assert_(file_item.read(hdrs=hdrs)) + self.assert_(file_item.info(hdrs=hdrs)) hdrs = {'If-Modified-Since': self.env.time_new} self.assertRaises(ResponseError, file_item.read, hdrs=hdrs) self.assert_status(304) + self.assertRaises(ResponseError, file_item.info, hdrs=hdrs) + self.assert_status(304) def testIfUnmodifiedSince(self): for file_item in self.env.files: hdrs = {'If-Unmodified-Since': self.env.time_new} self.assert_(file_item.read(hdrs=hdrs)) + self.assert_(file_item.info(hdrs=hdrs)) hdrs = {'If-Unmodified-Since': self.env.time_old_f2} self.assertRaises(ResponseError, file_item.read, hdrs=hdrs) self.assert_status(412) + self.assertRaises(ResponseError, file_item.info, hdrs=hdrs) + self.assert_status(412) def testIfMatchAndUnmodified(self): for file_item in self.env.files: @@ -1835,17 +2117,24 @@ class TestSloEnv(object): @classmethod def setUp(cls): - cls.conn = Connection(config) + cls.conn = Connection(tf.config) cls.conn.authenticate() + config2 = deepcopy(tf.config) + config2['account'] = tf.config['account2'] + config2['username'] = tf.config['username2'] + config2['password'] = tf.config['password2'] + cls.conn2 = Connection(config2) + cls.conn2.authenticate() + cls.account2 = cls.conn2.get_account() + cls.account2.delete_containers() if cls.slo_enabled is None: - cluster_info = cls.conn.cluster_info() cls.slo_enabled = 'slo' in cluster_info if not cls.slo_enabled: return - cls.account = Account(cls.conn, config.get('account', - config['username'])) + cls.account = Account(cls.conn, tf.config.get('account', + tf.config['username'])) cls.account.delete_containers() cls.container = cls.account.container(Utils.create_name()) @@ -1911,7 +2200,7 @@ class TestSlo(Base): set_up = False def setUp(self): - raise SkipTest("SLO not enabled yet in gluster-swift") + raise SkipTest("SLO not enabled yet in gluster-swift") super(TestSlo, self).setUp() if self.env.slo_enabled is False: raise SkipTest("SLO not enabled") @@ -2021,6 +2310,29 @@ class TestSlo(Base): copied_contents = copied.read(parms={'multipart-manifest': 'get'}) self.assertEqual(4 * 1024 * 1024 + 1, len(copied_contents)) + def test_slo_copy_account(self): + acct = self.env.conn.account_name + # same account copy + file_item = self.env.container.file("manifest-abcde") + file_item.copy_account(acct, self.env.container.name, "copied-abcde") + + copied = self.env.container.file("copied-abcde") + copied_contents = copied.read(parms={'multipart-manifest': 'get'}) + self.assertEqual(4 * 1024 * 1024 + 1, len(copied_contents)) + + # copy to different account + acct = self.env.conn2.account_name + dest_cont = self.env.account2.container(Utils.create_name()) + self.assert_(dest_cont.create(hdrs={ + 'X-Container-Write': self.env.conn.user_acl + })) + file_item = self.env.container.file("manifest-abcde") + file_item.copy_account(acct, dest_cont, "copied-abcde") + + copied = dest_cont.file("copied-abcde") + copied_contents = copied.read(parms={'multipart-manifest': 'get'}) + self.assertEqual(4 * 1024 * 1024 + 1, len(copied_contents)) + def test_slo_copy_the_manifest(self): file_item = self.env.container.file("manifest-abcde") file_item.copy(self.env.container.name, "copied-abcde-manifest-only", @@ -2033,6 +2345,40 @@ class TestSlo(Base): except ValueError: self.fail("COPY didn't copy the manifest (invalid json on GET)") + def test_slo_copy_the_manifest_account(self): + acct = self.env.conn.account_name + # same account + file_item = self.env.container.file("manifest-abcde") + file_item.copy_account(acct, + self.env.container.name, + "copied-abcde-manifest-only", + parms={'multipart-manifest': 'get'}) + + copied = self.env.container.file("copied-abcde-manifest-only") + copied_contents = copied.read(parms={'multipart-manifest': 'get'}) + try: + json.loads(copied_contents) + except ValueError: + self.fail("COPY didn't copy the manifest (invalid json on GET)") + + # different account + acct = self.env.conn2.account_name + dest_cont = self.env.account2.container(Utils.create_name()) + self.assert_(dest_cont.create(hdrs={ + 'X-Container-Write': self.env.conn.user_acl + })) + file_item.copy_account(acct, + dest_cont, + "copied-abcde-manifest-only", + parms={'multipart-manifest': 'get'}) + + copied = dest_cont.file("copied-abcde-manifest-only") + copied_contents = copied.read(parms={'multipart-manifest': 'get'}) + try: + json.loads(copied_contents) + except ValueError: + self.fail("COPY didn't copy the manifest (invalid json on GET)") + def test_slo_get_the_manifest(self): manifest = self.env.container.file("manifest-abcde") got_body = manifest.read(parms={'multipart-manifest': 'get'}) @@ -2051,6 +2397,50 @@ class TestSlo(Base): self.assertEqual('application/json; charset=utf-8', got_info['content_type']) + def test_slo_if_match_get(self): + manifest = self.env.container.file("manifest-abcde") + etag = manifest.info()['etag'] + + self.assertRaises(ResponseError, manifest.read, + hdrs={'If-Match': 'not-%s' % etag}) + self.assert_status(412) + + manifest.read(hdrs={'If-Match': etag}) + self.assert_status(200) + + def test_slo_if_none_match_get(self): + manifest = self.env.container.file("manifest-abcde") + etag = manifest.info()['etag'] + + self.assertRaises(ResponseError, manifest.read, + hdrs={'If-None-Match': etag}) + self.assert_status(304) + + manifest.read(hdrs={'If-None-Match': "not-%s" % etag}) + self.assert_status(200) + + def test_slo_if_match_head(self): + manifest = self.env.container.file("manifest-abcde") + etag = manifest.info()['etag'] + + self.assertRaises(ResponseError, manifest.info, + hdrs={'If-Match': 'not-%s' % etag}) + self.assert_status(412) + + manifest.info(hdrs={'If-Match': etag}) + self.assert_status(200) + + def test_slo_if_none_match_head(self): + manifest = self.env.container.file("manifest-abcde") + etag = manifest.info()['etag'] + + self.assertRaises(ResponseError, manifest.info, + hdrs={'If-None-Match': etag}) + self.assert_status(304) + + manifest.info(hdrs={'If-None-Match': "not-%s" % etag}) + self.assert_status(200) + class TestSloUTF8(Base2, TestSlo): set_up = False @@ -2061,11 +2451,19 @@ class TestObjectVersioningEnv(object): @classmethod def setUp(cls): - cls.conn = Connection(config) + cls.conn = Connection(tf.config) cls.conn.authenticate() - cls.account = Account(cls.conn, config.get('account', - config['username'])) + cls.account = Account(cls.conn, tf.config.get('account', + tf.config['username'])) + + # Second connection for ACL tests + config2 = deepcopy(tf.config) + config2['account'] = tf.config['account2'] + config2['username'] = tf.config['username2'] + config2['password'] = tf.config['password2'] + cls.conn2 = Connection(config2) + cls.conn2.authenticate() # avoid getting a prefix that stops halfway through an encoded # character @@ -2085,6 +2483,69 @@ class TestObjectVersioningEnv(object): cls.versioning_enabled = 'versions' in container_info +class TestCrossPolicyObjectVersioningEnv(object): + # tri-state: None initially, then True/False + versioning_enabled = None + multiple_policies_enabled = None + policies = None + + @classmethod + def setUp(cls): + cls.conn = Connection(tf.config) + cls.conn.authenticate() + + if cls.multiple_policies_enabled is None: + try: + cls.policies = tf.FunctionalStoragePolicyCollection.from_info() + except AssertionError: + pass + + if cls.policies and len(cls.policies) > 1: + cls.multiple_policies_enabled = True + else: + cls.multiple_policies_enabled = False + # We have to lie here that versioning is enabled. We actually + # don't know, but it does not matter. We know these tests cannot + # run without multiple policies present. If multiple policies are + # present, we won't be setting this field to any value, so it + # should all still work. + cls.versioning_enabled = True + return + + policy = cls.policies.select() + version_policy = cls.policies.exclude(name=policy['name']).select() + + cls.account = Account(cls.conn, tf.config.get('account', + tf.config['username'])) + + # Second connection for ACL tests + config2 = deepcopy(tf.config) + config2['account'] = tf.config['account2'] + config2['username'] = tf.config['username2'] + config2['password'] = tf.config['password2'] + cls.conn2 = Connection(config2) + cls.conn2.authenticate() + + # avoid getting a prefix that stops halfway through an encoded + # character + prefix = Utils.create_name().decode("utf-8")[:10].encode("utf-8") + + cls.versions_container = cls.account.container(prefix + "-versions") + if not cls.versions_container.create( + {'X-Storage-Policy': policy['name']}): + raise ResponseError(cls.conn.response) + + cls.container = cls.account.container(prefix + "-objs") + if not cls.container.create( + hdrs={'X-Versions-Location': cls.versions_container.name, + 'X-Storage-Policy': version_policy['name']}): + raise ResponseError(cls.conn.response) + + container_info = cls.container.info() + # if versioning is off, then X-Versions-Location won't persist + cls.versioning_enabled = 'versions' in container_info + + class TestObjectVersioning(Base): env = TestObjectVersioningEnv set_up = False @@ -2099,6 +2560,15 @@ class TestObjectVersioning(Base): "Expected versioning_enabled to be True/False, got %r" % (self.env.versioning_enabled,)) + def tearDown(self): + super(TestObjectVersioning, self).tearDown() + try: + # delete versions first! + self.env.versions_container.delete_files() + self.env.container.delete_files() + except ResponseError: + pass + def test_overwriting(self): container = self.env.container versions_container = self.env.versions_container @@ -2130,31 +2600,100 @@ class TestObjectVersioning(Base): versioned_obj.delete() self.assertRaises(ResponseError, versioned_obj.read) + def test_versioning_dlo(self): + raise SkipTest('SOF incompatible test') + container = self.env.container + versions_container = self.env.versions_container + obj_name = Utils.create_name() + + for i in ('1', '2', '3'): + time.sleep(.01) # guarantee that the timestamp changes + obj_name_seg = obj_name + '/' + i + versioned_obj = container.file(obj_name_seg) + versioned_obj.write(i) + versioned_obj.write(i + i) + + self.assertEqual(3, versions_container.info()['object_count']) + + man_file = container.file(obj_name) + man_file.write('', hdrs={"X-Object-Manifest": "%s/%s/" % + (self.env.container.name, obj_name)}) + + # guarantee that the timestamp changes + time.sleep(.01) + + # write manifest file again + man_file.write('', hdrs={"X-Object-Manifest": "%s/%s/" % + (self.env.container.name, obj_name)}) + + self.assertEqual(3, versions_container.info()['object_count']) + self.assertEqual("112233", man_file.read()) + + def test_versioning_check_acl(self): + container = self.env.container + versions_container = self.env.versions_container + versions_container.create(hdrs={'X-Container-Read': '.r:*,.rlistings'}) + + obj_name = Utils.create_name() + versioned_obj = container.file(obj_name) + versioned_obj.write("aaaaa") + self.assertEqual("aaaaa", versioned_obj.read()) + + versioned_obj.write("bbbbb") + self.assertEqual("bbbbb", versioned_obj.read()) + + # Use token from second account and try to delete the object + org_token = self.env.account.conn.storage_token + self.env.account.conn.storage_token = self.env.conn2.storage_token + try: + self.assertRaises(ResponseError, versioned_obj.delete) + finally: + self.env.account.conn.storage_token = org_token + + # Verify with token from first account + self.assertEqual("bbbbb", versioned_obj.read()) + + versioned_obj.delete() + self.assertEqual("aaaaa", versioned_obj.read()) + class TestObjectVersioningUTF8(Base2, TestObjectVersioning): set_up = False +class TestCrossPolicyObjectVersioning(TestObjectVersioning): + env = TestCrossPolicyObjectVersioningEnv + set_up = False + + def setUp(self): + super(TestCrossPolicyObjectVersioning, self).setUp() + if self.env.multiple_policies_enabled is False: + raise SkipTest('Cross policy test requires multiple policies') + elif self.env.multiple_policies_enabled is not True: + # just some sanity checking + raise Exception("Expected multiple_policies_enabled " + "to be True/False, got %r" % ( + self.env.versioning_enabled,)) + + class TestTempurlEnv(object): tempurl_enabled = None # tri-state: None initially, then True/False @classmethod def setUp(cls): - cls.conn = Connection(config) + cls.conn = Connection(tf.config) cls.conn.authenticate() if cls.tempurl_enabled is None: - cluster_info = cls.conn.cluster_info() cls.tempurl_enabled = 'tempurl' in cluster_info if not cls.tempurl_enabled: return - cls.tempurl_methods = cluster_info['tempurl']['methods'] cls.tempurl_key = Utils.create_name() cls.tempurl_key2 = Utils.create_name() cls.account = Account( - cls.conn, config.get('account', config['username'])) + cls.conn, tf.config.get('account', tf.config['username'])) cls.account.delete_containers() cls.account.update_metadata({ 'temp-url-key': cls.tempurl_key, @@ -2219,6 +2758,59 @@ class TestTempurl(Base): contents = self.env.obj.read(parms=parms, cfg={'no_auth_token': True}) self.assertEqual(contents, "obj contents") + def test_GET_DLO_inside_container(self): + seg1 = self.env.container.file( + "get-dlo-inside-seg1" + Utils.create_name()) + seg2 = self.env.container.file( + "get-dlo-inside-seg2" + Utils.create_name()) + seg1.write("one fish two fish ") + seg2.write("red fish blue fish") + + manifest = self.env.container.file("manifest" + Utils.create_name()) + manifest.write( + '', + hdrs={"X-Object-Manifest": "%s/get-dlo-inside-seg" % + (self.env.container.name,)}) + + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'GET', expires, self.env.conn.make_path(manifest.path), + self.env.tempurl_key) + parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + contents = manifest.read(parms=parms, cfg={'no_auth_token': True}) + self.assertEqual(contents, "one fish two fish red fish blue fish") + + def test_GET_DLO_outside_container(self): + seg1 = self.env.container.file( + "get-dlo-outside-seg1" + Utils.create_name()) + seg2 = self.env.container.file( + "get-dlo-outside-seg2" + Utils.create_name()) + seg1.write("one fish two fish ") + seg2.write("red fish blue fish") + + container2 = self.env.account.container(Utils.create_name()) + container2.create() + + manifest = container2.file("manifest" + Utils.create_name()) + manifest.write( + '', + hdrs={"X-Object-Manifest": "%s/get-dlo-outside-seg" % + (self.env.container.name,)}) + + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'GET', expires, self.env.conn.make_path(manifest.path), + self.env.tempurl_key) + parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + # cross container tempurl works fine for account tempurl key + contents = manifest.read(parms=parms, cfg={'no_auth_token': True}) + self.assertEqual(contents, "one fish two fish red fish blue fish") + self.assert_status([200]) + def test_PUT(self): new_obj = self.env.container.file(Utils.create_name()) @@ -2237,6 +2829,42 @@ class TestTempurl(Base): self.assert_(new_obj.info(parms=put_parms, cfg={'no_auth_token': True})) + def test_PUT_manifest_access(self): + new_obj = self.env.container.file(Utils.create_name()) + + # give out a signature which allows a PUT to new_obj + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'PUT', expires, self.env.conn.make_path(new_obj.path), + self.env.tempurl_key) + put_parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + # try to create manifest pointing to some random container + try: + new_obj.write('', { + 'x-object-manifest': '%s/foo' % 'some_random_container' + }, parms=put_parms, cfg={'no_auth_token': True}) + except ResponseError as e: + self.assertEqual(e.status, 400) + else: + self.fail('request did not error') + + # create some other container + other_container = self.env.account.container(Utils.create_name()) + if not other_container.create(): + raise ResponseError(self.conn.response) + + # try to create manifest pointing to new container + try: + new_obj.write('', { + 'x-object-manifest': '%s/foo' % other_container + }, parms=put_parms, cfg={'no_auth_token': True}) + except ResponseError as e: + self.assertEqual(e.status, 400) + else: + self.fail('request did not error') + def test_HEAD(self): expires = int(time.time()) + 86400 sig = self.tempurl_sig( @@ -2310,22 +2938,288 @@ class TestTempurlUTF8(Base2, TestTempurl): set_up = False +class TestContainerTempurlEnv(object): + tempurl_enabled = None # tri-state: None initially, then True/False + + @classmethod + def setUp(cls): + cls.conn = Connection(tf.config) + cls.conn.authenticate() + + if cls.tempurl_enabled is None: + cls.tempurl_enabled = 'tempurl' in cluster_info + if not cls.tempurl_enabled: + return + + cls.tempurl_key = Utils.create_name() + cls.tempurl_key2 = Utils.create_name() + + cls.account = Account( + cls.conn, tf.config.get('account', tf.config['username'])) + cls.account.delete_containers() + + # creating another account and connection + # for ACL tests + config2 = deepcopy(tf.config) + config2['account'] = tf.config['account2'] + config2['username'] = tf.config['username2'] + config2['password'] = tf.config['password2'] + cls.conn2 = Connection(config2) + cls.conn2.authenticate() + cls.account2 = Account( + cls.conn2, config2.get('account', config2['username'])) + cls.account2 = cls.conn2.get_account() + + cls.container = cls.account.container(Utils.create_name()) + if not cls.container.create({ + 'x-container-meta-temp-url-key': cls.tempurl_key, + 'x-container-meta-temp-url-key-2': cls.tempurl_key2, + 'x-container-read': cls.account2.name}): + raise ResponseError(cls.conn.response) + + cls.obj = cls.container.file(Utils.create_name()) + cls.obj.write("obj contents") + cls.other_obj = cls.container.file(Utils.create_name()) + cls.other_obj.write("other obj contents") + + +class TestContainerTempurl(Base): + env = TestContainerTempurlEnv + set_up = False + + def setUp(self): + super(TestContainerTempurl, self).setUp() + if self.env.tempurl_enabled is False: + raise SkipTest("TempURL not enabled") + elif self.env.tempurl_enabled is not True: + # just some sanity checking + raise Exception( + "Expected tempurl_enabled to be True/False, got %r" % + (self.env.tempurl_enabled,)) + + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'GET', expires, self.env.conn.make_path(self.env.obj.path), + self.env.tempurl_key) + self.obj_tempurl_parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + def tempurl_sig(self, method, expires, path, key): + return hmac.new( + key, + '%s\n%s\n%s' % (method, expires, urllib.unquote(path)), + hashlib.sha1).hexdigest() + + def test_GET(self): + contents = self.env.obj.read( + parms=self.obj_tempurl_parms, + cfg={'no_auth_token': True}) + self.assertEqual(contents, "obj contents") + + # GET tempurls also allow HEAD requests + self.assert_(self.env.obj.info(parms=self.obj_tempurl_parms, + cfg={'no_auth_token': True})) + + def test_GET_with_key_2(self): + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'GET', expires, self.env.conn.make_path(self.env.obj.path), + self.env.tempurl_key2) + parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + contents = self.env.obj.read(parms=parms, cfg={'no_auth_token': True}) + self.assertEqual(contents, "obj contents") + + def test_PUT(self): + new_obj = self.env.container.file(Utils.create_name()) + + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'PUT', expires, self.env.conn.make_path(new_obj.path), + self.env.tempurl_key) + put_parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + new_obj.write('new obj contents', + parms=put_parms, cfg={'no_auth_token': True}) + self.assertEqual(new_obj.read(), "new obj contents") + + # PUT tempurls also allow HEAD requests + self.assert_(new_obj.info(parms=put_parms, + cfg={'no_auth_token': True})) + + def test_HEAD(self): + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'HEAD', expires, self.env.conn.make_path(self.env.obj.path), + self.env.tempurl_key) + head_parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + self.assert_(self.env.obj.info(parms=head_parms, + cfg={'no_auth_token': True})) + # HEAD tempurls don't allow PUT or GET requests, despite the fact that + # PUT and GET tempurls both allow HEAD requests + self.assertRaises(ResponseError, self.env.other_obj.read, + cfg={'no_auth_token': True}, + parms=self.obj_tempurl_parms) + self.assert_status([401]) + + self.assertRaises(ResponseError, self.env.other_obj.write, + 'new contents', + cfg={'no_auth_token': True}, + parms=self.obj_tempurl_parms) + self.assert_status([401]) + + def test_different_object(self): + contents = self.env.obj.read( + parms=self.obj_tempurl_parms, + cfg={'no_auth_token': True}) + self.assertEqual(contents, "obj contents") + + self.assertRaises(ResponseError, self.env.other_obj.read, + cfg={'no_auth_token': True}, + parms=self.obj_tempurl_parms) + self.assert_status([401]) + + def test_changing_sig(self): + contents = self.env.obj.read( + parms=self.obj_tempurl_parms, + cfg={'no_auth_token': True}) + self.assertEqual(contents, "obj contents") + + parms = self.obj_tempurl_parms.copy() + if parms['temp_url_sig'][0] == 'a': + parms['temp_url_sig'] = 'b' + parms['temp_url_sig'][1:] + else: + parms['temp_url_sig'] = 'a' + parms['temp_url_sig'][1:] + + self.assertRaises(ResponseError, self.env.obj.read, + cfg={'no_auth_token': True}, + parms=parms) + self.assert_status([401]) + + def test_changing_expires(self): + contents = self.env.obj.read( + parms=self.obj_tempurl_parms, + cfg={'no_auth_token': True}) + self.assertEqual(contents, "obj contents") + + parms = self.obj_tempurl_parms.copy() + if parms['temp_url_expires'][-1] == '0': + parms['temp_url_expires'] = parms['temp_url_expires'][:-1] + '1' + else: + parms['temp_url_expires'] = parms['temp_url_expires'][:-1] + '0' + + self.assertRaises(ResponseError, self.env.obj.read, + cfg={'no_auth_token': True}, + parms=parms) + self.assert_status([401]) + + def test_tempurl_keys_visible_to_account_owner(self): + if not tf.cluster_info.get('tempauth'): + raise SkipTest('TEMP AUTH SPECIFIC TEST') + metadata = self.env.container.info() + self.assertEqual(metadata.get('tempurl_key'), self.env.tempurl_key) + self.assertEqual(metadata.get('tempurl_key2'), self.env.tempurl_key2) + + def test_tempurl_keys_hidden_from_acl_readonly(self): + if not tf.cluster_info.get('tempauth'): + raise SkipTest('TEMP AUTH SPECIFIC TEST') + original_token = self.env.container.conn.storage_token + self.env.container.conn.storage_token = self.env.conn2.storage_token + metadata = self.env.container.info() + self.env.container.conn.storage_token = original_token + + self.assertTrue('tempurl_key' not in metadata, + 'Container TempURL key found, should not be visible ' + 'to readonly ACLs') + self.assertTrue('tempurl_key2' not in metadata, + 'Container TempURL key-2 found, should not be visible ' + 'to readonly ACLs') + + def test_GET_DLO_inside_container(self): + seg1 = self.env.container.file( + "get-dlo-inside-seg1" + Utils.create_name()) + seg2 = self.env.container.file( + "get-dlo-inside-seg2" + Utils.create_name()) + seg1.write("one fish two fish ") + seg2.write("red fish blue fish") + + manifest = self.env.container.file("manifest" + Utils.create_name()) + manifest.write( + '', + hdrs={"X-Object-Manifest": "%s/get-dlo-inside-seg" % + (self.env.container.name,)}) + + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'GET', expires, self.env.conn.make_path(manifest.path), + self.env.tempurl_key) + parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + contents = manifest.read(parms=parms, cfg={'no_auth_token': True}) + self.assertEqual(contents, "one fish two fish red fish blue fish") + + def test_GET_DLO_outside_container(self): + container2 = self.env.account.container(Utils.create_name()) + container2.create() + seg1 = container2.file( + "get-dlo-outside-seg1" + Utils.create_name()) + seg2 = container2.file( + "get-dlo-outside-seg2" + Utils.create_name()) + seg1.write("one fish two fish ") + seg2.write("red fish blue fish") + + manifest = self.env.container.file("manifest" + Utils.create_name()) + manifest.write( + '', + hdrs={"X-Object-Manifest": "%s/get-dlo-outside-seg" % + (container2.name,)}) + + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'GET', expires, self.env.conn.make_path(manifest.path), + self.env.tempurl_key) + parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + # cross container tempurl does not work for container tempurl key + try: + manifest.read(parms=parms, cfg={'no_auth_token': True}) + except ResponseError as e: + self.assertEqual(e.status, 401) + else: + self.fail('request did not error') + try: + manifest.info(parms=parms, cfg={'no_auth_token': True}) + except ResponseError as e: + self.assertEqual(e.status, 401) + else: + self.fail('request did not error') + + +class TestContainerTempurlUTF8(Base2, TestContainerTempurl): + set_up = False + + class TestSloTempurlEnv(object): enabled = None # tri-state: None initially, then True/False @classmethod def setUp(cls): - cls.conn = Connection(config) + cls.conn = Connection(tf.config) cls.conn.authenticate() if cls.enabled is None: - cluster_info = cls.conn.cluster_info() cls.enabled = 'tempurl' in cluster_info and 'slo' in cluster_info cls.tempurl_key = Utils.create_name() cls.account = Account( - cls.conn, config.get('account', config['username'])) + cls.conn, tf.config.get('account', tf.config['username'])) cls.account.delete_containers() cls.account.update_metadata({'temp-url-key': cls.tempurl_key}) @@ -2398,5 +3292,174 @@ class TestSloTempurlUTF8(Base2, TestSloTempurl): set_up = False +class TestServiceToken(unittest.TestCase): + + def setUp(self): + if tf.skip_service_tokens: + raise SkipTest + + self.SET_TO_USERS_TOKEN = 1 + self.SET_TO_SERVICE_TOKEN = 2 + + # keystoneauth and tempauth differ in allowing PUT account + # Even if keystoneauth allows it, the proxy-server uses + # allow_account_management to decide if accounts can be created + self.put_account_expect = is_client_error + if tf.swift_test_auth_version != '1': + if cluster_info.get('swift').get('allow_account_management'): + self.put_account_expect = is_success + + def _scenario_generator(self): + paths = ((None, None), ('c', None), ('c', 'o')) + for path in paths: + for method in ('PUT', 'POST', 'HEAD', 'GET', 'OPTIONS'): + yield method, path[0], path[1] + for path in reversed(paths): + yield 'DELETE', path[0], path[1] + + def _assert_is_authed_response(self, method, container, object, resp): + resp.read() + expect = is_success + if method == 'DELETE' and not container: + expect = is_client_error + if method == 'PUT' and not container: + expect = self.put_account_expect + self.assertTrue(expect(resp.status), 'Unexpected %s for %s %s %s' + % (resp.status, method, container, object)) + + def _assert_not_authed_response(self, method, container, object, resp): + resp.read() + expect = is_client_error + if method == 'OPTIONS': + expect = is_success + self.assertTrue(expect(resp.status), 'Unexpected %s for %s %s %s' + % (resp.status, method, container, object)) + + def prepare_request(self, method, use_service_account=False, + container=None, obj=None, body=None, headers=None, + x_auth_token=None, + x_service_token=None, dbg=False): + """ + Setup for making the request + + When retry() calls the do_request() function, it calls it the + test user's token, the parsed path, a connection and (optionally) + a token from the test service user. We save options here so that + do_request() can make the appropriate request. + + :param method: The operation (e.g'. 'HEAD') + :param use_service_account: Optional. Set True to change the path to + be the service account + :param container: Optional. Adds a container name to the path + :param obj: Optional. Adds an object name to the path + :param body: Optional. Adds a body (string) in the request + :param headers: Optional. Adds additional headers. + :param x_auth_token: Optional. Default is SET_TO_USERS_TOKEN. One of: + SET_TO_USERS_TOKEN Put the test user's token in + X-Auth-Token + SET_TO_SERVICE_TOKEN Put the service token in X-Auth-Token + :param x_service_token: Optional. Default is to not set X-Service-Token + to any value. If specified, is one of following: + SET_TO_USERS_TOKEN Put the test user's token in + X-Service-Token + SET_TO_SERVICE_TOKEN Put the service token in + X-Service-Token + :param dbg: Optional. Set true to check request arguments + """ + self.method = method + self.use_service_account = use_service_account + self.container = container + self.obj = obj + self.body = body + self.headers = headers + if x_auth_token: + self.x_auth_token = x_auth_token + else: + self.x_auth_token = self.SET_TO_USERS_TOKEN + self.x_service_token = x_service_token + self.dbg = dbg + + def do_request(self, url, token, parsed, conn, service_token=''): + if self.use_service_account: + path = self._service_account(parsed.path) + else: + path = parsed.path + if self.container: + path += '/%s' % self.container + if self.obj: + path += '/%s' % self.obj + headers = {} + if self.body: + headers.update({'Content-Length': len(self.body)}) + if self.headers: + headers.update(self.headers) + if self.x_auth_token == self.SET_TO_USERS_TOKEN: + headers.update({'X-Auth-Token': token}) + elif self.x_auth_token == self.SET_TO_SERVICE_TOKEN: + headers.update({'X-Auth-Token': service_token}) + if self.x_service_token == self.SET_TO_USERS_TOKEN: + headers.update({'X-Service-Token': token}) + elif self.x_service_token == self.SET_TO_SERVICE_TOKEN: + headers.update({'X-Service-Token': service_token}) + if self.dbg: + print('DEBUG: conn.request: method:%s path:%s' + ' body:%s headers:%s' % (self.method, path, self.body, + headers)) + conn.request(self.method, path, self.body, headers=headers) + return check_response(conn) + + def _service_account(self, path): + parts = path.split('/', 3) + account = parts[2] + try: + project_id = account[account.index('_') + 1:] + except ValueError: + project_id = account + parts[2] = '%s%s' % (tf.swift_test_service_prefix, project_id) + return '/'.join(parts) + + def test_user_access_own_auth_account(self): + # This covers ground tested elsewhere (tests a user doing HEAD + # on own account). However, if this fails, none of the remaining + # tests will work + self.prepare_request('HEAD') + resp = retry(self.do_request) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + + def test_user_cannot_access_service_account(self): + for method, container, obj in self._scenario_generator(): + self.prepare_request(method, use_service_account=True, + container=container, obj=obj) + resp = retry(self.do_request) + self._assert_not_authed_response(method, container, obj, resp) + + def test_service_user_denied_with_x_auth_token(self): + for method, container, obj in self._scenario_generator(): + self.prepare_request(method, use_service_account=True, + container=container, obj=obj, + x_auth_token=self.SET_TO_SERVICE_TOKEN) + resp = retry(self.do_request, service_user=5) + self._assert_not_authed_response(method, container, obj, resp) + + def test_service_user_denied_with_x_service_token(self): + for method, container, obj in self._scenario_generator(): + self.prepare_request(method, use_service_account=True, + container=container, obj=obj, + x_auth_token=self.SET_TO_SERVICE_TOKEN, + x_service_token=self.SET_TO_SERVICE_TOKEN) + resp = retry(self.do_request, service_user=5) + self._assert_not_authed_response(method, container, obj, resp) + + def test_user_plus_service_can_access_service_account(self): + for method, container, obj in self._scenario_generator(): + self.prepare_request(method, use_service_account=True, + container=container, obj=obj, + x_auth_token=self.SET_TO_USERS_TOKEN, + x_service_token=self.SET_TO_SERVICE_TOKEN) + resp = retry(self.do_request, service_user=5) + self._assert_is_authed_response(method, container, obj, resp) + + if __name__ == '__main__': unittest.main() |