From 2a8f9f0f530327039c32e444b6a27130b12666bd Mon Sep 17 00:00:00 2001 From: Thiago da Silva Date: Tue, 22 Apr 2014 14:15:02 -0400 Subject: Update repo This is a squashed commit imported from this repo: https://github.com/openstack/swiftonfile/tree/icehouse Contains the follwing commits from above mentioned repo: eb50236 Merge "Backport: Fix metadata overall limits bug" into icehouse 79ea52a Backport: Fix metadata overall limits bug bc43f0b Fix inconsistent data being returned on GET ad0bb79 Import HTTPBadRequest from swift's module 74d02e6 Exclude .trashcan dir from container listing b2dbc15 Catch ESTALE in addition to ENOENT 8d60b48 Properly handle read_metadata() exceptions 6762fc6 Fix object server leaking file descriptors 2842e82 Fix API incompatibility in update_metadata() 2beeef6 Merge "Remove swiftkerbauth code" into icehouse 93dbcb5 Update object-expirer.conf with explanations c9d2f09 Merge "Check if /etc/swift exists in ring builder" into icehouse d66c14c Remove swiftkerbauth code 3142ed2 Add object expiration functests 97153d1 Merge "Cleanup functest and undo old patch" into icehouse bc234d0 Remove old travis config file and fix typo 260c8ef Check if /etc/swift exists in ring builder 637dac9 Cleanup functest and undo old patch 051e068 Merge pull request #35 from prashanthpai/backport-1 be104a3 Merge pull request #36 from prashanthpai/backport-2 ff76f42 fix issue with GET on large object (icehouse-backport) 04d0a99 Fix unlink call after successful rename 4c6ca1d updating README file with project name change 10b2680 Merge pull request #18 from thiagol11/icehouse 5bcab8f Updating version on __init__ file 5c2cba2 Merge pull request #15 from thiagol11/update_spec 52b00a8 updating spec file to add dependency on swift icehouse ae7c93b Merge pull request #6 from prashanthpai/rebase 191e55b Revert: allow non-root user to run functests cb7e968 Modify unit tests and func tests d23fd1b Sync with OpenStack Swift v1.13.1 b6d1671 Merge pull request #12 from pushpesh/functionalnosetestremove 962622b Merge pull request #8 from thiagol11/update_readme 4560857 Merge pull request #9 from prashanthpai/spec-expirer be0ae7e Minor update 65000f1 Removing functionalnosetests 8ab1069 Fix object-expirer.conf-gluster RPM build error afee30f added new support filesystem section 527b01f updated README.md to Swift-On-File 9a240c7 Merge pull request #3 from thiagol11/add_jenkins_to_travis 34b5a8b removing blank lines 3568b64 fixing missing fi d8f5b0f adding support to run jenkins triggered by travis 6f4a88c Removing functionalnosetests 8041944 Update README.md c015148 Merge pull request #2 from thiagol11/master 3ddd952 fixing travis file to run correct unit test c582669 adding travis status badge to README 8093096 adding py26 unit testing to travis 37835fd trigger travis build cb6332a adding travis ci testing All tests have been run sucessfully against this. tox -e p2p8,py27,functest Change-Id: I096b611da852d3eb3913844034b443b8272c2ac4 Signed-off-by: Prashanth Pai Reviewed-on: http://review.gluster.org/13188 --- test/functional/gluster_swift_tests.py | 91 ++- test/functional/swift_test_client.py | 47 +- test/functional/swift_testing.py | 45 ++ test/functional/test_account.py | 584 +++++++++++++-- test/functional/test_container.py | 833 +++++++++++++++++++-- test/functional/test_object.py | 524 +++++++++++-- test/functional/tests.py | 470 ++++++++++-- .../common_conf/account-server.conf | 36 + .../common_conf/container-server.conf | 39 + test/functional_auth/common_conf/fs.conf | 19 + .../common_conf/object-expirer.conf | 27 + .../functional_auth/common_conf/object-server.conf | 53 ++ test/functional_auth/common_conf/swift.conf | 85 +++ test/functional_auth/common_conf/test.conf | 58 ++ .../gswauth/conf/account-server.conf | 32 - .../gswauth/conf/container-server.conf | 35 - test/functional_auth/gswauth/conf/fs.conf | 19 - .../gswauth/conf/object-expirer.conf | 17 - .../gswauth/conf/object-server.conf | 48 -- .../functional_auth/gswauth/conf/proxy-server.conf | 26 +- test/functional_auth/gswauth/conf/swift.conf | 85 --- .../keystone/conf/account-server.conf | 32 - .../keystone/conf/container-server.conf | 35 - test/functional_auth/keystone/conf/fs.conf | 19 - .../keystone/conf/object-server.conf | 48 -- .../keystone/conf/proxy-server.conf | 27 +- test/functional_auth/keystone/conf/swift.conf | 85 --- test/functional_auth/swiftkerbauth/__init__.py | 0 .../swiftkerbauth/conf/account-server.conf | 36 - .../swiftkerbauth/conf/container-server.conf | 36 - test/functional_auth/swiftkerbauth/conf/fs.conf | 13 - .../swiftkerbauth/conf/object-server.conf | 51 -- .../swiftkerbauth/conf/proxy-server.conf | 17 +- test/functional_auth/swiftkerbauth/conf/swift.conf | 84 --- test/functional_auth/swiftkerbauth/conf/test.conf | 49 -- .../swiftkerbauth/test_swkrbath_active.py | 93 --- .../tempauth/conf/account-server.conf | 32 - .../tempauth/conf/container-server.conf | 35 - test/functional_auth/tempauth/conf/fs.conf | 19 - .../tempauth/conf/object-expirer.conf | 17 - .../tempauth/conf/object-server.conf | 48 -- .../tempauth/conf/proxy-server.conf | 21 +- test/functional_auth/tempauth/conf/swift.conf | 85 --- test/object_expirer_functional/__init__.py | 0 .../test_object_expirer.py | 332 ++++++++ test/unit/__init__.py | 39 +- .../common/middleware/swiftkerbauth/__init__.py | 0 .../middleware/swiftkerbauth/test_kerbauth.py | 478 ------------ .../swiftkerbauth/test_kerbauth_utils.py | 77 -- test/unit/common/test_constraints.py | 73 +- test/unit/common/test_diskdir.py | 14 +- test/unit/common/test_utils.py | 24 +- test/unit/obj/test_diskfile.py | 55 +- test/unit/obj/test_expirer.py | 14 +- test/unit/proxy/controllers/test_obj.py | 98 ++- test/unit/proxy/test_server.py | 794 +++++++++++++------- 56 files changed, 3712 insertions(+), 2341 deletions(-) create mode 100644 test/functional_auth/common_conf/account-server.conf create mode 100644 test/functional_auth/common_conf/container-server.conf create mode 100644 test/functional_auth/common_conf/fs.conf create mode 100644 test/functional_auth/common_conf/object-expirer.conf create mode 100644 test/functional_auth/common_conf/object-server.conf create mode 100644 test/functional_auth/common_conf/swift.conf create mode 100644 test/functional_auth/common_conf/test.conf delete mode 100644 test/functional_auth/gswauth/conf/account-server.conf delete mode 100644 test/functional_auth/gswauth/conf/container-server.conf delete mode 100644 test/functional_auth/gswauth/conf/fs.conf delete mode 100644 test/functional_auth/gswauth/conf/object-expirer.conf delete mode 100644 test/functional_auth/gswauth/conf/object-server.conf delete mode 100644 test/functional_auth/gswauth/conf/swift.conf delete mode 100644 test/functional_auth/keystone/conf/account-server.conf delete mode 100644 test/functional_auth/keystone/conf/container-server.conf delete mode 100644 test/functional_auth/keystone/conf/fs.conf delete mode 100644 test/functional_auth/keystone/conf/object-server.conf delete mode 100644 test/functional_auth/keystone/conf/swift.conf delete mode 100644 test/functional_auth/swiftkerbauth/__init__.py delete mode 100644 test/functional_auth/swiftkerbauth/conf/account-server.conf delete mode 100644 test/functional_auth/swiftkerbauth/conf/container-server.conf delete mode 100644 test/functional_auth/swiftkerbauth/conf/fs.conf delete mode 100644 test/functional_auth/swiftkerbauth/conf/object-server.conf delete mode 100644 test/functional_auth/swiftkerbauth/conf/swift.conf delete mode 100644 test/functional_auth/swiftkerbauth/conf/test.conf delete mode 100644 test/functional_auth/swiftkerbauth/test_swkrbath_active.py delete mode 100644 test/functional_auth/tempauth/conf/account-server.conf delete mode 100644 test/functional_auth/tempauth/conf/container-server.conf delete mode 100644 test/functional_auth/tempauth/conf/fs.conf delete mode 100644 test/functional_auth/tempauth/conf/object-expirer.conf delete mode 100644 test/functional_auth/tempauth/conf/object-server.conf delete mode 100644 test/functional_auth/tempauth/conf/swift.conf create mode 100644 test/object_expirer_functional/__init__.py create mode 100644 test/object_expirer_functional/test_object_expirer.py delete mode 100644 test/unit/common/middleware/swiftkerbauth/__init__.py delete mode 100644 test/unit/common/middleware/swiftkerbauth/test_kerbauth.py delete mode 100644 test/unit/common/middleware/swiftkerbauth/test_kerbauth_utils.py (limited to 'test') diff --git a/test/functional/gluster_swift_tests.py b/test/functional/gluster_swift_tests.py index 2768f9d..b4514c9 100644 --- a/test/functional/gluster_swift_tests.py +++ b/test/functional/gluster_swift_tests.py @@ -58,44 +58,13 @@ class TestFile(Base): data_read = file.read() self.assertEquals(data,data_read) - def testInvalidHeadersPUT(self): - #TODO: Although we now support x-delete-at and x-delete-after, - #retained this test case as we may add some other header to - #unsupported list in future - raise SkipTest() - file = self.env.container.file(Utils.create_name()) - self.assertRaises(ResponseError, - file.write_random, - self.env.file_size, - hdrs={'X-Delete-At': '9876545321'}) - self.assert_status(400) - self.assertRaises(ResponseError, - file.write_random, - self.env.file_size, - hdrs={'X-Delete-After': '60'}) - self.assert_status(400) - - def testInvalidHeadersPOST(self): - #TODO: Although we now support x-delete-at and x-delete-after, - #retained this test case as we may add some other header to - #unsupported list in future - raise SkipTest() - file = self.env.container.file(Utils.create_name()) - file.write_random(self.env.file_size) - headers = file.make_headers(cfg={}) - headers.update({ 'X-Delete-At' : '987654321'}) - # Need to call conn.make_request instead of file.sync_metadata - # because sync_metadata calls make_headers. make_headers() - # overwrites any headers in file.metadata as 'user' metadata - # by appending 'X-Object-Meta-' to any of the headers - # in file.metadata. - file.conn.make_request('POST', file.path, hdrs=headers, cfg={}) - self.assertEqual(400, file.conn.response.status) - - headers = file.make_headers(cfg={}) - headers.update({ 'X-Delete-After' : '60'}) - file.conn.make_request('POST', file.path, hdrs=headers, cfg={}) - self.assertEqual(400, file.conn.response.status) + def test_PUT_large_object(self): + file_item = self.env.container.file(Utils.create_name()) + data = File.random_data(1024 * 1024 * 2) + self.assertTrue(file_item.write(data)) + self.assert_status(201) + self.assertTrue(data == file_item.read()) + self.assert_status(200) class TestFileUTF8(Base2, TestFile): @@ -375,3 +344,49 @@ class TestMultiProtocolAccess(Base): md5_returned = hashlib.md5(data_read_from_mountP).hexdigest() self.assertEquals(md5_returned,file_info['etag']) fhOnMountPoint.close() + + def testObjectMetadataWhenFileModified(self): + data = "I'm whatever Gotham needs me to be " + data_hash = hashlib.md5(data).hexdigest() + # Create an object through object interface + object_name = Utils.create_name() + object_item = self.env.container.file(object_name) + object_item.write(data) + # Make sure GET works + self.assertEqual(data, object_item.read()) + self.assert_status(200) + # Check Etag is right + self.assertEqual(data_hash, object_item.info()['etag']) + self.assert_status(200) + + # Extend/append more data to file from filesystem interface + file_path = os.path.join(self.env.root_dir, + self.env.container.name, + object_name) + more_data = "- Batman" + with open(file_path, 'a') as f: + f.write(more_data) + total_data = data + more_data + total_data_hash = hashlib.md5(total_data).hexdigest() + # Make sure GET works + self.assertEqual(total_data, object_item.read()) + self.assert_status(200) + # Check Etag and content-length is right + metadata = object_item.info() + self.assert_status(200) + self.assertEqual(total_data_hash, metadata['etag']) + self.assertEqual(len(total_data), int(metadata['content_length'])) + + # Re-write the file to be shorter + new_data = "I am Batman" + new_data_hash = hashlib.md5(new_data).hexdigest() + with open(file_path, 'w') as f: + f.write(new_data) + # Make sure GET works + self.assertEqual(new_data, object_item.read()) + self.assert_status(200) + # Check Etag and content-length is right + metadata = object_item.info() + self.assert_status(200) + self.assertEqual(new_data_hash, metadata['etag']) + self.assertEqual(len(new_data), int(metadata['content_length'])) diff --git a/test/functional/swift_test_client.py b/test/functional/swift_test_client.py index a7c7c96..27e025b 100644 --- a/test/functional/swift_test_client.py +++ b/test/functional/swift_test_client.py @@ -144,6 +144,7 @@ class Connection(object): auth_scheme = 'https://' if self.auth_ssl else 'http://' auth_netloc = "%s:%d" % (self.auth_host, self.auth_port) auth_url = auth_scheme + auth_netloc + auth_path + (storage_url, storage_token) = get_auth( auth_url, auth_user, self.password, snet=False, tenant_name=self.account, auth_version=self.auth_version, @@ -166,17 +167,29 @@ class Connection(object): self.storage_host = x[2].split(':')[0] if ':' in x[2]: self.storage_port = int(x[2].split(':')[1]) - # Make sure storage_url and the storage_token are - # strings and not unicode, since + # Make sure storage_url is a string and not unicode, since # keystoneclient (called by swiftclient) returns them in # unicode and this would cause troubles when doing # no_safe_quote query. self.storage_url = str('/%s/%s' % (x[3], x[4])) - self.storage_token = str(storage_token) + + self.storage_token = storage_token self.http_connect() return self.storage_url, self.storage_token + def cluster_info(self): + """ + Retrieve the data in /info, or {} on 404 + """ + status = self.make_request('GET', '/info', + cfg={'absolute_path': True}) + if status == 404: + return {} + if not 200 <= status <= 299: + raise ResponseError(self.response, 'GET', '/info') + return json.loads(self.response.read()) + def http_connect(self): self.connection = self.conn_class(self.storage_host, port=self.storage_port) @@ -207,8 +220,8 @@ class Connection(object): def make_request(self, method, path=[], data='', hdrs={}, parms={}, cfg={}): - if not cfg.get('verbatim_path'): - # Set verbatim_path=True to make a request to exactly the given + if not cfg.get('absolute_path'): + # Set absolute_path=True to make a request to exactly the given # path, not storage path + given path. Useful for # non-account/container/object requests. path = self.make_path(path, cfg=cfg) @@ -305,7 +318,7 @@ class Connection(object): return self.response.status -class Base: +class Base(object): def __str__(self): return self.name @@ -339,6 +352,16 @@ class Account(Base): self.conn = conn self.name = str(name) + def update_metadata(self, metadata={}, cfg={}): + headers = dict(("X-Account-Meta-%s" % k, v) + for k, v in metadata.items()) + + self.conn.make_request('POST', self.path, hdrs=headers, cfg=cfg) + if not 200 <= self.conn.response.status <= 299: + raise ResponseError(self.conn.response, 'POST', + self.conn.make_path(self.path)) + return True + def container(self, container_name): return Container(self.conn, self.name, container_name) @@ -532,6 +555,11 @@ class File(Base): else: headers['Content-Type'] = 'application/octet-stream' + if cfg.get('x_delete_at'): + headers['X-Delete-At'] = cfg.get('x_delete_at') + if cfg.get('x_delete_after'): + headers['X-Delete-After'] = cfg.get('x_delete_after') + for key in self.metadata: headers['X-Object-Meta-' + key] = self.metadata[key] @@ -588,7 +616,11 @@ class File(Base): ['last_modified', 'last-modified'], ['etag', 'etag']] - header_fields = self.header_fields(fields) + optional_fields = [['x_delete_at', 'x-delete-at'], + ['x_delete_after', 'x-delete-after']] + + header_fields = self.header_fields(fields, + optional_fields=optional_fields) header_fields['etag'] = header_fields['etag'].strip('"') return header_fields @@ -705,7 +737,6 @@ class File(Base): cfg.get('set_content_length') else: headers['Content-Length'] = 0 - self.conn.make_request('POST', self.path, hdrs=headers, cfg=cfg) if self.conn.response.status not in (201, 202): diff --git a/test/functional/swift_testing.py b/test/functional/swift_testing.py index f05cb48..2a1e1fa 100644 --- a/test/functional/swift_testing.py +++ b/test/functional/swift_testing.py @@ -19,10 +19,13 @@ import socket import sys from time import sleep from urlparse import urlparse +import functools +from nose import SkipTest from test import get_config from swiftclient import get_auth, http_connection +from test.functional.swift_test_client import Connection conf = get_config('func_test') web_front_end = conf.get('web_front_end', 'integral') @@ -184,3 +187,45 @@ def check_response(conn): resp.read() raise InternalServerError() return resp + +cluster_info = {} + + +def get_cluster_info(): + conn = Connection(conf) + conn.authenticate() + global cluster_info + cluster_info = conn.cluster_info() + + +def reset_acl(): + def post(url, token, parsed, conn): + conn.request('POST', parsed.path, '', { + 'X-Auth-Token': token, + 'X-Account-Access-Control': '{}' + }) + return check_response(conn) + resp = retry(post, use_account=1) + resp.read() + + +def requires_acls(f): + @functools.wraps(f) + def wrapper(*args, **kwargs): + if skip: + raise SkipTest + if not cluster_info: + get_cluster_info() + # Determine whether this cluster has account ACLs; if not, skip test + if not cluster_info.get('tempauth', {}).get('account_acls'): + raise SkipTest + if 'keystoneauth' in cluster_info: + # remove when keystoneauth supports account acls + raise SkipTest + reset_acl() + try: + rv = f(*args, **kwargs) + finally: + reset_acl() + return rv + return wrapper diff --git a/test/functional/test_account.py b/test/functional/test_account.py index d456090..1cc61bc 100755 --- a/test/functional/test_account.py +++ b/test/functional/test_account.py @@ -17,19 +17,57 @@ import unittest import json +from uuid import uuid4 from nose import SkipTest +from string import letters from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \ MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH from swift.common.middleware.acl import format_acl -from test.functional.swift_test_client import Connection -from test import get_config -from swift_testing import check_response, retry, skip, web_front_end +from swift_testing import (check_response, retry, skip, skip2, skip3, + web_front_end, requires_acls) import swift_testing +from test.functional.tests import load_constraint class TestAccount(unittest.TestCase): + def setUp(self): + self.max_meta_count = load_constraint('max_meta_count') + self.max_meta_name_length = load_constraint('max_meta_name_length') + self.max_meta_overall_size = load_constraint('max_meta_overall_size') + self.max_meta_value_length = load_constraint('max_meta_value_length') + + def head(url, token, parsed, conn): + conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(head) + self.existing_metadata = set([ + k for k, v in resp.getheaders() if + k.lower().startswith('x-account-meta')]) + + def tearDown(self): + def head(url, token, parsed, conn): + conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(head) + resp.read() + new_metadata = set( + [k for k, v in resp.getheaders() if + k.lower().startswith('x-account-meta')]) + + def clear_meta(url, token, parsed, conn, remove_metadata_keys): + headers = {'X-Auth-Token': token} + headers.update((k, '') for k in remove_metadata_keys) + conn.request('POST', parsed.path, '', headers) + return check_response(conn) + extra_metadata = list(self.existing_metadata ^ new_metadata) + for i in range(0, len(extra_metadata), 90): + batch = extra_metadata[i:i + 90] + resp = retry(clear_meta, batch) + resp.read() + self.assertEqual(resp.status // 100, 2) + def test_metadata(self): if skip: raise SkipTest @@ -49,49 +87,338 @@ class TestAccount(unittest.TestCase): resp = retry(post, '') resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader('x-account-meta-test'), None) + self.assertEqual(resp.getheader('x-account-meta-test'), None) resp = retry(get) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader('x-account-meta-test'), None) + self.assertEqual(resp.getheader('x-account-meta-test'), None) resp = retry(post, 'Value') resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader('x-account-meta-test'), 'Value') + self.assertEqual(resp.getheader('x-account-meta-test'), 'Value') resp = retry(get) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader('x-account-meta-test'), 'Value') + self.assertEqual(resp.getheader('x-account-meta-test'), 'Value') - def test_tempauth_account_acls(self): - if skip: + def test_invalid_acls(self): + def post(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + # needs to be an acceptable header size + num_keys = 8 + max_key_size = load_constraint('max_header_size') / num_keys + acl = {'admin': [c * max_key_size for c in letters[:num_keys]]} + headers = {'x-account-access-control': format_acl( + version=2, acl_dict=acl)} + resp = retry(post, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 400) + + # and again a touch smaller + acl = {'admin': [c * max_key_size for c in letters[:num_keys - 1]]} + headers = {'x-account-access-control': format_acl( + version=2, acl_dict=acl)} + resp = retry(post, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + @requires_acls + def test_invalid_acl_keys(self): + def post(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + # needs to be json + resp = retry(post, headers={'X-Account-Access-Control': 'invalid'}, + use_account=1) + resp.read() + self.assertEqual(resp.status, 400) + + acl_user = swift_testing.swift_test_user[1] + acl = {'admin': [acl_user], 'invalid_key': 'invalid_value'} + headers = {'x-account-access-control': format_acl( + version=2, acl_dict=acl)} + + resp = retry(post, headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 400) + self.assertEqual(resp.getheader('X-Account-Access-Control'), None) + + @requires_acls + def test_invalid_acl_values(self): + def post(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + acl = {'admin': 'invalid_value'} + headers = {'x-account-access-control': format_acl( + version=2, acl_dict=acl)} + + resp = retry(post, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 400) + self.assertEqual(resp.getheader('X-Account-Access-Control'), None) + + @requires_acls + def test_read_only_acl(self): + if skip3: raise SkipTest - # Determine whether this cluster has account ACLs; if not, skip test - conn = Connection(get_config('func_test')) - conn.authenticate() - status = conn.make_request( - 'GET', '/info', cfg={'verbatim_path': True}) - if status // 100 != 2: - # Can't tell if account ACLs are enabled; skip tests proactively. + def get(url, token, parsed, conn): + conn.request('GET', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + + def post(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + # cannot read account + resp = retry(get, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # grant read access + acl_user = swift_testing.swift_test_user[2] + acl = {'read-only': [acl_user]} + headers = {'x-account-access-control': format_acl( + version=2, acl_dict=acl)} + resp = retry(post, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # read-only can read account headers + resp = retry(get, use_account=3) + resp.read() + self.assert_(resp.status in (200, 204)) + # but not acls + self.assertEqual(resp.getheader('X-Account-Access-Control'), None) + + # read-only can not write metadata + headers = {'x-account-meta-test': 'value'} + resp = retry(post, headers=headers, use_account=3) + resp.read() + self.assertEqual(resp.status, 403) + + # but they can read it + headers = {'x-account-meta-test': 'value'} + resp = retry(post, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + resp = retry(get, use_account=3) + resp.read() + self.assert_(resp.status in (200, 204)) + self.assertEqual(resp.getheader('X-Account-Meta-Test'), 'value') + + @requires_acls + def test_read_write_acl(self): + if skip3: + raise SkipTest + + def get(url, token, parsed, conn): + conn.request('GET', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + + def post(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + # cannot read account + resp = retry(get, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # grant read-write access + acl_user = swift_testing.swift_test_user[2] + acl = {'read-write': [acl_user]} + headers = {'x-account-access-control': format_acl( + version=2, acl_dict=acl)} + resp = retry(post, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # read-write can read account headers + resp = retry(get, use_account=3) + resp.read() + self.assert_(resp.status in (200, 204)) + # but not acls + self.assertEqual(resp.getheader('X-Account-Access-Control'), None) + + # read-write can not write account metadata + headers = {'x-account-meta-test': 'value'} + resp = retry(post, headers=headers, use_account=3) + resp.read() + self.assertEqual(resp.status, 403) + + @requires_acls + def test_admin_acl(self): + if skip3: + raise SkipTest + + def get(url, token, parsed, conn): + conn.request('GET', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + + def post(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + # cannot read account + resp = retry(get, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # grant admin access + acl_user = swift_testing.swift_test_user[2] + acl = {'admin': [acl_user]} + acl_json_str = format_acl(version=2, acl_dict=acl) + headers = {'x-account-access-control': acl_json_str} + resp = retry(post, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # admin can read account headers + resp = retry(get, use_account=3) + resp.read() + self.assert_(resp.status in (200, 204)) + # including acls + self.assertEqual(resp.getheader('X-Account-Access-Control'), + acl_json_str) + + # admin can write account metadata + value = str(uuid4()) + headers = {'x-account-meta-test': value} + resp = retry(post, headers=headers, use_account=3) + resp.read() + self.assertEqual(resp.status, 204) + resp = retry(get, use_account=3) + resp.read() + self.assert_(resp.status in (200, 204)) + self.assertEqual(resp.getheader('X-Account-Meta-Test'), value) + + # admin can even revoke their own access + headers = {'x-account-access-control': '{}'} + resp = retry(post, headers=headers, use_account=3) + resp.read() + self.assertEqual(resp.status, 204) + + # and again, cannot read account + resp = retry(get, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + @requires_acls + def test_protected_tempurl(self): + if skip3: + raise SkipTest + + def get(url, token, parsed, conn): + conn.request('GET', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + + def post(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + # add a account metadata, and temp-url-key to account + value = str(uuid4()) + headers = { + 'x-account-meta-temp-url-key': 'secret', + 'x-account-meta-test': value, + } + resp = retry(post, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # grant read-only access to tester3 + acl_user = swift_testing.swift_test_user[2] + acl = {'read-only': [acl_user]} + acl_json_str = format_acl(version=2, acl_dict=acl) + headers = {'x-account-access-control': acl_json_str} + resp = retry(post, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # read-only tester3 can read account metadata + resp = retry(get, use_account=3) + resp.read() + self.assert_(resp.status in (200, 204), + 'Expected status in (200, 204), got %s' % resp.status) + self.assertEqual(resp.getheader('X-Account-Meta-Test'), value) + # but not temp-url-key + self.assertEqual(resp.getheader('X-Account-Meta-Temp-Url-Key'), None) + + # grant read-write access to tester3 + acl_user = swift_testing.swift_test_user[2] + acl = {'read-write': [acl_user]} + acl_json_str = format_acl(version=2, acl_dict=acl) + headers = {'x-account-access-control': acl_json_str} + resp = retry(post, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # read-write tester3 can read account metadata + resp = retry(get, use_account=3) + resp.read() + self.assert_(resp.status in (200, 204), + 'Expected status in (200, 204), got %s' % resp.status) + self.assertEqual(resp.getheader('X-Account-Meta-Test'), value) + # but not temp-url-key + self.assertEqual(resp.getheader('X-Account-Meta-Temp-Url-Key'), None) + + # grant admin access to tester3 + acl_user = swift_testing.swift_test_user[2] + acl = {'admin': [acl_user]} + acl_json_str = format_acl(version=2, acl_dict=acl) + headers = {'x-account-access-control': acl_json_str} + resp = retry(post, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # admin tester3 can read account metadata + resp = retry(get, use_account=3) + resp.read() + self.assert_(resp.status in (200, 204), + 'Expected status in (200, 204), got %s' % resp.status) + self.assertEqual(resp.getheader('X-Account-Meta-Test'), value) + # including temp-url-key + self.assertEqual(resp.getheader('X-Account-Meta-Temp-Url-Key'), + 'secret') + + # admin tester3 can even change temp-url-key + secret = str(uuid4()) + headers = { + 'x-account-meta-temp-url-key': secret, + } + resp = retry(post, headers=headers, use_account=3) + resp.read() + self.assertEqual(resp.status, 204) + resp = retry(get, use_account=3) + resp.read() + self.assert_(resp.status in (200, 204), + 'Expected status in (200, 204), got %s' % resp.status) + self.assertEqual(resp.getheader('X-Account-Meta-Temp-Url-Key'), + secret) + + @requires_acls + def test_account_acls(self): + if skip2: raise SkipTest - else: - cluster_info = json.loads(conn.response.read()) - if not cluster_info.get('tempauth', {}).get('account_acls'): - raise SkipTest - if 'keystoneauth' in cluster_info: - # Unfortunate hack -- tempauth (with account ACLs) is expected - # to play nice with Keystone (without account ACLs), but Zuul - # functest framework doesn't give us an easy way to get a - # tempauth user. - raise SkipTest def post(url, token, parsed, conn, headers): new_headers = dict({'X-Auth-Token': token}, **headers) @@ -212,6 +539,137 @@ class TestAccount(unittest.TestCase): use_account=1) resp.read() + @requires_acls + def test_swift_account_acls(self): + if skip: + raise SkipTest + + def post(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + def head(url, token, parsed, conn): + conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + + def get(url, token, parsed, conn): + conn.request('GET', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + + try: + # User1 can POST to their own account + resp = retry(post, headers={'X-Account-Access-Control': '{}'}) + resp.read() + self.assertEqual(resp.status, 204) + self.assertEqual(resp.getheader('X-Account-Access-Control'), None) + + # User1 can GET their own empty account + resp = retry(get) + resp.read() + self.assertEqual(resp.status // 100, 2) + self.assertEqual(resp.getheader('X-Account-Access-Control'), None) + + # User1 can POST non-empty data + acl_json = '{"admin":["bob"]}' + resp = retry(post, headers={'X-Account-Access-Control': acl_json}) + resp.read() + self.assertEqual(resp.status, 204) + + # User1 can GET the non-empty data + resp = retry(get) + resp.read() + self.assertEqual(resp.status // 100, 2) + self.assertEqual(resp.getheader('X-Account-Access-Control'), + acl_json) + + # POST non-JSON ACL should fail + resp = retry(post, headers={'X-Account-Access-Control': 'yuck'}) + resp.read() + # resp.status will be 400 if tempauth or some other ACL-aware + # auth middleware rejects it, or 200 (but silently swallowed by + # core Swift) if ACL-unaware auth middleware approves it. + + # A subsequent GET should show the old, valid data, not the garbage + resp = retry(get) + resp.read() + self.assertEqual(resp.status // 100, 2) + self.assertEqual(resp.getheader('X-Account-Access-Control'), + acl_json) + + finally: + # Make sure to clean up even if tests fail -- User2 should not + # have access to User1's account in other functional tests! + resp = retry(post, headers={'X-Account-Access-Control': '{}'}) + resp.read() + + def test_swift_prohibits_garbage_account_acls(self): + if skip: + raise SkipTest + + def post(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + def get(url, token, parsed, conn): + conn.request('GET', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + + try: + # User1 can POST to their own account + resp = retry(post, headers={'X-Account-Access-Control': '{}'}) + resp.read() + self.assertEqual(resp.status, 204) + self.assertEqual(resp.getheader('X-Account-Access-Control'), None) + + # User1 can GET their own empty account + resp = retry(get) + resp.read() + self.assertEqual(resp.status // 100, 2) + self.assertEqual(resp.getheader('X-Account-Access-Control'), None) + + # User1 can POST non-empty data + acl_json = '{"admin":["bob"]}' + resp = retry(post, headers={'X-Account-Access-Control': acl_json}) + resp.read() + self.assertEqual(resp.status, 204) + # If this request is handled by ACL-aware auth middleware, then the + # ACL will be persisted. If it is handled by ACL-unaware auth + # middleware, then the header will be thrown out. But the request + # should return successfully in any case. + + # User1 can GET the non-empty data + resp = retry(get) + resp.read() + self.assertEqual(resp.status // 100, 2) + # ACL will be set if some ACL-aware auth middleware (e.g. tempauth) + # propagates it to sysmeta; if no ACL-aware auth middleware does, + # then X-Account-Access-Control will still be empty. + + # POST non-JSON ACL should fail + resp = retry(post, headers={'X-Account-Access-Control': 'yuck'}) + resp.read() + # resp.status will be 400 if tempauth or some other ACL-aware + # auth middleware rejects it, or 200 (but silently swallowed by + # core Swift) if ACL-unaware auth middleware approves it. + + # A subsequent GET should either show the old, valid data (if + # ACL-aware auth middleware is propagating it) or show nothing + # (if no auth middleware in the pipeline is ACL-aware), but should + # never return the garbage ACL. + resp = retry(get) + resp.read() + self.assertEqual(resp.status // 100, 2) + self.assertNotEqual(resp.getheader('X-Account-Access-Control'), + 'yuck') + + finally: + # Make sure to clean up even if tests fail -- User2 should not + # have access to User1's account in other functional tests! + resp = retry(post, headers={'X-Account-Access-Control': '{}'}) + resp.read() + def test_unicode_metadata(self): if skip: raise SkipTest @@ -233,24 +691,24 @@ class TestAccount(unittest.TestCase): resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader(uni_key.encode('utf-8')), '1') + self.assertEqual(resp.getheader(uni_key.encode('utf-8')), '1') resp = retry(post, 'X-Account-Meta-uni', uni_value) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader('X-Account-Meta-uni'), - uni_value.encode('utf-8')) + self.assertEqual(resp.getheader('X-Account-Meta-uni'), + uni_value.encode('utf-8')) if (web_front_end == 'integral'): resp = retry(post, uni_key, uni_value) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader(uni_key.encode('utf-8')), - uni_value.encode('utf-8')) + self.assertEqual(resp.getheader(uni_key.encode('utf-8')), + uni_value.encode('utf-8')) def test_multi_metadata(self): if skip: @@ -267,19 +725,19 @@ class TestAccount(unittest.TestCase): resp = retry(post, 'X-Account-Meta-One', '1') resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader('x-account-meta-one'), '1') + self.assertEqual(resp.getheader('x-account-meta-one'), '1') resp = retry(post, 'X-Account-Meta-Two', '2') resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader('x-account-meta-one'), '1') - self.assertEquals(resp.getheader('x-account-meta-two'), '2') + self.assertEqual(resp.getheader('x-account-meta-one'), '1') + self.assertEqual(resp.getheader('x-account-meta-two'), '2') def test_bad_metadata(self): if skip: @@ -294,35 +752,65 @@ class TestAccount(unittest.TestCase): resp = retry(post, {'X-Account-Meta-' + ('k' * MAX_META_NAME_LENGTH): 'v'}) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry( post, {'X-Account-Meta-' + ('k' * (MAX_META_NAME_LENGTH + 1)): 'v'}) resp.read() - self.assertEquals(resp.status, 400) + self.assertEqual(resp.status, 400) resp = retry(post, {'X-Account-Meta-Too-Long': 'k' * MAX_META_VALUE_LENGTH}) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry( post, {'X-Account-Meta-Too-Long': 'k' * (MAX_META_VALUE_LENGTH + 1)}) resp.read() - self.assertEquals(resp.status, 400) + self.assertEqual(resp.status, 400) + + def test_bad_metadata2(self): + if skip: + raise SkipTest + + def post(url, token, parsed, conn, extra_headers): + headers = {'X-Auth-Token': token} + headers.update(extra_headers) + conn.request('POST', parsed.path, '', headers) + return check_response(conn) + + # TODO: Find the test that adds these and remove them. + headers = {'x-remove-account-meta-temp-url-key': 'remove', + 'x-remove-account-meta-temp-url-key-2': 'remove'} + resp = retry(post, headers) headers = {} for x in xrange(MAX_META_COUNT): headers['X-Account-Meta-%d' % x] = 'v' resp = retry(post, headers) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) headers = {} for x in xrange(MAX_META_COUNT + 1): headers['X-Account-Meta-%d' % x] = 'v' resp = retry(post, headers) resp.read() - self.assertEquals(resp.status, 400) + self.assertEqual(resp.status, 400) + + def test_bad_metadata3(self): + if skip: + raise SkipTest + + def post(url, token, parsed, conn, extra_headers): + headers = {'X-Auth-Token': token} + headers.update(extra_headers) + conn.request('POST', parsed.path, '', headers) + return check_response(conn) + + # TODO: Find the test that adds these and remove them. + headers = {'x-remove-account-meta-temp-url-key': 'remove', + 'x-remove-account-meta-temp-url-key-2': 'remove'} + resp = retry(post, headers) headers = {} header_value = 'k' * MAX_META_VALUE_LENGTH @@ -337,12 +825,12 @@ class TestAccount(unittest.TestCase): 'v' * (MAX_META_OVERALL_SIZE - size - 1) resp = retry(post, headers) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) headers['X-Account-Meta-k'] = \ 'v' * (MAX_META_OVERALL_SIZE - size) resp = retry(post, headers) resp.read() - self.assertEquals(resp.status, 400) + self.assertEqual(resp.status, 400) if __name__ == '__main__': diff --git a/test/functional/test_container.py b/test/functional/test_container.py index 15f7fc1..91702e9 100755 --- a/test/functional/test_container.py +++ b/test/functional/test_container.py @@ -24,7 +24,7 @@ from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \ MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH from swift_testing import check_response, retry, skip, skip2, skip3, \ - swift_test_perm, web_front_end + swift_test_perm, web_front_end, requires_acls, swift_test_user class TestContainer(unittest.TestCase): @@ -41,7 +41,7 @@ class TestContainer(unittest.TestCase): resp = retry(put) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) def tearDown(self): if skip: @@ -68,7 +68,7 @@ class TestContainer(unittest.TestCase): for obj in objs: resp = retry(delete, obj) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) def delete(url, token, parsed, conn): conn.request('DELETE', parsed.path + '/' + self.name, '', @@ -77,7 +77,7 @@ class TestContainer(unittest.TestCase): resp = retry(delete) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) def test_multi_metadata(self): if skip: @@ -95,19 +95,19 @@ class TestContainer(unittest.TestCase): resp = retry(post, 'X-Container-Meta-One', '1') resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader('x-container-meta-one'), '1') + self.assertEqual(resp.getheader('x-container-meta-one'), '1') resp = retry(post, 'X-Container-Meta-Two', '2') resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader('x-container-meta-one'), '1') - self.assertEquals(resp.getheader('x-container-meta-two'), '2') + self.assertEqual(resp.getheader('x-container-meta-one'), '1') + self.assertEqual(resp.getheader('x-container-meta-two'), '2') def test_unicode_metadata(self): if skip: @@ -128,28 +128,28 @@ class TestContainer(unittest.TestCase): if (web_front_end == 'integral'): resp = retry(post, uni_key, '1') resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader(uni_key.encode('utf-8')), '1') + self.assertEqual(resp.getheader(uni_key.encode('utf-8')), '1') resp = retry(post, 'X-Container-Meta-uni', uni_value) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader('X-Container-Meta-uni'), - uni_value.encode('utf-8')) + self.assertEqual(resp.getheader('X-Container-Meta-uni'), + uni_value.encode('utf-8')) if (web_front_end == 'integral'): resp = retry(post, uni_key, uni_value) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader(uni_key.encode('utf-8')), - uni_value.encode('utf-8')) + self.assertEqual(resp.getheader(uni_key.encode('utf-8')), + uni_value.encode('utf-8')) def test_PUT_metadata(self): if skip: @@ -179,34 +179,34 @@ class TestContainer(unittest.TestCase): name = uuid4().hex resp = retry(put, name, 'Value') resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) resp = retry(head, name) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader('x-container-meta-test'), 'Value') + self.assertEqual(resp.getheader('x-container-meta-test'), 'Value') resp = retry(get, name) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader('x-container-meta-test'), 'Value') + self.assertEqual(resp.getheader('x-container-meta-test'), 'Value') resp = retry(delete, name) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) name = uuid4().hex resp = retry(put, name, '') resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) resp = retry(head, name) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader('x-container-meta-test'), None) + self.assertEqual(resp.getheader('x-container-meta-test'), None) resp = retry(get, name) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader('x-container-meta-test'), None) + self.assertEqual(resp.getheader('x-container-meta-test'), None) resp = retry(delete, name) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) def test_POST_metadata(self): if skip: @@ -231,22 +231,22 @@ class TestContainer(unittest.TestCase): resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader('x-container-meta-test'), None) + self.assertEqual(resp.getheader('x-container-meta-test'), None) resp = retry(get) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader('x-container-meta-test'), None) + self.assertEqual(resp.getheader('x-container-meta-test'), None) resp = retry(post, 'Value') resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader('x-container-meta-test'), 'Value') + self.assertEqual(resp.getheader('x-container-meta-test'), 'Value') resp = retry(get) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader('x-container-meta-test'), 'Value') + self.assertEqual(resp.getheader('x-container-meta-test'), 'Value') def test_PUT_bad_metadata(self): if skip: @@ -268,38 +268,38 @@ class TestContainer(unittest.TestCase): put, name, {'X-Container-Meta-' + ('k' * MAX_META_NAME_LENGTH): 'v'}) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) resp = retry(delete, name) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) name = uuid4().hex resp = retry( put, name, {'X-Container-Meta-' + ('k' * (MAX_META_NAME_LENGTH + 1)): 'v'}) resp.read() - self.assertEquals(resp.status, 400) + self.assertEqual(resp.status, 400) resp = retry(delete, name) resp.read() - self.assertEquals(resp.status, 404) + self.assertEqual(resp.status, 404) name = uuid4().hex resp = retry( put, name, {'X-Container-Meta-Too-Long': 'k' * MAX_META_VALUE_LENGTH}) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) resp = retry(delete, name) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) name = uuid4().hex resp = retry( put, name, {'X-Container-Meta-Too-Long': 'k' * (MAX_META_VALUE_LENGTH + 1)}) resp.read() - self.assertEquals(resp.status, 400) + self.assertEqual(resp.status, 400) resp = retry(delete, name) resp.read() - self.assertEquals(resp.status, 404) + self.assertEqual(resp.status, 404) name = uuid4().hex headers = {} @@ -307,20 +307,20 @@ class TestContainer(unittest.TestCase): headers['X-Container-Meta-%d' % x] = 'v' resp = retry(put, name, headers) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) resp = retry(delete, name) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) name = uuid4().hex headers = {} for x in xrange(MAX_META_COUNT + 1): headers['X-Container-Meta-%d' % x] = 'v' resp = retry(put, name, headers) resp.read() - self.assertEquals(resp.status, 400) + self.assertEqual(resp.status, 400) resp = retry(delete, name) resp.read() - self.assertEquals(resp.status, 404) + self.assertEqual(resp.status, 404) name = uuid4().hex headers = {} @@ -336,19 +336,19 @@ class TestContainer(unittest.TestCase): 'v' * (MAX_META_OVERALL_SIZE - size - 1) resp = retry(put, name, headers) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) resp = retry(delete, name) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) name = uuid4().hex headers['X-Container-Meta-k'] = \ 'v' * (MAX_META_OVERALL_SIZE - size) resp = retry(put, name, headers) resp.read() - self.assertEquals(resp.status, 400) + self.assertEqual(resp.status, 400) resp = retry(delete, name) resp.read() - self.assertEquals(resp.status, 404) + self.assertEqual(resp.status, 404) def test_POST_bad_metadata(self): if skip: @@ -364,36 +364,56 @@ class TestContainer(unittest.TestCase): post, {'X-Container-Meta-' + ('k' * MAX_META_NAME_LENGTH): 'v'}) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry( post, {'X-Container-Meta-' + ('k' * (MAX_META_NAME_LENGTH + 1)): 'v'}) resp.read() - self.assertEquals(resp.status, 400) + self.assertEqual(resp.status, 400) resp = retry( post, {'X-Container-Meta-Too-Long': 'k' * MAX_META_VALUE_LENGTH}) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry( post, {'X-Container-Meta-Too-Long': 'k' * (MAX_META_VALUE_LENGTH + 1)}) resp.read() - self.assertEquals(resp.status, 400) + self.assertEqual(resp.status, 400) + + def test_POST_bad_metadata2(self): + if skip: + raise SkipTest + + def post(url, token, parsed, conn, extra_headers): + headers = {'X-Auth-Token': token} + headers.update(extra_headers) + conn.request('POST', parsed.path + '/' + self.name, '', headers) + return check_response(conn) headers = {} for x in xrange(MAX_META_COUNT): headers['X-Container-Meta-%d' % x] = 'v' resp = retry(post, headers) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) headers = {} for x in xrange(MAX_META_COUNT + 1): headers['X-Container-Meta-%d' % x] = 'v' resp = retry(post, headers) resp.read() - self.assertEquals(resp.status, 400) + self.assertEqual(resp.status, 400) + + def test_POST_bad_metadata3(self): + if skip: + raise SkipTest + + def post(url, token, parsed, conn, extra_headers): + headers = {'X-Auth-Token': token} + headers.update(extra_headers) + conn.request('POST', parsed.path + '/' + self.name, '', headers) + return check_response(conn) headers = {} header_value = 'k' * MAX_META_VALUE_LENGTH @@ -408,12 +428,12 @@ class TestContainer(unittest.TestCase): 'v' * (MAX_META_OVERALL_SIZE - size - 1) resp = retry(post, headers) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) headers['X-Container-Meta-k'] = \ 'v' * (MAX_META_OVERALL_SIZE - size) resp = retry(post, headers) resp.read() - self.assertEquals(resp.status, 400) + self.assertEqual(resp.status, 400) def test_public_container(self): if skip: @@ -437,10 +457,10 @@ class TestContainer(unittest.TestCase): resp = retry(post) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry(get) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) def post(url, token, parsed, conn): conn.request('POST', parsed.path + '/' + self.name, '', @@ -449,7 +469,7 @@ class TestContainer(unittest.TestCase): resp = retry(post) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) try: resp = retry(get) raise Exception('Should not have been able to GET') @@ -479,7 +499,7 @@ class TestContainer(unittest.TestCase): resp = retry(get2, use_account=2) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # Make the container accessible by the second account def post(url, token, parsed, conn): @@ -491,11 +511,11 @@ class TestContainer(unittest.TestCase): resp = retry(post) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # Ensure we can now use the container with the second account resp = retry(get2, use_account=2) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # Make the container private again def post(url, token, parsed, conn): @@ -506,11 +526,11 @@ class TestContainer(unittest.TestCase): resp = retry(post) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # Ensure we can't access the container with the second account again resp = retry(get2, use_account=2) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) def test_cross_account_public_container(self): if skip or skip2: @@ -535,7 +555,7 @@ class TestContainer(unittest.TestCase): resp = retry(get2, use_account=2) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # Make the container completely public def post(url, token, parsed, conn): @@ -546,11 +566,11 @@ class TestContainer(unittest.TestCase): resp = retry(post) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # Ensure we can now read the container with the second account resp = retry(get2, use_account=2) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # But we shouldn't be able to write with the second account def put2(url, token, parsed, conn): @@ -560,7 +580,7 @@ class TestContainer(unittest.TestCase): resp = retry(put2, use_account=2) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # Now make the container also writeable by the second account def post(url, token, parsed, conn): @@ -571,15 +591,15 @@ class TestContainer(unittest.TestCase): resp = retry(post) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # Ensure we can still read the container with the second account resp = retry(get2, use_account=2) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # And that we can now write with the second account resp = retry(put2, use_account=2) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) def test_nonadmin_user(self): if skip or skip3: @@ -604,7 +624,7 @@ class TestContainer(unittest.TestCase): resp = retry(get3, use_account=3) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # Make the container accessible by the third account def post(url, token, parsed, conn): @@ -615,11 +635,11 @@ class TestContainer(unittest.TestCase): resp = retry(post) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # Ensure we can now read the container with the third account resp = retry(get3, use_account=3) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # But we shouldn't be able to write with the third account def put3(url, token, parsed, conn): @@ -629,7 +649,7 @@ class TestContainer(unittest.TestCase): resp = retry(put3, use_account=3) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # Now make the container also writeable by the third account def post(url, token, parsed, conn): @@ -640,15 +660,666 @@ class TestContainer(unittest.TestCase): resp = retry(post) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # Ensure we can still read the container with the third account resp = retry(get3, use_account=3) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # And that we can now write with the third account resp = retry(put3, use_account=3) resp.read() + self.assertEqual(resp.status, 201) + + @requires_acls + def test_read_only_acl_listings(self): + if skip3: + raise SkipTest + + def get(url, token, parsed, conn): + conn.request('GET', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + + def post_account(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + def put(url, token, parsed, conn, name): + conn.request('PUT', parsed.path + '/%s' % name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + # cannot list containers + resp = retry(get, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # grant read-only access + acl_user = swift_test_user[2] + acl = {'read-only': [acl_user]} + headers = {'x-account-access-control': json.dumps(acl)} + resp = retry(post_account, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # read-only can list containers + resp = retry(get, use_account=3) + listing = resp.read() + self.assertEquals(resp.status, 200) + self.assert_(self.name in listing) + + # read-only can not create containers + new_container_name = str(uuid4()) + resp = retry(put, new_container_name, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # but it can see newly created ones + resp = retry(put, new_container_name, use_account=1) + resp.read() self.assertEquals(resp.status, 201) + resp = retry(get, use_account=3) + listing = resp.read() + self.assertEquals(resp.status, 200) + self.assert_(new_container_name in listing) + + @requires_acls + def test_read_only_acl_metadata(self): + if skip3: + raise SkipTest + + def get(url, token, parsed, conn, name): + conn.request('GET', parsed.path + '/%s' % name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + def post_account(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + def post(url, token, parsed, conn, name, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path + '/%s' % name, '', new_headers) + return check_response(conn) + + # add some metadata + value = str(uuid4()) + headers = {'x-container-meta-test': value} + resp = retry(post, self.name, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + resp = retry(get, self.name, use_account=1) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) + + # cannot see metadata + resp = retry(get, self.name, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # grant read-only access + acl_user = swift_test_user[2] + acl = {'read-only': [acl_user]} + headers = {'x-account-access-control': json.dumps(acl)} + resp = retry(post_account, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # read-only can NOT write container metadata + new_value = str(uuid4()) + headers = {'x-container-meta-test': new_value} + resp = retry(post, self.name, headers=headers, use_account=3) + resp.read() + self.assertEqual(resp.status, 403) + + # read-only can read container metadata + resp = retry(get, self.name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) + + @requires_acls + def test_read_write_acl_listings(self): + if skip3: + raise SkipTest + + def get(url, token, parsed, conn): + conn.request('GET', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + + def post(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + def put(url, token, parsed, conn, name): + conn.request('PUT', parsed.path + '/%s' % name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + def delete(url, token, parsed, conn, name): + conn.request('DELETE', parsed.path + '/%s' % name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + # cannot list containers + resp = retry(get, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # grant read-write access + acl_user = swift_test_user[2] + acl = {'read-write': [acl_user]} + headers = {'x-account-access-control': json.dumps(acl)} + resp = retry(post, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # can list containers + resp = retry(get, use_account=3) + listing = resp.read() + self.assertEquals(resp.status, 200) + self.assert_(self.name in listing) + + # can create new containers + new_container_name = str(uuid4()) + resp = retry(put, new_container_name, use_account=3) + resp.read() + self.assertEquals(resp.status, 201) + resp = retry(get, use_account=3) + listing = resp.read() + self.assertEquals(resp.status, 200) + self.assert_(new_container_name in listing) + + # can also delete them + resp = retry(delete, new_container_name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(get, use_account=3) + listing = resp.read() + self.assertEquals(resp.status, 200) + self.assert_(new_container_name not in listing) + + # even if they didn't create them + empty_container_name = str(uuid4()) + resp = retry(put, empty_container_name, use_account=1) + resp.read() + self.assertEquals(resp.status, 201) + resp = retry(delete, empty_container_name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + + @requires_acls + def test_read_write_acl_metadata(self): + if skip3: + raise SkipTest + + def get(url, token, parsed, conn, name): + conn.request('GET', parsed.path + '/%s' % name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + def post_account(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + def post(url, token, parsed, conn, name, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path + '/%s' % name, '', new_headers) + return check_response(conn) + + # add some metadata + value = str(uuid4()) + headers = {'x-container-meta-test': value} + resp = retry(post, self.name, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + resp = retry(get, self.name, use_account=1) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) + + # cannot see metadata + resp = retry(get, self.name, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # grant read-write access + acl_user = swift_test_user[2] + acl = {'read-write': [acl_user]} + headers = {'x-account-access-control': json.dumps(acl)} + resp = retry(post_account, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # read-write can read container metadata + resp = retry(get, self.name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) + + # read-write can also write container metadata + new_value = str(uuid4()) + headers = {'x-container-meta-test': new_value} + resp = retry(post, self.name, headers=headers, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(get, self.name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Meta-Test'), new_value) + + # and remove it + headers = {'x-remove-container-meta-test': 'true'} + resp = retry(post, self.name, headers=headers, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(get, self.name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Meta-Test'), None) + + @requires_acls + def test_admin_acl_listing(self): + if skip3: + raise SkipTest + + def get(url, token, parsed, conn): + conn.request('GET', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + + def post(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + def put(url, token, parsed, conn, name): + conn.request('PUT', parsed.path + '/%s' % name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + def delete(url, token, parsed, conn, name): + conn.request('DELETE', parsed.path + '/%s' % name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + # cannot list containers + resp = retry(get, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # grant admin access + acl_user = swift_test_user[2] + acl = {'admin': [acl_user]} + headers = {'x-account-access-control': json.dumps(acl)} + resp = retry(post, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # can list containers + resp = retry(get, use_account=3) + listing = resp.read() + self.assertEquals(resp.status, 200) + self.assert_(self.name in listing) + + # can create new containers + new_container_name = str(uuid4()) + resp = retry(put, new_container_name, use_account=3) + resp.read() + self.assertEquals(resp.status, 201) + resp = retry(get, use_account=3) + listing = resp.read() + self.assertEquals(resp.status, 200) + self.assert_(new_container_name in listing) + + # can also delete them + resp = retry(delete, new_container_name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(get, use_account=3) + listing = resp.read() + self.assertEquals(resp.status, 200) + self.assert_(new_container_name not in listing) + + # even if they didn't create them + empty_container_name = str(uuid4()) + resp = retry(put, empty_container_name, use_account=1) + resp.read() + self.assertEquals(resp.status, 201) + resp = retry(delete, empty_container_name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + + @requires_acls + def test_admin_acl_metadata(self): + if skip3: + raise SkipTest + + def get(url, token, parsed, conn, name): + conn.request('GET', parsed.path + '/%s' % name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + def post_account(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + def post(url, token, parsed, conn, name, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path + '/%s' % name, '', new_headers) + return check_response(conn) + + # add some metadata + value = str(uuid4()) + headers = {'x-container-meta-test': value} + resp = retry(post, self.name, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + resp = retry(get, self.name, use_account=1) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) + + # cannot see metadata + resp = retry(get, self.name, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # grant access + acl_user = swift_test_user[2] + acl = {'admin': [acl_user]} + headers = {'x-account-access-control': json.dumps(acl)} + resp = retry(post_account, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # can read container metadata + resp = retry(get, self.name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) + + # can also write container metadata + new_value = str(uuid4()) + headers = {'x-container-meta-test': new_value} + resp = retry(post, self.name, headers=headers, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(get, self.name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Meta-Test'), new_value) + + # and remove it + headers = {'x-remove-container-meta-test': 'true'} + resp = retry(post, self.name, headers=headers, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(get, self.name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Meta-Test'), None) + + @requires_acls + def test_protected_container_sync(self): + if skip3: + raise SkipTest + + def get(url, token, parsed, conn, name): + conn.request('GET', parsed.path + '/%s' % name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + def post_account(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + def post(url, token, parsed, conn, name, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path + '/%s' % name, '', new_headers) + return check_response(conn) + + # add some metadata + value = str(uuid4()) + headers = { + 'x-container-sync-key': 'secret', + 'x-container-meta-test': value, + } + resp = retry(post, self.name, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + resp = retry(get, self.name, use_account=1) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Sync-Key'), 'secret') + self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) + + # grant read-only access + acl_user = swift_test_user[2] + acl = {'read-only': [acl_user]} + headers = {'x-account-access-control': json.dumps(acl)} + resp = retry(post_account, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # can read container metadata + resp = retry(get, self.name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) + # but not sync-key + self.assertEqual(resp.getheader('X-Container-Sync-Key'), None) + + # and can not write + headers = {'x-container-sync-key': str(uuid4())} + resp = retry(post, self.name, headers=headers, use_account=3) + resp.read() + self.assertEqual(resp.status, 403) + + # grant read-write access + acl_user = swift_test_user[2] + acl = {'read-write': [acl_user]} + headers = {'x-account-access-control': json.dumps(acl)} + resp = retry(post_account, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # can read container metadata + resp = retry(get, self.name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) + # but not sync-key + self.assertEqual(resp.getheader('X-Container-Sync-Key'), None) + + # sanity check sync-key w/ account1 + resp = retry(get, self.name, use_account=1) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Sync-Key'), 'secret') + + # and can write + new_value = str(uuid4()) + headers = { + 'x-container-sync-key': str(uuid4()), + 'x-container-meta-test': new_value, + } + resp = retry(post, self.name, headers=headers, use_account=3) + resp.read() + self.assertEqual(resp.status, 204) + resp = retry(get, self.name, use_account=1) # validate w/ account1 + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Meta-Test'), new_value) + # but can not write sync-key + self.assertEqual(resp.getheader('X-Container-Sync-Key'), 'secret') + + # grant admin access + acl_user = swift_test_user[2] + acl = {'admin': [acl_user]} + headers = {'x-account-access-control': json.dumps(acl)} + resp = retry(post_account, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # admin can read container metadata + resp = retry(get, self.name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Meta-Test'), new_value) + # and ALSO sync-key + self.assertEqual(resp.getheader('X-Container-Sync-Key'), 'secret') + + # admin tester3 can even change sync-key + new_secret = str(uuid4()) + headers = {'x-container-sync-key': new_secret} + resp = retry(post, self.name, headers=headers, use_account=3) + resp.read() + self.assertEqual(resp.status, 204) + resp = retry(get, self.name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Sync-Key'), new_secret) + + @requires_acls + def test_protected_container_acl(self): + if skip3: + raise SkipTest + + def get(url, token, parsed, conn, name): + conn.request('GET', parsed.path + '/%s' % name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + def post_account(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + def post(url, token, parsed, conn, name, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path + '/%s' % name, '', new_headers) + return check_response(conn) + + # add some container acls + value = str(uuid4()) + headers = { + 'x-container-read': 'jdoe', + 'x-container-write': 'jdoe', + 'x-container-meta-test': value, + } + resp = retry(post, self.name, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + resp = retry(get, self.name, use_account=1) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Read'), 'jdoe') + self.assertEqual(resp.getheader('X-Container-Write'), 'jdoe') + self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) + + # grant read-only access + acl_user = swift_test_user[2] + acl = {'read-only': [acl_user]} + headers = {'x-account-access-control': json.dumps(acl)} + resp = retry(post_account, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # can read container metadata + resp = retry(get, self.name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) + # but not container acl + self.assertEqual(resp.getheader('X-Container-Read'), None) + self.assertEqual(resp.getheader('X-Container-Write'), None) + + # and can not write + headers = { + 'x-container-read': 'frank', + 'x-container-write': 'frank', + } + resp = retry(post, self.name, headers=headers, use_account=3) + resp.read() + self.assertEqual(resp.status, 403) + + # grant read-write access + acl_user = swift_test_user[2] + acl = {'read-write': [acl_user]} + headers = {'x-account-access-control': json.dumps(acl)} + resp = retry(post_account, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # can read container metadata + resp = retry(get, self.name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) + # but not container acl + self.assertEqual(resp.getheader('X-Container-Read'), None) + self.assertEqual(resp.getheader('X-Container-Write'), None) + + # sanity check container acls with account1 + resp = retry(get, self.name, use_account=1) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Read'), 'jdoe') + self.assertEqual(resp.getheader('X-Container-Write'), 'jdoe') + + # and can write + new_value = str(uuid4()) + headers = { + 'x-container-read': 'frank', + 'x-container-write': 'frank', + 'x-container-meta-test': new_value, + } + resp = retry(post, self.name, headers=headers, use_account=3) + resp.read() + self.assertEqual(resp.status, 204) + resp = retry(get, self.name, use_account=1) # validate w/ account1 + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Meta-Test'), new_value) + # but can not write container acls + self.assertEqual(resp.getheader('X-Container-Read'), 'jdoe') + self.assertEqual(resp.getheader('X-Container-Write'), 'jdoe') + + # grant admin access + acl_user = swift_test_user[2] + acl = {'admin': [acl_user]} + headers = {'x-account-access-control': json.dumps(acl)} + resp = retry(post_account, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # admin can read container metadata + resp = retry(get, self.name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Meta-Test'), new_value) + # and ALSO container acls + self.assertEqual(resp.getheader('X-Container-Read'), 'jdoe') + self.assertEqual(resp.getheader('X-Container-Write'), 'jdoe') + + # admin tester3 can even change container acls + new_value = str(uuid4()) + headers = { + 'x-container-read': '.r:*', + } + resp = retry(post, self.name, headers=headers, use_account=3) + resp.read() + self.assertEqual(resp.status, 204) + resp = retry(get, self.name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Read'), '.r:*') def test_long_name_content_type(self): if skip: @@ -662,9 +1333,9 @@ class TestContainer(unittest.TestCase): resp = retry(put) resp.read() - self.assertEquals(resp.status, 400) - self.assertEquals(resp.getheader('Content-Type'), - 'text/html; charset=UTF-8') + self.assertEqual(resp.status, 400) + self.assertEqual(resp.getheader('Content-Type'), + 'text/html; charset=UTF-8') def test_null_name(self): if skip: @@ -677,10 +1348,10 @@ class TestContainer(unittest.TestCase): resp = retry(put) if (web_front_end == 'apache2'): - self.assertEquals(resp.status, 404) + self.assertEqual(resp.status, 404) else: - self.assertEquals(resp.read(), 'Invalid UTF8 or contains NULL') - self.assertEquals(resp.status, 412) + self.assertEqual(resp.read(), 'Invalid UTF8 or contains NULL') + self.assertEqual(resp.status, 412) if __name__ == '__main__': diff --git a/test/functional/test_object.py b/test/functional/test_object.py index dad8635..675de30 100755 --- a/test/functional/test_object.py +++ b/test/functional/test_object.py @@ -19,8 +19,10 @@ import unittest from nose import SkipTest from uuid import uuid4 +from swift.common.utils import json + from swift_testing import check_response, retry, skip, skip3, \ - swift_test_perm, web_front_end + swift_test_perm, web_front_end, requires_acls, swift_test_user class TestObject(unittest.TestCase): @@ -36,7 +38,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(put) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) self.obj = uuid4().hex def put(url, token, parsed, conn): @@ -46,7 +48,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(put) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) def tearDown(self): if skip: @@ -66,13 +68,13 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(list) object_listing = resp.read() - self.assertEquals(resp.status, 200) + self.assertEqual(resp.status, 200) # iterate over object listing and delete all objects for obj in object_listing.splitlines(): resp = retry(delete, obj) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # delete the container def delete(url, token, parsed, conn): @@ -81,7 +83,33 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(delete) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) + + def test_if_none_match(self): + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s/%s' % ( + parsed.path, self.container, 'if_none_match_test'), '', + {'X-Auth-Token': token, + 'Content-Length': '0', + 'If-None-Match': '*'}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 201) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 412) + + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s/%s' % ( + parsed.path, self.container, 'if_none_match_test'), '', + {'X-Auth-Token': token, + 'Content-Length': '0', + 'If-None-Match': 'somethingelse'}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 400) def test_copy_object(self): if skip: @@ -98,8 +126,8 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(get_source) source_contents = resp.read() - self.assertEquals(resp.status, 200) - self.assertEquals(source_contents, 'test') + self.assertEqual(resp.status, 200) + self.assertEqual(source_contents, 'test') # copy source to dest with X-Copy-From def put(url, token, parsed, conn): @@ -110,7 +138,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(put) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) # contents of dest should be the same as source def get_dest(url, token, parsed, conn): @@ -120,8 +148,8 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(get_dest) dest_contents = resp.read() - self.assertEquals(resp.status, 200) - self.assertEquals(dest_contents, source_contents) + self.assertEqual(resp.status, 200) + self.assertEqual(dest_contents, source_contents) # delete the copy def delete(url, token, parsed, conn): @@ -130,11 +158,11 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(delete) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # verify dest does not exist resp = retry(get_dest) resp.read() - self.assertEquals(resp.status, 404) + self.assertEqual(resp.status, 404) # copy source to dest with COPY def copy(url, token, parsed, conn): @@ -144,18 +172,18 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(copy) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) # contents of dest should be the same as source resp = retry(get_dest) dest_contents = resp.read() - self.assertEquals(resp.status, 200) - self.assertEquals(dest_contents, source_contents) + self.assertEqual(resp.status, 200) + self.assertEqual(dest_contents, source_contents) # delete the copy resp = retry(delete) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) def test_public_object(self): if skip: @@ -178,10 +206,10 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(post) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry(get) resp.read() - self.assertEquals(resp.status, 200) + self.assertEqual(resp.status, 200) def post(url, token, parsed, conn): conn.request('POST', parsed.path + '/' + self.container, '', @@ -189,7 +217,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(post) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) try: resp = retry(get) raise Exception('Should not have been able to GET') @@ -208,7 +236,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(get, use_account=3) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # create a shared container writable by account3 shared_container = uuid4().hex @@ -222,7 +250,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(put) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) # verify third account can not copy from private container def copy(url, token, parsed, conn): @@ -234,7 +262,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(copy, use_account=3) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # verify third account can write "obj1" to shared container def put(url, token, parsed, conn): @@ -244,7 +272,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(put, use_account=3) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) # verify third account can copy "obj1" to shared container def copy2(url, token, parsed, conn): @@ -255,7 +283,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(copy2, use_account=3) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) # verify third account STILL can not copy from private container def copy3(url, token, parsed, conn): @@ -267,7 +295,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(copy3, use_account=3) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # clean up "obj1" def delete(url, token, parsed, conn): @@ -277,7 +305,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(delete) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # clean up shared_container def delete(url, token, parsed, conn): @@ -287,8 +315,251 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(delete) resp.read() + self.assertEqual(resp.status, 204) + + @requires_acls + def test_read_only(self): + if skip3: + raise SkipTest + + def get_listing(url, token, parsed, conn): + conn.request('GET', '%s/%s' % (parsed.path, self.container), '', + {'X-Auth-Token': token}) + return check_response(conn) + + def post_account(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + def get(url, token, parsed, conn, name): + conn.request('GET', '%s/%s/%s' % ( + parsed.path, self.container, name), '', + {'X-Auth-Token': token}) + return check_response(conn) + + def put(url, token, parsed, conn, name): + conn.request('PUT', '%s/%s/%s' % ( + parsed.path, self.container, name), 'test', + {'X-Auth-Token': token}) + return check_response(conn) + + def delete(url, token, parsed, conn, name): + conn.request('PUT', '%s/%s/%s' % ( + parsed.path, self.container, name), '', + {'X-Auth-Token': token}) + return check_response(conn) + + # cannot list objects + resp = retry(get_listing, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # cannot get object + resp = retry(get, self.obj, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # grant read-only access + acl_user = swift_test_user[2] + acl = {'read-only': [acl_user]} + headers = {'x-account-access-control': json.dumps(acl)} + resp = retry(post_account, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # can list objects + resp = retry(get_listing, use_account=3) + listing = resp.read() + self.assertEquals(resp.status, 200) + self.assert_(self.obj in listing) + + # can get object + resp = retry(get, self.obj, use_account=3) + body = resp.read() + self.assertEquals(resp.status, 200) + self.assertEquals(body, 'test') + + # can not put an object + obj_name = str(uuid4()) + resp = retry(put, obj_name, use_account=3) + body = resp.read() + self.assertEquals(resp.status, 403) + + # can not delete an object + resp = retry(delete, self.obj, use_account=3) + body = resp.read() + self.assertEquals(resp.status, 403) + + # sanity with account1 + resp = retry(get_listing, use_account=3) + listing = resp.read() + self.assertEquals(resp.status, 200) + self.assert_(obj_name not in listing) + self.assert_(self.obj in listing) + + @requires_acls + def test_read_write(self): + if skip3: + raise SkipTest + + def get_listing(url, token, parsed, conn): + conn.request('GET', '%s/%s' % (parsed.path, self.container), '', + {'X-Auth-Token': token}) + return check_response(conn) + + def post_account(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + def get(url, token, parsed, conn, name): + conn.request('GET', '%s/%s/%s' % ( + parsed.path, self.container, name), '', + {'X-Auth-Token': token}) + return check_response(conn) + + def put(url, token, parsed, conn, name): + conn.request('PUT', '%s/%s/%s' % ( + parsed.path, self.container, name), 'test', + {'X-Auth-Token': token}) + return check_response(conn) + + def delete(url, token, parsed, conn, name): + conn.request('DELETE', '%s/%s/%s' % ( + parsed.path, self.container, name), '', + {'X-Auth-Token': token}) + return check_response(conn) + + # cannot list objects + resp = retry(get_listing, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # cannot get object + resp = retry(get, self.obj, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # grant read-write access + acl_user = swift_test_user[2] + acl = {'read-write': [acl_user]} + headers = {'x-account-access-control': json.dumps(acl)} + resp = retry(post_account, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # can list objects + resp = retry(get_listing, use_account=3) + listing = resp.read() + self.assertEquals(resp.status, 200) + self.assert_(self.obj in listing) + + # can get object + resp = retry(get, self.obj, use_account=3) + body = resp.read() + self.assertEquals(resp.status, 200) + self.assertEquals(body, 'test') + + # can put an object + obj_name = str(uuid4()) + resp = retry(put, obj_name, use_account=3) + body = resp.read() + self.assertEquals(resp.status, 201) + + # can delete an object + resp = retry(delete, self.obj, use_account=3) + body = resp.read() self.assertEquals(resp.status, 204) + # sanity with account1 + resp = retry(get_listing, use_account=3) + listing = resp.read() + self.assertEquals(resp.status, 200) + self.assert_(obj_name in listing) + self.assert_(self.obj not in listing) + + @requires_acls + def test_admin(self): + if skip3: + raise SkipTest + + def get_listing(url, token, parsed, conn): + conn.request('GET', '%s/%s' % (parsed.path, self.container), '', + {'X-Auth-Token': token}) + return check_response(conn) + + def post_account(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + def get(url, token, parsed, conn, name): + conn.request('GET', '%s/%s/%s' % ( + parsed.path, self.container, name), '', + {'X-Auth-Token': token}) + return check_response(conn) + + def put(url, token, parsed, conn, name): + conn.request('PUT', '%s/%s/%s' % ( + parsed.path, self.container, name), 'test', + {'X-Auth-Token': token}) + return check_response(conn) + + def delete(url, token, parsed, conn, name): + conn.request('DELETE', '%s/%s/%s' % ( + parsed.path, self.container, name), '', + {'X-Auth-Token': token}) + return check_response(conn) + + # cannot list objects + resp = retry(get_listing, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # cannot get object + resp = retry(get, self.obj, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # grant admin access + acl_user = swift_test_user[2] + acl = {'admin': [acl_user]} + headers = {'x-account-access-control': json.dumps(acl)} + resp = retry(post_account, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # can list objects + resp = retry(get_listing, use_account=3) + listing = resp.read() + self.assertEquals(resp.status, 200) + self.assert_(self.obj in listing) + + # can get object + resp = retry(get, self.obj, use_account=3) + body = resp.read() + self.assertEquals(resp.status, 200) + self.assertEquals(body, 'test') + + # can put an object + obj_name = str(uuid4()) + resp = retry(put, obj_name, use_account=3) + body = resp.read() + self.assertEquals(resp.status, 201) + + # can delete an object + resp = retry(delete, self.obj, use_account=3) + body = resp.read() + self.assertEquals(resp.status, 204) + + # sanity with account1 + resp = retry(get_listing, use_account=3) + listing = resp.read() + self.assertEquals(resp.status, 200) + self.assert_(obj_name in listing) + self.assert_(self.obj not in listing) + def test_manifest(self): if skip: raise SkipTest @@ -306,7 +577,7 @@ class TestObject(unittest.TestCase): for objnum in xrange(len(segments1)): resp = retry(put, objnum) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) # Upload the manifest def put(url, token, parsed, conn): @@ -318,7 +589,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(put) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) # Get the manifest (should get all the segments as the body) def get(url, token, parsed, conn): @@ -326,9 +597,9 @@ class TestObject(unittest.TestCase): parsed.path, self.container), '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(get) - self.assertEquals(resp.read(), ''.join(segments1)) - self.assertEquals(resp.status, 200) - self.assertEquals(resp.getheader('content-type'), 'text/jibberish') + self.assertEqual(resp.read(), ''.join(segments1)) + self.assertEqual(resp.status, 200) + self.assertEqual(resp.getheader('content-type'), 'text/jibberish') # Get with a range at the start of the second segment def get(url, token, parsed, conn): @@ -337,8 +608,8 @@ class TestObject(unittest.TestCase): 'X-Auth-Token': token, 'Range': 'bytes=3-'}) return check_response(conn) resp = retry(get) - self.assertEquals(resp.read(), ''.join(segments1[1:])) - self.assertEquals(resp.status, 206) + self.assertEqual(resp.read(), ''.join(segments1[1:])) + self.assertEqual(resp.status, 206) # Get with a range in the middle of the second segment def get(url, token, parsed, conn): @@ -347,8 +618,8 @@ class TestObject(unittest.TestCase): 'X-Auth-Token': token, 'Range': 'bytes=5-'}) return check_response(conn) resp = retry(get) - self.assertEquals(resp.read(), ''.join(segments1)[5:]) - self.assertEquals(resp.status, 206) + self.assertEqual(resp.read(), ''.join(segments1)[5:]) + self.assertEqual(resp.status, 206) # Get with a full start and stop range def get(url, token, parsed, conn): @@ -357,8 +628,8 @@ class TestObject(unittest.TestCase): 'X-Auth-Token': token, 'Range': 'bytes=5-10'}) return check_response(conn) resp = retry(get) - self.assertEquals(resp.read(), ''.join(segments1)[5:11]) - self.assertEquals(resp.status, 206) + self.assertEqual(resp.read(), ''.join(segments1)[5:11]) + self.assertEqual(resp.status, 206) # Upload the second set of segments def put(url, token, parsed, conn, objnum): @@ -369,7 +640,7 @@ class TestObject(unittest.TestCase): for objnum in xrange(len(segments2)): resp = retry(put, objnum) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) # Get the manifest (should still be the first segments of course) def get(url, token, parsed, conn): @@ -377,8 +648,8 @@ class TestObject(unittest.TestCase): parsed.path, self.container), '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(get) - self.assertEquals(resp.read(), ''.join(segments1)) - self.assertEquals(resp.status, 200) + self.assertEqual(resp.read(), ''.join(segments1)) + self.assertEqual(resp.status, 200) # Update the manifest def put(url, token, parsed, conn): @@ -390,7 +661,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(put) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) # Get the manifest (should be the second set of segments now) def get(url, token, parsed, conn): @@ -398,8 +669,8 @@ class TestObject(unittest.TestCase): parsed.path, self.container), '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(get) - self.assertEquals(resp.read(), ''.join(segments2)) - self.assertEquals(resp.status, 200) + self.assertEqual(resp.read(), ''.join(segments2)) + self.assertEqual(resp.status, 200) if not skip3: @@ -410,7 +681,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(get, use_account=3) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # Grant access to the third account def post(url, token, parsed, conn): @@ -420,7 +691,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(post) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # The third account should be able to get the manifest now def get(url, token, parsed, conn): @@ -428,8 +699,8 @@ class TestObject(unittest.TestCase): parsed.path, self.container), '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(get, use_account=3) - self.assertEquals(resp.read(), ''.join(segments2)) - self.assertEquals(resp.status, 200) + self.assertEqual(resp.read(), ''.join(segments2)) + self.assertEqual(resp.status, 200) # Create another container for the third set of segments acontainer = uuid4().hex @@ -440,7 +711,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(put) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) # Upload the third set of segments in the other container def put(url, token, parsed, conn, objnum): @@ -451,7 +722,7 @@ class TestObject(unittest.TestCase): for objnum in xrange(len(segments3)): resp = retry(put, objnum) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) # Update the manifest def put(url, token, parsed, conn): @@ -463,7 +734,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(put) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) # Get the manifest to ensure it's the third set of segments def get(url, token, parsed, conn): @@ -471,8 +742,8 @@ class TestObject(unittest.TestCase): parsed.path, self.container), '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(get) - self.assertEquals(resp.read(), ''.join(segments3)) - self.assertEquals(resp.status, 200) + self.assertEqual(resp.read(), ''.join(segments3)) + self.assertEqual(resp.status, 200) if not skip3: @@ -486,7 +757,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(get, use_account=3) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # Grant access to the third account def post(url, token, parsed, conn): @@ -496,7 +767,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(post) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # The third account should be able to get the manifest now def get(url, token, parsed, conn): @@ -504,8 +775,8 @@ class TestObject(unittest.TestCase): parsed.path, self.container), '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(get, use_account=3) - self.assertEquals(resp.read(), ''.join(segments3)) - self.assertEquals(resp.status, 200) + self.assertEqual(resp.read(), ''.join(segments3)) + self.assertEqual(resp.status, 200) # Delete the manifest def delete(url, token, parsed, conn, objnum): @@ -515,7 +786,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(delete, objnum) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # Delete the third set of segments def delete(url, token, parsed, conn, objnum): @@ -526,7 +797,7 @@ class TestObject(unittest.TestCase): for objnum in xrange(len(segments3)): resp = retry(delete, objnum) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # Delete the second set of segments def delete(url, token, parsed, conn, objnum): @@ -537,7 +808,7 @@ class TestObject(unittest.TestCase): for objnum in xrange(len(segments2)): resp = retry(delete, objnum) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # Delete the first set of segments def delete(url, token, parsed, conn, objnum): @@ -548,7 +819,7 @@ class TestObject(unittest.TestCase): for objnum in xrange(len(segments1)): resp = retry(delete, objnum) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # Delete the extra container def delete(url, token, parsed, conn): @@ -557,7 +828,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(delete) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) def test_delete_content_type(self): if skip: @@ -569,7 +840,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(put) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) def delete(url, token, parsed, conn): conn.request('DELETE', '%s/%s/hi' % (parsed.path, self.container), @@ -577,9 +848,9 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(delete) resp.read() - self.assertEquals(resp.status, 204) - self.assertEquals(resp.getheader('Content-Type'), - 'text/html; charset=UTF-8') + self.assertEqual(resp.status, 204) + self.assertEqual(resp.getheader('Content-Type'), + 'text/html; charset=UTF-8') def test_delete_if_delete_at_bad(self): if skip: @@ -592,7 +863,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(put) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) def delete(url, token, parsed, conn): conn.request('DELETE', '%s/%s/hi' % (parsed.path, self.container), @@ -601,7 +872,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(delete) resp.read() - self.assertEquals(resp.status, 400) + self.assertEqual(resp.status, 400) def test_null_name(self): if skip: @@ -614,10 +885,121 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(put) if (web_front_end == 'apache2'): - self.assertEquals(resp.status, 404) + self.assertEqual(resp.status, 404) else: - self.assertEquals(resp.read(), 'Invalid UTF8 or contains NULL') - self.assertEquals(resp.status, 412) + self.assertEqual(resp.read(), 'Invalid UTF8 or contains NULL') + self.assertEqual(resp.status, 412) + + def test_cors(self): + if skip: + raise SkipTest + + def is_strict_mode(url, token, parsed, conn): + conn.request('GET', '/info') + resp = conn.getresponse() + if resp.status // 100 == 2: + info = json.loads(resp.read()) + return info.get('swift', {}).get('strict_cors_mode', False) + return False + + def put_cors_cont(url, token, parsed, conn, orig): + conn.request( + 'PUT', '%s/%s' % (parsed.path, self.container), + '', {'X-Auth-Token': token, + 'X-Container-Meta-Access-Control-Allow-Origin': orig}) + return check_response(conn) + + def put_obj(url, token, parsed, conn, obj): + conn.request( + 'PUT', '%s/%s/%s' % (parsed.path, self.container, obj), + 'test', {'X-Auth-Token': token}) + return check_response(conn) + + def check_cors(url, token, parsed, conn, + method, obj, headers): + if method != 'OPTIONS': + headers['X-Auth-Token'] = token + conn.request( + method, '%s/%s/%s' % (parsed.path, self.container, obj), + '', headers) + return conn.getresponse() + + strict_cors = retry(is_strict_mode) + + resp = retry(put_cors_cont, '*') + resp.read() + self.assertEquals(resp.status // 100, 2) + + resp = retry(put_obj, 'cat') + resp.read() + self.assertEquals(resp.status // 100, 2) + + resp = retry(check_cors, + 'OPTIONS', 'cat', {'Origin': 'http://m.com'}) + self.assertEquals(resp.status, 401) + + resp = retry(check_cors, + 'OPTIONS', 'cat', + {'Origin': 'http://m.com', + 'Access-Control-Request-Method': 'GET'}) + + self.assertEquals(resp.status, 200) + resp.read() + headers = dict((k.lower(), v) for k, v in resp.getheaders()) + self.assertEquals(headers.get('access-control-allow-origin'), + '*') + + resp = retry(check_cors, + 'GET', 'cat', {'Origin': 'http://m.com'}) + self.assertEquals(resp.status, 200) + headers = dict((k.lower(), v) for k, v in resp.getheaders()) + self.assertEquals(headers.get('access-control-allow-origin'), + '*') + + resp = retry(check_cors, + 'GET', 'cat', {'Origin': 'http://m.com', + 'X-Web-Mode': 'True'}) + self.assertEquals(resp.status, 200) + headers = dict((k.lower(), v) for k, v in resp.getheaders()) + self.assertEquals(headers.get('access-control-allow-origin'), + '*') + + #################### + + resp = retry(put_cors_cont, 'http://secret.com') + resp.read() + self.assertEquals(resp.status // 100, 2) + + resp = retry(check_cors, + 'OPTIONS', 'cat', + {'Origin': 'http://m.com', + 'Access-Control-Request-Method': 'GET'}) + resp.read() + self.assertEquals(resp.status, 401) + + if strict_cors: + resp = retry(check_cors, + 'GET', 'cat', {'Origin': 'http://m.com'}) + resp.read() + self.assertEquals(resp.status, 200) + headers = dict((k.lower(), v) for k, v in resp.getheaders()) + self.assertTrue('access-control-allow-origin' not in headers) + + resp = retry(check_cors, + 'GET', 'cat', {'Origin': 'http://secret.com'}) + resp.read() + self.assertEquals(resp.status, 200) + headers = dict((k.lower(), v) for k, v in resp.getheaders()) + self.assertEquals(headers.get('access-control-allow-origin'), + 'http://secret.com') + else: + resp = retry(check_cors, + 'GET', 'cat', {'Origin': 'http://m.com'}) + resp.read() + self.assertEquals(resp.status, 200) + headers = dict((k.lower(), v) for k, v in resp.getheaders()) + self.assertEquals(headers.get('access-control-allow-origin'), + 'http://m.com') if __name__ == '__main__': diff --git a/test/functional/tests.py b/test/functional/tests.py index 0d9a9ef..ad87d7e 100644 --- a/test/functional/tests.py +++ b/test/functional/tests.py @@ -19,14 +19,16 @@ from datetime import datetime import os import hashlib +import hmac import json import locale import random import StringIO import time import threading -import uuid import unittest +import urllib +import uuid from nose import SkipTest from ConfigParser import ConfigParser @@ -36,7 +38,7 @@ from test.functional.swift_test_client import Account, Connection, File, \ from swift.common.constraints import MAX_FILE_SIZE, MAX_META_NAME_LENGTH, \ MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE, \ MAX_OBJECT_NAME_LENGTH, CONTAINER_LISTING_LIMIT, ACCOUNT_LISTING_LIMIT, \ - MAX_ACCOUNT_NAME_LENGTH, MAX_CONTAINER_NAME_LENGTH + MAX_ACCOUNT_NAME_LENGTH, MAX_CONTAINER_NAME_LENGTH, MAX_HEADER_SIZE from gluster.swift.common.constraints import \ set_object_name_component_length, get_object_name_component_length @@ -50,7 +52,8 @@ default_constraints = dict(( ('container_listing_limit', CONTAINER_LISTING_LIMIT), ('account_listing_limit', ACCOUNT_LISTING_LIMIT), ('max_account_name_length', MAX_ACCOUNT_NAME_LENGTH), - ('max_container_name_length', MAX_CONTAINER_NAME_LENGTH))) + ('max_container_name_length', MAX_CONTAINER_NAME_LENGTH), + ('max_header_size', MAX_HEADER_SIZE))) constraints_conf = ConfigParser() conf_exists = constraints_conf.read('/etc/swift/swift.conf') # Constraints are set first from the test config, then from @@ -285,7 +288,7 @@ class TestAccount(Base): if try_count < 5: time.sleep(1) - self.assertEquals(info['container_count'], len(self.env.containers)) + self.assertEqual(info['container_count'], len(self.env.containers)) self.assert_status(204) def testContainerSerializedInfo(self): @@ -309,11 +312,11 @@ class TestAccount(Base): headers = dict(self.env.conn.response.getheaders()) if format_type == 'json': - self.assertEquals(headers['content-type'], - 'application/json; charset=utf-8') + self.assertEqual(headers['content-type'], + 'application/json; charset=utf-8') elif format_type == 'xml': - self.assertEquals(headers['content-type'], - 'application/xml; charset=utf-8') + self.assertEqual(headers['content-type'], + 'application/xml; charset=utf-8') def testListingLimit(self): limit = load_constraint('account_listing_limit') @@ -337,7 +340,7 @@ class TestAccount(Base): if isinstance(b[0], dict): b = [x['name'] for x in b] - self.assertEquals(a, b) + self.assertEqual(a, b) def testInvalidAuthToken(self): hdrs = {'X-Auth-Token': 'bogus_auth_token'} @@ -347,12 +350,12 @@ class TestAccount(Base): def testLastContainerMarker(self): for format_type in [None, 'json', 'xml']: containers = self.env.account.containers({'format': format_type}) - self.assertEquals(len(containers), len(self.env.containers)) + self.assertEqual(len(containers), len(self.env.containers)) self.assert_status(200) containers = self.env.account.containers( parms={'format': format_type, 'marker': containers[-1]}) - self.assertEquals(len(containers), 0) + self.assertEqual(len(containers), 0) if format_type is None: self.assert_status(204) else: @@ -380,8 +383,8 @@ class TestAccount(Base): parms={'format': format_type}) if isinstance(containers[0], dict): containers = [x['name'] for x in containers] - self.assertEquals(sorted(containers, cmp=locale.strcoll), - containers) + self.assertEqual(sorted(containers, cmp=locale.strcoll), + containers) class TestAccountUTF8(Base2, TestAccount): @@ -518,13 +521,13 @@ class TestContainer(Base): for format_type in [None, 'json', 'xml']: for prefix in prefixs: files = cont.files(parms={'prefix': prefix}) - self.assertEquals(files, sorted(prefix_files[prefix])) + self.assertEqual(files, sorted(prefix_files[prefix])) for format_type in [None, 'json', 'xml']: for prefix in prefixs: files = cont.files(parms={'limit': limit_count, 'prefix': prefix}) - self.assertEquals(len(files), limit_count) + self.assertEqual(len(files), limit_count) for file_item in files: self.assert_(file_item.startswith(prefix)) @@ -548,7 +551,7 @@ class TestContainer(Base): container = self.env.account.container(valid_utf8) self.assert_(container.create(cfg={'no_path_quote': True})) self.assert_(container.name in self.env.account.containers()) - self.assertEquals(container.files(), []) + self.assertEqual(container.files(), []) self.assert_(container.delete()) container = self.env.account.container(invalid_utf8) @@ -614,12 +617,12 @@ class TestContainer(Base): def testLastFileMarker(self): for format_type in [None, 'json', 'xml']: files = self.env.container.files({'format': format_type}) - self.assertEquals(len(files), len(self.env.files)) + self.assertEqual(len(files), len(self.env.files)) self.assert_status(200) files = self.env.container.files( parms={'format': format_type, 'marker': files[-1]}) - self.assertEquals(len(files), 0) + self.assertEqual(len(files), 0) if format_type is None: self.assert_status(204) @@ -665,14 +668,14 @@ class TestContainer(Base): files = self.env.container.files(parms={'format': format_type}) if isinstance(files[0], dict): files = [x['name'] for x in files] - self.assertEquals(sorted(files, cmp=locale.strcoll), files) + self.assertEqual(sorted(files, cmp=locale.strcoll), files) def testContainerInfo(self): info = self.env.container.info() self.assert_status(204) - self.assertEquals(info['object_count'], self.env.file_count) - self.assertEquals(info['bytes_used'], - self.env.file_count * self.env.file_size) + self.assertEqual(info['object_count'], self.env.file_count) + self.assertEqual(info['bytes_used'], + self.env.file_count * self.env.file_size) def testContainerInfoOnContainerThatDoesNotExist(self): container = self.env.account.container(Utils.create_name()) @@ -683,7 +686,7 @@ class TestContainer(Base): for format_type in [None, 'json', 'xml']: files = self.env.container.files(parms={'format': format_type, 'limit': 2}) - self.assertEquals(len(files), 2) + self.assertEqual(len(files), 2) def testTooLongName(self): cont = self.env.account.container('x' * 257) @@ -838,7 +841,7 @@ class TestContainerPaths(Base): if isinstance(files[0], dict): files = [str(x['name']) for x in files] - self.assertEquals(files, self.env.stored_files) + self.assertEqual(files, self.env.stored_files) for format_type in ('json', 'xml'): for file_item in self.env.container.files(parms={'format': @@ -846,13 +849,13 @@ class TestContainerPaths(Base): self.assert_(int(file_item['bytes']) >= 0) self.assert_('last_modified' in file_item) if file_item['name'].endswith('/'): - self.assertEquals(file_item['content_type'], - 'application/directory') + self.assertEqual(file_item['content_type'], + 'application/directory') def testStructure(self): def assert_listing(path, file_list): files = self.env.container.files(parms={'path': path}) - self.assertEquals(sorted(file_list, cmp=locale.strcoll), files) + self.assertEqual(sorted(file_list, cmp=locale.strcoll), files) if not normalized_urls: assert_listing('/', ['/dir1/', '/dir2/', '/file1', '/file A']) assert_listing('/dir1', @@ -1176,7 +1179,7 @@ class TestFile(Base): for i in container.files(parms={'format': 'json'}): file_types_read[i['name'].split('.')[1]] = i['content_type'] - self.assertEquals(file_types, file_types_read) + self.assertEqual(file_types, file_types_read) def testRangedGets(self): file_length = 10000 @@ -1201,7 +1204,7 @@ class TestFile(Base): self.assertRaises(ResponseError, file_item.read, hdrs=hdrs) self.assert_status(416) else: - self.assertEquals(file_item.read(hdrs=hdrs), data[-i:]) + self.assertEqual(file_item.read(hdrs=hdrs), data[-i:]) range_string = 'bytes=%d-' % (i) hdrs = {'Range': range_string} @@ -1350,9 +1353,9 @@ class TestFile(Base): info = file_item.info() self.assert_status(200) - self.assertEquals(info['content_length'], self.env.file_size) - self.assertEquals(info['etag'], md5) - self.assertEquals(info['content_type'], content_type) + self.assertEqual(info['content_length'], self.env.file_size) + self.assertEqual(info['etag'], md5) + self.assertEqual(info['content_type'], content_type) self.assert_('last_modified' in info) def testDeleteOfFileThatDoesNotExist(self): @@ -1395,7 +1398,7 @@ class TestFile(Base): file_item = self.env.container.file(file_item.name) self.assert_(file_item.initialize()) self.assert_status(200) - self.assertEquals(file_item.metadata, metadata) + self.assertEqual(file_item.metadata, metadata) def testGetContentType(self): file_name = Utils.create_name() @@ -1408,7 +1411,7 @@ class TestFile(Base): file_item = self.env.container.file(file_name) file_item.read() - self.assertEquals(content_type, file_item.content_type) + self.assertEqual(content_type, file_item.content_type) def testGetOnFileThatDoesNotExist(self): # in container that exists @@ -1449,7 +1452,7 @@ class TestFile(Base): file_item = self.env.container.file(file_item.name) self.assert_(file_item.initialize()) self.assert_status(200) - self.assertEquals(file_item.metadata, metadata) + self.assertEqual(file_item.metadata, metadata) def testSerialization(self): container = self.env.account.container(Utils.create_name()) @@ -1478,9 +1481,9 @@ class TestFile(Base): if f['name'] != file_item['name']: continue - self.assertEquals(file_item['content_type'], - f['content_type']) - self.assertEquals(int(file_item['bytes']), f['bytes']) + self.assertEqual(file_item['content_type'], + f['content_type']) + self.assertEqual(int(file_item['bytes']), f['bytes']) d = datetime.strptime( file_item['last_modified'].split('.')[0], @@ -1488,7 +1491,7 @@ class TestFile(Base): lm = time.mktime(d.timetuple()) if 'last_modified' in f: - self.assertEquals(f['last_modified'], lm) + self.assertEqual(f['last_modified'], lm) else: f['last_modified'] = lm @@ -1500,11 +1503,11 @@ class TestFile(Base): headers = dict(self.env.conn.response.getheaders()) if format_type == 'json': - self.assertEquals(headers['content-type'], - 'application/json; charset=utf-8') + self.assertEqual(headers['content-type'], + 'application/json; charset=utf-8') elif format_type == 'xml': - self.assertEquals(headers['content-type'], - 'application/xml; charset=utf-8') + self.assertEqual(headers['content-type'], + 'application/xml; charset=utf-8') lm_diff = max([f['last_modified'] for f in files]) -\ min([f['last_modified'] for f in files]) @@ -1547,7 +1550,7 @@ class TestFile(Base): self.assert_('etag' in headers.keys()) header_etag = headers['etag'].strip('"') - self.assertEquals(etag, header_etag) + self.assertEqual(etag, header_etag) def testChunkedPut(self): if (web_front_end == 'apache2'): @@ -1565,7 +1568,7 @@ class TestFile(Base): self.assert_(data == file_item.read()) info = file_item.info() - self.assertEquals(etag, info['etag']) + self.assertEqual(etag, info['etag']) class TestFileUTF8(Base2, TestFile): @@ -1677,12 +1680,30 @@ class TestDlo(Base): file_contents, "aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffff") + def test_copy_manifest(self): + # Copying the manifest should result in another manifest + try: + man1_item = self.env.container.file('man1') + man1_item.copy(self.env.container.name, "copied-man1", + parms={'multipart-manifest': 'get'}) + + copied = self.env.container.file("copied-man1") + copied_contents = copied.read(parms={'multipart-manifest': 'get'}) + self.assertEqual(copied_contents, "man1-contents") + + copied_contents = copied.read() + self.assertEqual( + copied_contents, + "aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeee") + finally: + # try not to leave this around for other tests to stumble over + self.env.container.file("copied-man1").delete() class TestDloUTF8(Base2, TestDlo): set_up = False -class TestFileComparisonEnv: +class TestFileComparisonEnv(object): @classmethod def setUp(cls): cls.conn = Connection(config) @@ -1806,19 +1827,8 @@ class TestSloEnv(object): cls.conn.authenticate() if cls.slo_enabled is None: - status = cls.conn.make_request('GET', '/info', - cfg={'verbatim_path': True}) - if not (200 <= status <= 299): - # Can't tell if SLO is enabled or not since we're running - # against an old cluster, so let's skip the tests instead of - # possibly having spurious failures. - cls.slo_enabled = False - else: - # Don't bother looking for ValueError here. If something is - # responding to a GET /info request with invalid JSON, then - # the cluster is broken and a test failure will let us know. - cluster_info = json.loads(cls.conn.response.read()) - cls.slo_enabled = 'slo' in cluster_info + cluster_info = cls.conn.cluster_info() + cls.slo_enabled = 'slo' in cluster_info if not cls.slo_enabled: return @@ -2034,5 +2044,347 @@ class TestSloUTF8(Base2, TestSlo): set_up = False +class TestObjectVersioningEnv(object): + versioning_enabled = None # tri-state: None initially, then True/False + + @classmethod + def setUp(cls): + cls.conn = Connection(config) + cls.conn.authenticate() + + cls.account = Account(cls.conn, config.get('account', + config['username'])) + + # avoid getting a prefix that stops halfway through an encoded + # character + prefix = Utils.create_name().decode("utf-8")[:10].encode("utf-8") + + cls.versions_container = cls.account.container(prefix + "-versions") + if not cls.versions_container.create(): + raise ResponseError(cls.conn.response) + + cls.container = cls.account.container(prefix + "-objs") + if not cls.container.create( + hdrs={'X-Versions-Location': cls.versions_container.name}): + raise ResponseError(cls.conn.response) + + container_info = cls.container.info() + # if versioning is off, then X-Versions-Location won't persist + cls.versioning_enabled = 'versions' in container_info + + +class TestObjectVersioning(Base): + env = TestObjectVersioningEnv + set_up = False + + def setUp(self): + super(TestObjectVersioning, self).setUp() + if self.env.versioning_enabled is False: + raise SkipTest("Object versioning not enabled") + elif self.env.versioning_enabled is not True: + # just some sanity checking + raise Exception( + "Expected versioning_enabled to be True/False, got %r" % + (self.env.versioning_enabled,)) + + def test_overwriting(self): + container = self.env.container + versions_container = self.env.versions_container + obj_name = Utils.create_name() + + versioned_obj = container.file(obj_name) + versioned_obj.write("aaaaa") + + self.assertEqual(0, versions_container.info()['object_count']) + + versioned_obj.write("bbbbb") + + # the old version got saved off + self.assertEqual(1, versions_container.info()['object_count']) + versioned_obj_name = versions_container.files()[0] + self.assertEqual( + "aaaaa", versions_container.file(versioned_obj_name).read()) + + # if we overwrite it again, there are two versions + versioned_obj.write("ccccc") + self.assertEqual(2, versions_container.info()['object_count']) + + # as we delete things, the old contents return + self.assertEqual("ccccc", versioned_obj.read()) + versioned_obj.delete() + self.assertEqual("bbbbb", versioned_obj.read()) + versioned_obj.delete() + self.assertEqual("aaaaa", versioned_obj.read()) + versioned_obj.delete() + self.assertRaises(ResponseError, versioned_obj.read) + + +class TestObjectVersioningUTF8(Base2, TestObjectVersioning): + set_up = False + + +class TestTempurlEnv(object): + tempurl_enabled = None # tri-state: None initially, then True/False + + @classmethod + def setUp(cls): + cls.conn = Connection(config) + cls.conn.authenticate() + + if cls.tempurl_enabled is None: + cluster_info = cls.conn.cluster_info() + cls.tempurl_enabled = 'tempurl' in cluster_info + if not cls.tempurl_enabled: + return + cls.tempurl_methods = cluster_info['tempurl']['methods'] + + cls.tempurl_key = Utils.create_name() + cls.tempurl_key2 = Utils.create_name() + + cls.account = Account( + cls.conn, config.get('account', config['username'])) + cls.account.delete_containers() + cls.account.update_metadata({ + 'temp-url-key': cls.tempurl_key, + 'temp-url-key-2': cls.tempurl_key2 + }) + + cls.container = cls.account.container(Utils.create_name()) + if not cls.container.create(): + raise ResponseError(cls.conn.response) + + cls.obj = cls.container.file(Utils.create_name()) + cls.obj.write("obj contents") + cls.other_obj = cls.container.file(Utils.create_name()) + cls.other_obj.write("other obj contents") + + +class TestTempurl(Base): + env = TestTempurlEnv + set_up = False + + def setUp(self): + super(TestTempurl, self).setUp() + if self.env.tempurl_enabled is False: + raise SkipTest("TempURL not enabled") + elif self.env.tempurl_enabled is not True: + # just some sanity checking + raise Exception( + "Expected tempurl_enabled to be True/False, got %r" % + (self.env.tempurl_enabled,)) + + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'GET', expires, self.env.conn.make_path(self.env.obj.path), + self.env.tempurl_key) + self.obj_tempurl_parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + def tempurl_sig(self, method, expires, path, key): + return hmac.new( + key, + '%s\n%s\n%s' % (method, expires, urllib.unquote(path)), + hashlib.sha1).hexdigest() + + def test_GET(self): + contents = self.env.obj.read( + parms=self.obj_tempurl_parms, + cfg={'no_auth_token': True}) + self.assertEqual(contents, "obj contents") + + # GET tempurls also allow HEAD requests + self.assert_(self.env.obj.info(parms=self.obj_tempurl_parms, + cfg={'no_auth_token': True})) + + def test_GET_with_key_2(self): + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'GET', expires, self.env.conn.make_path(self.env.obj.path), + self.env.tempurl_key2) + parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + contents = self.env.obj.read(parms=parms, cfg={'no_auth_token': True}) + self.assertEqual(contents, "obj contents") + + def test_PUT(self): + new_obj = self.env.container.file(Utils.create_name()) + + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'PUT', expires, self.env.conn.make_path(new_obj.path), + self.env.tempurl_key) + put_parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + new_obj.write('new obj contents', + parms=put_parms, cfg={'no_auth_token': True}) + self.assertEqual(new_obj.read(), "new obj contents") + + # PUT tempurls also allow HEAD requests + self.assert_(new_obj.info(parms=put_parms, + cfg={'no_auth_token': True})) + + def test_HEAD(self): + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'HEAD', expires, self.env.conn.make_path(self.env.obj.path), + self.env.tempurl_key) + head_parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + self.assert_(self.env.obj.info(parms=head_parms, + cfg={'no_auth_token': True})) + # HEAD tempurls don't allow PUT or GET requests, despite the fact that + # PUT and GET tempurls both allow HEAD requests + self.assertRaises(ResponseError, self.env.other_obj.read, + cfg={'no_auth_token': True}, + parms=self.obj_tempurl_parms) + self.assert_status([401]) + + self.assertRaises(ResponseError, self.env.other_obj.write, + 'new contents', + cfg={'no_auth_token': True}, + parms=self.obj_tempurl_parms) + self.assert_status([401]) + + def test_different_object(self): + contents = self.env.obj.read( + parms=self.obj_tempurl_parms, + cfg={'no_auth_token': True}) + self.assertEqual(contents, "obj contents") + + self.assertRaises(ResponseError, self.env.other_obj.read, + cfg={'no_auth_token': True}, + parms=self.obj_tempurl_parms) + self.assert_status([401]) + + def test_changing_sig(self): + contents = self.env.obj.read( + parms=self.obj_tempurl_parms, + cfg={'no_auth_token': True}) + self.assertEqual(contents, "obj contents") + + parms = self.obj_tempurl_parms.copy() + if parms['temp_url_sig'][0] == 'a': + parms['temp_url_sig'] = 'b' + parms['temp_url_sig'][1:] + else: + parms['temp_url_sig'] = 'a' + parms['temp_url_sig'][1:] + + self.assertRaises(ResponseError, self.env.obj.read, + cfg={'no_auth_token': True}, + parms=parms) + self.assert_status([401]) + + def test_changing_expires(self): + contents = self.env.obj.read( + parms=self.obj_tempurl_parms, + cfg={'no_auth_token': True}) + self.assertEqual(contents, "obj contents") + + parms = self.obj_tempurl_parms.copy() + if parms['temp_url_expires'][-1] == '0': + parms['temp_url_expires'] = parms['temp_url_expires'][:-1] + '1' + else: + parms['temp_url_expires'] = parms['temp_url_expires'][:-1] + '0' + + self.assertRaises(ResponseError, self.env.obj.read, + cfg={'no_auth_token': True}, + parms=parms) + self.assert_status([401]) + + +class TestTempurlUTF8(Base2, TestTempurl): + set_up = False + + +class TestSloTempurlEnv(object): + enabled = None # tri-state: None initially, then True/False + + @classmethod + def setUp(cls): + cls.conn = Connection(config) + cls.conn.authenticate() + + if cls.enabled is None: + cluster_info = cls.conn.cluster_info() + cls.enabled = 'tempurl' in cluster_info and 'slo' in cluster_info + + cls.tempurl_key = Utils.create_name() + + cls.account = Account( + cls.conn, config.get('account', config['username'])) + cls.account.delete_containers() + cls.account.update_metadata({'temp-url-key': cls.tempurl_key}) + + cls.manifest_container = cls.account.container(Utils.create_name()) + cls.segments_container = cls.account.container(Utils.create_name()) + if not cls.manifest_container.create(): + raise ResponseError(cls.conn.response) + if not cls.segments_container.create(): + raise ResponseError(cls.conn.response) + + seg1 = cls.segments_container.file(Utils.create_name()) + seg1.write('1' * 1024 * 1024) + + seg2 = cls.segments_container.file(Utils.create_name()) + seg2.write('2' * 1024 * 1024) + + cls.manifest_data = [{'size_bytes': 1024 * 1024, + 'etag': seg1.md5, + 'path': '/%s/%s' % (cls.segments_container.name, + seg1.name)}, + {'size_bytes': 1024 * 1024, + 'etag': seg2.md5, + 'path': '/%s/%s' % (cls.segments_container.name, + seg2.name)}] + + cls.manifest = cls.manifest_container.file(Utils.create_name()) + cls.manifest.write( + json.dumps(cls.manifest_data), + parms={'multipart-manifest': 'put'}) + + +class TestSloTempurl(Base): + env = TestSloTempurlEnv + set_up = False + + def setUp(self): + super(TestSloTempurl, self).setUp() + if self.env.enabled is False: + raise SkipTest("TempURL and SLO not both enabled") + elif self.env.enabled is not True: + # just some sanity checking + raise Exception( + "Expected enabled to be True/False, got %r" % + (self.env.enabled,)) + + def tempurl_sig(self, method, expires, path, key): + return hmac.new( + key, + '%s\n%s\n%s' % (method, expires, urllib.unquote(path)), + hashlib.sha1).hexdigest() + + def test_GET(self): + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'GET', expires, self.env.conn.make_path(self.env.manifest.path), + self.env.tempurl_key) + parms = {'temp_url_sig': sig, 'temp_url_expires': str(expires)} + + contents = self.env.manifest.read( + parms=parms, + cfg={'no_auth_token': True}) + self.assertEqual(len(contents), 2 * 1024 * 1024) + + # GET tempurls also allow HEAD requests + self.assert_(self.env.manifest.info( + parms=parms, cfg={'no_auth_token': True})) + + +class TestSloTempurlUTF8(Base2, TestSloTempurl): + set_up = False + + if __name__ == '__main__': unittest.main() diff --git a/test/functional_auth/common_conf/account-server.conf b/test/functional_auth/common_conf/account-server.conf new file mode 100644 index 0000000..549a094 --- /dev/null +++ b/test/functional_auth/common_conf/account-server.conf @@ -0,0 +1,36 @@ +[DEFAULT] +devices = /mnt/gluster-object +# +# Once you are confident that your startup processes will always have your +# gluster volumes properly mounted *before* the account-server workers start, +# you can *consider* setting this value to "false" to reduce the per-request +# overhead it can incur. +# +# *** Keep false for Functional Tests *** +mount_check = false +bind_port = 6012 +# +# Override swift's default behaviour for fallocate. +disable_fallocate = true +# +# One or two workers should be sufficient for almost any installation of +# Gluster. +workers = 1 + +[pipeline:main] +pipeline = account-server + +[app:account-server] +use = egg:gluster_swift#account +user = root +log_facility = LOG_LOCAL2 +log_level = WARN +# +# After ensuring things are running in a stable manner, you can turn off +# normal request logging for the account server to unclutter the log +# files. Warnings and errors will still be logged. +log_requests = off + +# The following parameter is used by object-expirer and needs to be same +# across all conf files! +auto_create_account_prefix = gs diff --git a/test/functional_auth/common_conf/container-server.conf b/test/functional_auth/common_conf/container-server.conf new file mode 100644 index 0000000..487ccb8 --- /dev/null +++ b/test/functional_auth/common_conf/container-server.conf @@ -0,0 +1,39 @@ +[DEFAULT] +devices = /mnt/gluster-object +# +# Once you are confident that your startup processes will always have your +# gluster volumes properly mounted *before* the container-server workers +# start, you can *consider* setting this value to "false" to reduce the +# per-request overhead it can incur. +# +# *** Keep false for Functional Tests *** +mount_check = false +bind_port = 6011 +# +# Override swift's default behaviour for fallocate. +disable_fallocate = true +# +# One or two workers should be sufficient for almost any installation of +# Gluster. +workers = 1 + +[pipeline:main] +pipeline = container-server + +[app:container-server] +use = egg:gluster_swift#container +user = root +log_facility = LOG_LOCAL2 +log_level = WARN +# +# After ensuring things are running in a stable manner, you can turn off +# normal request logging for the container server to unclutter the log +# files. Warnings and errors will still be logged. +log_requests = off + +#enable object versioning for functional test +allow_versions = on + +# The following parameter is used by object-expirer and needs to be same +# across all conf files! +auto_create_account_prefix = gs diff --git a/test/functional_auth/common_conf/fs.conf b/test/functional_auth/common_conf/fs.conf new file mode 100644 index 0000000..b06a854 --- /dev/null +++ b/test/functional_auth/common_conf/fs.conf @@ -0,0 +1,19 @@ +[DEFAULT] +# +# IP address of a node in the GlusterFS server cluster hosting the +# volumes to be served via Swift API. +mount_ip = localhost + +# Performance optimization parameter. When turned off, the filesystem will +# see a reduced number of stat calls, resulting in substantially faster +# response time for GET and HEAD container requests on containers with large +# numbers of objects, at the expense of an accurate count of combined bytes +# used by all objects in the container. For most installations "off" works +# fine. +# +# *** Keep on for Functional Tests *** +accurate_size_in_listing = on + +# *** Keep on for Functional Tests *** +container_update_object_count = on +account_update_container_count = on diff --git a/test/functional_auth/common_conf/object-expirer.conf b/test/functional_auth/common_conf/object-expirer.conf new file mode 100644 index 0000000..4449ee2 --- /dev/null +++ b/test/functional_auth/common_conf/object-expirer.conf @@ -0,0 +1,27 @@ +#TODO: Add documentation to explain various options +#For now, refer: https://github.com/openstack/swift/blob/master/etc/object-expirer.conf-sample + +[DEFAULT] + +[object-expirer] +user = root +log_facility = LOG_LOCAL2 +log_level = DEBUG +# The following parameters are used by object-expirer and needs to be same +# across all conf files! +auto_create_account_prefix = gs +expiring_objects_account_name = expiring + +interval = 30 + +[pipeline:main] +pipeline = catch_errors cache proxy-server + +[app:proxy-server] +use = egg:gluster_swift#proxy + +[filter:cache] +use = egg:swift#memcache + +[filter:catch_errors] +use = egg:swift#catch_errors diff --git a/test/functional_auth/common_conf/object-server.conf b/test/functional_auth/common_conf/object-server.conf new file mode 100644 index 0000000..2599c87 --- /dev/null +++ b/test/functional_auth/common_conf/object-server.conf @@ -0,0 +1,53 @@ +[DEFAULT] +devices = /mnt/gluster-object +# +# Once you are confident that your startup processes will always have your +# gluster volumes properly mounted *before* the object-server workers start, +# you can *consider* setting this value to "false" to reduce the per-request +# overhead it can incur. +# +# *** Keep false for Functional Tests *** +mount_check = false +bind_port = 6010 +# +# Maximum number of clients one worker can process simultaneously (it will +# actually accept N + 1). Setting this to one (1) will only handle one request +# at a time, without accepting another request concurrently. By increasing the +# number of workers to a much higher value, one can prevent slow file system +# operations for one request from starving other requests. +max_clients = 1024 +# +# If not doing the above, setting this value initially to match the number of +# CPUs is a good starting point for determining the right value. +workers = 1 +# Override swift's default behaviour for fallocate. +disable_fallocate = true + +[pipeline:main] +pipeline = object-server + +[app:object-server] +use = egg:gluster_swift#object +user = root +log_facility = LOG_LOCAL2 +log_level = WARN +# +# For performance, after ensuring things are running in a stable manner, you +# can turn off normal request logging for the object server to reduce the +# per-request overhead and unclutter the log files. Warnings and errors will +# still be logged. +log_requests = off +# +# Adjust this value to match the stripe width of the underlying storage array +# (not the stripe element size). This will provide a reasonable starting point +# for tuning this value. +disk_chunk_size = 65536 +# +# Adjust this value match whatever is set for the disk_chunk_size initially. +# This will provide a reasonable starting point for tuning this value. +network_chunk_size = 65556 + +# The following parameter is used by object-expirer and needs to be same +# across all conf files! +auto_create_account_prefix = gs +expiring_objects_account_name = expiring diff --git a/test/functional_auth/common_conf/swift.conf b/test/functional_auth/common_conf/swift.conf new file mode 100644 index 0000000..f64ba5a --- /dev/null +++ b/test/functional_auth/common_conf/swift.conf @@ -0,0 +1,85 @@ +[DEFAULT] + + +[swift-hash] +# random unique string that can never change (DO NOT LOSE) +swift_hash_path_suffix = gluster + + +# The swift-constraints section sets the basic constraints on data +# saved in the swift cluster. + +[swift-constraints] + +# max_file_size is the largest "normal" object that can be saved in +# the cluster. This is also the limit on the size of each segment of +# a "large" object when using the large object manifest support. +# This value is set in bytes. Setting it to lower than 1MiB will cause +# some tests to fail. +# Default is 1 TiB = 2**30*1024 +max_file_size = 1099511627776 + + +# max_meta_name_length is the max number of bytes in the utf8 encoding +# of the name portion of a metadata header. + +#max_meta_name_length = 128 + + +# max_meta_value_length is the max number of bytes in the utf8 encoding +# of a metadata value + +#max_meta_value_length = 256 + + +# max_meta_count is the max number of metadata keys that can be stored +# on a single account, container, or object + +#max_meta_count = 90 + + +# max_meta_overall_size is the max number of bytes in the utf8 encoding +# of the metadata (keys + values) + +#max_meta_overall_size = 4096 + + +# max_object_name_length is the max number of bytes in the utf8 encoding of an +# object name: Gluster FS can handle much longer file names, but the length +# between the slashes of the URL is handled below. Remember that most web +# clients can't handle anything greater than 2048, and those that do are +# rather clumsy. + +max_object_name_length = 2048 + +# max_object_name_component_length (GlusterFS) is the max number of bytes in +# the utf8 encoding of an object name component (the part between the +# slashes); this is a limit imposed by the underlying file system (for XFS it +# is 255 bytes). + +max_object_name_component_length = 255 + +# container_listing_limit is the default (and max) number of items +# returned for a container listing request + +#container_listing_limit = 10000 + + +# account_listing_limit is the default (and max) number of items returned +# for an account listing request + +#account_listing_limit = 10000 + + +# max_account_name_length is the max number of bytes in the utf8 encoding of +# an account name: Gluster FS Filename limit (XFS limit?), must be the same +# size as max_object_name_component_length above. + +max_account_name_length = 255 + + +# max_container_name_length is the max number of bytes in the utf8 encoding +# of a container name: Gluster FS Filename limit (XFS limit?), must be the same +# size as max_object_name_component_length above. + +max_container_name_length = 255 diff --git a/test/functional_auth/common_conf/test.conf b/test/functional_auth/common_conf/test.conf new file mode 100644 index 0000000..15c9aea --- /dev/null +++ b/test/functional_auth/common_conf/test.conf @@ -0,0 +1,58 @@ +[func_test] +# sample config +auth_host = 127.0.0.1 +auth_port = 8080 +auth_ssl = no +auth_prefix = /auth/ +## sample config for Swift with Keystone +#auth_version = 2 +#auth_host = localhost +#auth_port = 5000 +#auth_ssl = no +#auth_prefix = /v2.0/ + +# GSWauth internal admin user configuration information +admin_key = gswauthkey +admin_user = .super_admin + +# Gluster setup information +devices = /mnt/gluster-object +gsmetadata_volume = gsmetadata + +# Primary functional test account (needs admin access to the account) +account = test +username = tester +password = testing + +# User on a second account (needs admin access to the account) +account2 = test2 +username2 = tester2 +password2 = testing2 + +# User on same account as first, but without admin access +username3 = tester3 +password3 = testing3 + +# Default constraints if not defined here, the test runner will try +# to set them from /etc/swift/swift.conf. If that file isn't found, +# the test runner will skip tests that depend on these values. +# Note that the cluster must have "sane" values for the test suite to pass. +#max_file_size = 5368709122 +#max_meta_name_length = 128 +#max_meta_value_length = 256 +#max_meta_count = 90 +#max_meta_overall_size = 4096 +#max_object_name_length = 1024 +#container_listing_limit = 10000 +#account_listing_limit = 10000 +#max_account_name_length = 256 +#max_container_name_length = 256 +normalized_urls = True + +collate = C + +[unit_test] +fake_syslog = False + +[probe_test] +# check_server_timeout = 30 diff --git a/test/functional_auth/gswauth/conf/account-server.conf b/test/functional_auth/gswauth/conf/account-server.conf deleted file mode 100644 index 4996367..0000000 --- a/test/functional_auth/gswauth/conf/account-server.conf +++ /dev/null @@ -1,32 +0,0 @@ -[DEFAULT] -devices = /mnt/gluster-object -# -# Once you are confident that your startup processes will always have your -# gluster volumes properly mounted *before* the account-server workers start, -# you can *consider* setting this value to "false" to reduce the per-request -# overhead it can incur. -# -# *** Keep false for Functional Tests *** -mount_check = false -bind_port = 6012 -# -# Override swift's default behaviour for fallocate. -disable_fallocate = true -# -# One or two workers should be sufficient for almost any installation of -# Gluster. -workers = 1 - -[pipeline:main] -pipeline = account-server - -[app:account-server] -use = egg:gluster_swift#account -user = root -log_facility = LOG_LOCAL2 -log_level = WARN -# -# After ensuring things are running in a stable manner, you can turn off -# normal request logging for the account server to unclutter the log -# files. Warnings and errors will still be logged. -log_requests = off diff --git a/test/functional_auth/gswauth/conf/container-server.conf b/test/functional_auth/gswauth/conf/container-server.conf deleted file mode 100644 index 122d97e..0000000 --- a/test/functional_auth/gswauth/conf/container-server.conf +++ /dev/null @@ -1,35 +0,0 @@ -[DEFAULT] -devices = /mnt/gluster-object -# -# Once you are confident that your startup processes will always have your -# gluster volumes properly mounted *before* the container-server workers -# start, you can *consider* setting this value to "false" to reduce the -# per-request overhead it can incur. -# -# *** Keep false for Functional Tests *** -mount_check = false -bind_port = 6011 -# -# Override swift's default behaviour for fallocate. -disable_fallocate = true -# -# One or two workers should be sufficient for almost any installation of -# Gluster. -workers = 1 - -[pipeline:main] -pipeline = container-server - -[app:container-server] -use = egg:gluster_swift#container -user = root -log_facility = LOG_LOCAL2 -log_level = WARN -# -# After ensuring things are running in a stable manner, you can turn off -# normal request logging for the container server to unclutter the log -# files. Warnings and errors will still be logged. -log_requests = off - -#enable object versioning for functional test -allow_versions = on diff --git a/test/functional_auth/gswauth/conf/fs.conf b/test/functional_auth/gswauth/conf/fs.conf deleted file mode 100644 index b06a854..0000000 --- a/test/functional_auth/gswauth/conf/fs.conf +++ /dev/null @@ -1,19 +0,0 @@ -[DEFAULT] -# -# IP address of a node in the GlusterFS server cluster hosting the -# volumes to be served via Swift API. -mount_ip = localhost - -# Performance optimization parameter. When turned off, the filesystem will -# see a reduced number of stat calls, resulting in substantially faster -# response time for GET and HEAD container requests on containers with large -# numbers of objects, at the expense of an accurate count of combined bytes -# used by all objects in the container. For most installations "off" works -# fine. -# -# *** Keep on for Functional Tests *** -accurate_size_in_listing = on - -# *** Keep on for Functional Tests *** -container_update_object_count = on -account_update_container_count = on diff --git a/test/functional_auth/gswauth/conf/object-expirer.conf b/test/functional_auth/gswauth/conf/object-expirer.conf deleted file mode 100644 index b75963c..0000000 --- a/test/functional_auth/gswauth/conf/object-expirer.conf +++ /dev/null @@ -1,17 +0,0 @@ -[DEFAULT] - -[object-expirer] -# auto_create_account_prefix = . - -[pipeline:main] -pipeline = catch_errors cache proxy-server - -[app:proxy-server] -use = egg:swift#proxy - -[filter:cache] -use = egg:swift#memcache -memcache_servers = 127.0.0.1:11211 - -[filter:catch_errors] -use = egg:swift#catch_errors diff --git a/test/functional_auth/gswauth/conf/object-server.conf b/test/functional_auth/gswauth/conf/object-server.conf deleted file mode 100644 index 3cb9ead..0000000 --- a/test/functional_auth/gswauth/conf/object-server.conf +++ /dev/null @@ -1,48 +0,0 @@ -[DEFAULT] -devices = /mnt/gluster-object -# -# Once you are confident that your startup processes will always have your -# gluster volumes properly mounted *before* the object-server workers start, -# you can *consider* setting this value to "false" to reduce the per-request -# overhead it can incur. -# -# *** Keep false for Functional Tests *** -mount_check = false -bind_port = 6010 -# -# Maximum number of clients one worker can process simultaneously (it will -# actually accept N + 1). Setting this to one (1) will only handle one request -# at a time, without accepting another request concurrently. By increasing the -# number of workers to a much higher value, one can prevent slow file system -# operations for one request from starving other requests. -max_clients = 1024 -# -# If not doing the above, setting this value initially to match the number of -# CPUs is a good starting point for determining the right value. -workers = 1 -# Override swift's default behaviour for fallocate. -disable_fallocate = true - -[pipeline:main] -pipeline = object-server - -[app:object-server] -use = egg:gluster_swift#object -user = root -log_facility = LOG_LOCAL2 -log_level = WARN -# -# For performance, after ensuring things are running in a stable manner, you -# can turn off normal request logging for the object server to reduce the -# per-request overhead and unclutter the log files. Warnings and errors will -# still be logged. -log_requests = off -# -# Adjust this value to match the stripe width of the underlying storage array -# (not the stripe element size). This will provide a reasonable starting point -# for tuning this value. -disk_chunk_size = 65536 -# -# Adjust this value match whatever is set for the disk_chunk_size initially. -# This will provide a reasonable starting point for tuning this value. -network_chunk_size = 65556 diff --git a/test/functional_auth/gswauth/conf/proxy-server.conf b/test/functional_auth/gswauth/conf/proxy-server.conf index 165cb0c..0a3c1f6 100644 --- a/test/functional_auth/gswauth/conf/proxy-server.conf +++ b/test/functional_auth/gswauth/conf/proxy-server.conf @@ -5,7 +5,7 @@ user = root workers = 1 [pipeline:main] -pipeline = catch_errors healthcheck proxy-logging cache gswauth proxy-logging proxy-server +pipeline = catch_errors healthcheck proxy-logging cache tempurl gswauth proxy-logging proxy-server [app:proxy-server] use = egg:gluster_swift#proxy @@ -48,6 +48,10 @@ object_chunk_size = 65536 # amount of memory available on the system can accommodate increased values # for object_chunk_size. put_queue_depth = 10 +# The following parameter is used by object-expirer and needs to be same +# across all conf files! +auto_create_account_prefix = gs +expiring_objects_account_name = expiring [filter:catch_errors] use = egg:swift#catch_errors @@ -58,21 +62,17 @@ use = egg:swift#proxy_logging [filter:healthcheck] use = egg:swift#healthcheck -[filter:tempauth] -use = egg:swift#tempauth -user_admin_admin = admin .admin .reseller_admin -user_test_tester = testing .admin -user_test2_tester2 = testing2 .admin -user_test_tester3 = testing3 +[filter:cache] +use = egg:swift#memcache +# Update this line to contain a comma separated list of memcache servers +# shared by all nodes running the proxy-server service. +memcache_servers = localhost:11211 + +[filter:tempurl] +use = egg:swift#tempurl [filter:gswauth] use = egg:gluster_swift#gswauth set log_name = gswauth super_admin_key = gswauthkey metadata_volume = gsmetadata - -[filter:cache] -use = egg:swift#memcache -# Update this line to contain a comma separated list of memcache servers -# shared by all nodes running the proxy-server service. -memcache_servers = localhost:11211 diff --git a/test/functional_auth/gswauth/conf/swift.conf b/test/functional_auth/gswauth/conf/swift.conf deleted file mode 100644 index f64ba5a..0000000 --- a/test/functional_auth/gswauth/conf/swift.conf +++ /dev/null @@ -1,85 +0,0 @@ -[DEFAULT] - - -[swift-hash] -# random unique string that can never change (DO NOT LOSE) -swift_hash_path_suffix = gluster - - -# The swift-constraints section sets the basic constraints on data -# saved in the swift cluster. - -[swift-constraints] - -# max_file_size is the largest "normal" object that can be saved in -# the cluster. This is also the limit on the size of each segment of -# a "large" object when using the large object manifest support. -# This value is set in bytes. Setting it to lower than 1MiB will cause -# some tests to fail. -# Default is 1 TiB = 2**30*1024 -max_file_size = 1099511627776 - - -# max_meta_name_length is the max number of bytes in the utf8 encoding -# of the name portion of a metadata header. - -#max_meta_name_length = 128 - - -# max_meta_value_length is the max number of bytes in the utf8 encoding -# of a metadata value - -#max_meta_value_length = 256 - - -# max_meta_count is the max number of metadata keys that can be stored -# on a single account, container, or object - -#max_meta_count = 90 - - -# max_meta_overall_size is the max number of bytes in the utf8 encoding -# of the metadata (keys + values) - -#max_meta_overall_size = 4096 - - -# max_object_name_length is the max number of bytes in the utf8 encoding of an -# object name: Gluster FS can handle much longer file names, but the length -# between the slashes of the URL is handled below. Remember that most web -# clients can't handle anything greater than 2048, and those that do are -# rather clumsy. - -max_object_name_length = 2048 - -# max_object_name_component_length (GlusterFS) is the max number of bytes in -# the utf8 encoding of an object name component (the part between the -# slashes); this is a limit imposed by the underlying file system (for XFS it -# is 255 bytes). - -max_object_name_component_length = 255 - -# container_listing_limit is the default (and max) number of items -# returned for a container listing request - -#container_listing_limit = 10000 - - -# account_listing_limit is the default (and max) number of items returned -# for an account listing request - -#account_listing_limit = 10000 - - -# max_account_name_length is the max number of bytes in the utf8 encoding of -# an account name: Gluster FS Filename limit (XFS limit?), must be the same -# size as max_object_name_component_length above. - -max_account_name_length = 255 - - -# max_container_name_length is the max number of bytes in the utf8 encoding -# of a container name: Gluster FS Filename limit (XFS limit?), must be the same -# size as max_object_name_component_length above. - -max_container_name_length = 255 diff --git a/test/functional_auth/keystone/conf/account-server.conf b/test/functional_auth/keystone/conf/account-server.conf deleted file mode 100644 index 4996367..0000000 --- a/test/functional_auth/keystone/conf/account-server.conf +++ /dev/null @@ -1,32 +0,0 @@ -[DEFAULT] -devices = /mnt/gluster-object -# -# Once you are confident that your startup processes will always have your -# gluster volumes properly mounted *before* the account-server workers start, -# you can *consider* setting this value to "false" to reduce the per-request -# overhead it can incur. -# -# *** Keep false for Functional Tests *** -mount_check = false -bind_port = 6012 -# -# Override swift's default behaviour for fallocate. -disable_fallocate = true -# -# One or two workers should be sufficient for almost any installation of -# Gluster. -workers = 1 - -[pipeline:main] -pipeline = account-server - -[app:account-server] -use = egg:gluster_swift#account -user = root -log_facility = LOG_LOCAL2 -log_level = WARN -# -# After ensuring things are running in a stable manner, you can turn off -# normal request logging for the account server to unclutter the log -# files. Warnings and errors will still be logged. -log_requests = off diff --git a/test/functional_auth/keystone/conf/container-server.conf b/test/functional_auth/keystone/conf/container-server.conf deleted file mode 100644 index 122d97e..0000000 --- a/test/functional_auth/keystone/conf/container-server.conf +++ /dev/null @@ -1,35 +0,0 @@ -[DEFAULT] -devices = /mnt/gluster-object -# -# Once you are confident that your startup processes will always have your -# gluster volumes properly mounted *before* the container-server workers -# start, you can *consider* setting this value to "false" to reduce the -# per-request overhead it can incur. -# -# *** Keep false for Functional Tests *** -mount_check = false -bind_port = 6011 -# -# Override swift's default behaviour for fallocate. -disable_fallocate = true -# -# One or two workers should be sufficient for almost any installation of -# Gluster. -workers = 1 - -[pipeline:main] -pipeline = container-server - -[app:container-server] -use = egg:gluster_swift#container -user = root -log_facility = LOG_LOCAL2 -log_level = WARN -# -# After ensuring things are running in a stable manner, you can turn off -# normal request logging for the container server to unclutter the log -# files. Warnings and errors will still be logged. -log_requests = off - -#enable object versioning for functional test -allow_versions = on diff --git a/test/functional_auth/keystone/conf/fs.conf b/test/functional_auth/keystone/conf/fs.conf deleted file mode 100644 index b06a854..0000000 --- a/test/functional_auth/keystone/conf/fs.conf +++ /dev/null @@ -1,19 +0,0 @@ -[DEFAULT] -# -# IP address of a node in the GlusterFS server cluster hosting the -# volumes to be served via Swift API. -mount_ip = localhost - -# Performance optimization parameter. When turned off, the filesystem will -# see a reduced number of stat calls, resulting in substantially faster -# response time for GET and HEAD container requests on containers with large -# numbers of objects, at the expense of an accurate count of combined bytes -# used by all objects in the container. For most installations "off" works -# fine. -# -# *** Keep on for Functional Tests *** -accurate_size_in_listing = on - -# *** Keep on for Functional Tests *** -container_update_object_count = on -account_update_container_count = on diff --git a/test/functional_auth/keystone/conf/object-server.conf b/test/functional_auth/keystone/conf/object-server.conf deleted file mode 100644 index 3cb9ead..0000000 --- a/test/functional_auth/keystone/conf/object-server.conf +++ /dev/null @@ -1,48 +0,0 @@ -[DEFAULT] -devices = /mnt/gluster-object -# -# Once you are confident that your startup processes will always have your -# gluster volumes properly mounted *before* the object-server workers start, -# you can *consider* setting this value to "false" to reduce the per-request -# overhead it can incur. -# -# *** Keep false for Functional Tests *** -mount_check = false -bind_port = 6010 -# -# Maximum number of clients one worker can process simultaneously (it will -# actually accept N + 1). Setting this to one (1) will only handle one request -# at a time, without accepting another request concurrently. By increasing the -# number of workers to a much higher value, one can prevent slow file system -# operations for one request from starving other requests. -max_clients = 1024 -# -# If not doing the above, setting this value initially to match the number of -# CPUs is a good starting point for determining the right value. -workers = 1 -# Override swift's default behaviour for fallocate. -disable_fallocate = true - -[pipeline:main] -pipeline = object-server - -[app:object-server] -use = egg:gluster_swift#object -user = root -log_facility = LOG_LOCAL2 -log_level = WARN -# -# For performance, after ensuring things are running in a stable manner, you -# can turn off normal request logging for the object server to reduce the -# per-request overhead and unclutter the log files. Warnings and errors will -# still be logged. -log_requests = off -# -# Adjust this value to match the stripe width of the underlying storage array -# (not the stripe element size). This will provide a reasonable starting point -# for tuning this value. -disk_chunk_size = 65536 -# -# Adjust this value match whatever is set for the disk_chunk_size initially. -# This will provide a reasonable starting point for tuning this value. -network_chunk_size = 65556 diff --git a/test/functional_auth/keystone/conf/proxy-server.conf b/test/functional_auth/keystone/conf/proxy-server.conf index 084e6a5..72a84da 100644 --- a/test/functional_auth/keystone/conf/proxy-server.conf +++ b/test/functional_auth/keystone/conf/proxy-server.conf @@ -6,7 +6,7 @@ workers = 1 [pipeline:main] #pipeline = catch_errors healthcheck proxy-logging cache tempauth proxy-logging proxy-server -pipeline = catch_errors healthcheck proxy-logging cache authtoken keystoneauth proxy-logging proxy-server +pipeline = catch_errors healthcheck proxy-logging cache tempurl authtoken keystoneauth proxy-logging proxy-server [app:proxy-server] use = egg:gluster_swift#proxy @@ -49,6 +49,10 @@ object_chunk_size = 65536 # amount of memory available on the system can accommodate increased values # for object_chunk_size. put_queue_depth = 10 +# The following parameter is used by object-expirer and needs to be same +# across all conf files! +auto_create_account_prefix = gs +expiring_objects_account_name = expiring [filter:catch_errors] use = egg:swift#catch_errors @@ -59,13 +63,14 @@ use = egg:swift#proxy_logging [filter:healthcheck] use = egg:swift#healthcheck -[filter:tempauth] -use = egg:swift#tempauth -user_admin_admin = admin .admin .reseller_admin -user_d4dde08c621a4f0fb4cde0ac6a62aa0c_tester = testing .admin -user_test_tester = testing .admin -user_test2_tester2 = testing2 .admin -user_test_tester3 = testing3 +[filter:cache] +use = egg:swift#memcache +# Update this line to contain a comma separated list of memcache servers +# shared by all nodes running the proxy-server service. +memcache_servers = localhost:11211 + +[filter:tempurl] +use = egg:swift#tempurl [filter:authtoken] paste.filter_factory = keystoneclient.middleware.auth_token:filter_factory @@ -87,8 +92,4 @@ operator_roles = admin is_admin = true cache = swift.cache -[filter:cache] -use = egg:swift#memcache -# Update this line to contain a comma separated list of memcache servers -# shared by all nodes running the proxy-server service. -memcache_servers = localhost:11211 + diff --git a/test/functional_auth/keystone/conf/swift.conf b/test/functional_auth/keystone/conf/swift.conf deleted file mode 100644 index ce9a4d0..0000000 --- a/test/functional_auth/keystone/conf/swift.conf +++ /dev/null @@ -1,85 +0,0 @@ -[DEFAULT] - - -[swift-hash] -# random unique string that can never change (DO NOT LOSE) -swift_hash_path_suffix = gluster - - -# The swift-constraints section sets the basic constraints on data -# saved in the swift cluster. - -[swift-constraints] - -# max_file_size is the largest "normal" object that can be saved in -# the cluster. This is also the limit on the size of each segment of -# a "large" object when using the large object manifest support. -# This value is set in bytes. Setting it to lower than 1MiB will cause -# some tests to fail. -# Default is 1 TiB = 2**30*1024 -max_file_size = 1099511627776 - - -# max_meta_name_length is the max number of bytes in the utf8 encoding -# of the name portion of a metadata header. - -#max_meta_name_length = 128 - - -# max_meta_value_length is the max number of bytes in the utf8 encoding -# of a metadata value - -#max_meta_value_length = 256 - - -# max_meta_count is the max number of metadata keys that can be stored -# on a single account, container, or object - -#max_meta_count = 90 - - -# max_meta_overall_size is the max number of bytes in the utf8 encoding -# of the metadata (keys + values) - -#max_meta_overall_size = 4096 - - -# max_object_name_length is the max number of bytes in the utf8 encoding of an -# object name: Gluster FS can handle much longer file names, but the length -# between the slashes of the URL is handled below. Remember that most web -# clients can't handle anything greater than 2048, and those that do are -# rather clumsy. - -max_object_name_length = 2048 - -# max_object_name_component_length (GlusterFS) is the max number of bytes in -# the utf8 encoding of an object name component (the part between the -# slashes); this is a limit imposed by the underlying file system (for XFS it -# is 255 bytes). - -max_object_name_component_length = 255 - -# container_listing_limit is the default (and max) number of items -# returned for a container listing request - -#container_listing_limit = 10000 - - -# account_listing_limit is the default (and max) number of items returned -# for an account listing request - -#account_listing_limit = 10000 - - -# max_account_name_length is the max number of bytes in the utf8 encoding of -# an account name: Gluster FS Filename limit (XFS limit?), must be the same -# size as max_object_name_component_length above. - -max_account_name_length = 255 - - -# max_container_name_length is the max number of bytes in the utf8 encoding -# of a container name: Gluster FS Filename limit (XFS limit?), must be the same -# size as max_object_name_component_length above. - -max_container_name_length = 255 diff --git a/test/functional_auth/swiftkerbauth/__init__.py b/test/functional_auth/swiftkerbauth/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/test/functional_auth/swiftkerbauth/conf/account-server.conf b/test/functional_auth/swiftkerbauth/conf/account-server.conf deleted file mode 100644 index 9ad458a..0000000 --- a/test/functional_auth/swiftkerbauth/conf/account-server.conf +++ /dev/null @@ -1,36 +0,0 @@ -[DEFAULT] -# -# Default gluster mount point to be used for object store,can be changed by -# setting the following value in {account,container,object}-server.conf files. -# It is recommended to keep this value same for all the three services but can -# be kept different if environment demands. -devices = /mnt/gluster-object -# -# Once you are confident that your startup processes will always have your -# gluster volumes properly mounted *before* the account-server workers start, -# you can *consider* setting this value to "false" to reduce the per-request -# overhead it can incur. -mount_check = true -bind_port = 6012 -# -# Override swift's default behaviour for fallocate. -disable_fallocate = true -# -# One or two workers should be sufficient for almost any installation of -# Gluster. -workers = 1 - -[pipeline:main] -pipeline = account-server - -[app:account-server] -use = egg:gluster_swift#account -user = root -log_facility = LOG_LOCAL2 -log_level = WARN -# -# After ensuring things are running in a stable manner, you can turn off -# normal request logging for the account server to unclutter the log -# files. Warnings and errors will still be logged. -log_requests = off - diff --git a/test/functional_auth/swiftkerbauth/conf/container-server.conf b/test/functional_auth/swiftkerbauth/conf/container-server.conf deleted file mode 100644 index a406b4d..0000000 --- a/test/functional_auth/swiftkerbauth/conf/container-server.conf +++ /dev/null @@ -1,36 +0,0 @@ -[DEFAULT] -# -# Default gluster mount point to be used for object store,can be changed by -# setting the following value in {account,container,object}-server.conf files. -# It is recommended to keep this value same for all the three services but can -# be kept different if environment demands. -devices = /mnt/gluster-object -# -# Once you are confident that your startup processes will always have your -# gluster volumes properly mounted *before* the container-server workers -# start, you can *consider* setting this value to "false" to reduce the -# per-request overhead it can incur. -mount_check = true -bind_port = 6011 -# -# Override swift's default behaviour for fallocate. -disable_fallocate = true -# -# One or two workers should be sufficient for almost any installation of -# Gluster. -workers = 1 - -[pipeline:main] -pipeline = container-server - -[app:container-server] -use = egg:gluster_swift#container -user = root -log_facility = LOG_LOCAL2 -log_level = WARN -# -# After ensuring things are running in a stable manner, you can turn off -# normal request logging for the container server to unclutter the log -# files. Warnings and errors will still be logged. -log_requests = off - diff --git a/test/functional_auth/swiftkerbauth/conf/fs.conf b/test/functional_auth/swiftkerbauth/conf/fs.conf deleted file mode 100644 index 6d2a791..0000000 --- a/test/functional_auth/swiftkerbauth/conf/fs.conf +++ /dev/null @@ -1,13 +0,0 @@ -[DEFAULT] -# -# IP address of a node in the GlusterFS server cluster hosting the -# volumes to be served via Swift API. -mount_ip = localhost - -# Performance optimization parameter. When turned off, the filesystem will -# see a reduced number of stat calls, resulting in substantially faster -# response time for GET and HEAD container requests on containers with large -# numbers of objects, at the expense of an accurate count of combined bytes -# used by all objects in the container. For most installations "off" works -# fine. -accurate_size_in_listing = off \ No newline at end of file diff --git a/test/functional_auth/swiftkerbauth/conf/object-server.conf b/test/functional_auth/swiftkerbauth/conf/object-server.conf deleted file mode 100644 index d10d282..0000000 --- a/test/functional_auth/swiftkerbauth/conf/object-server.conf +++ /dev/null @@ -1,51 +0,0 @@ -[DEFAULT] -# -# Default gluster mount point to be used for object store,can be changed by -# setting the following value in {account,container,object}-server.conf files. -# It is recommended to keep this value same for all the three services but can -# be kept different if environment demands. -devices = /mnt/gluster-object -# -# Once you are confident that your startup processes will always have your -# gluster volumes properly mounted *before* the object-server workers start, -# you can *consider* setting this value to "false" to reduce the per-request -# overhead it can incur. -mount_check = true -bind_port = 6010 -# -# Maximum number of clients one worker can process simultaneously (it will -# actually accept N + 1). Setting this to one (1) will only handle one request -# at a time, without accepting another request concurrently. By increasing the -# number of workers to a much higher value, one can prevent slow file system -# operations for one request from starving other requests. -max_clients = 1024 -# -# If not doing the above, setting this value initially to match the number of -# CPUs is a good starting point for determining the right value. -workers = 1 -# Override swift's default behaviour for fallocate. -disable_fallocate = true - -[pipeline:main] -pipeline = object-server - -[app:object-server] -use = egg:gluster_swift#object -user = root -log_facility = LOG_LOCAL2 -log_level = WARN -# -# For performance, after ensuring things are running in a stable manner, you -# can turn off normal request logging for the object server to reduce the -# per-request overhead and unclutter the log files. Warnings and errors will -# still be logged. -log_requests = off -# -# Adjust this value to match the stripe width of the underlying storage array -# (not the stripe element size). This will provide a reasonable starting point -# for tuning this value. -disk_chunk_size = 65536 -# -# Adjust this value match whatever is set for the disk_chunk_size initially. -# This will provide a reasonable starting point for tuning this value. -network_chunk_size = 65536 diff --git a/test/functional_auth/swiftkerbauth/conf/proxy-server.conf b/test/functional_auth/swiftkerbauth/conf/proxy-server.conf index e6f8aa1..248f87c 100644 --- a/test/functional_auth/swiftkerbauth/conf/proxy-server.conf +++ b/test/functional_auth/swiftkerbauth/conf/proxy-server.conf @@ -5,7 +5,7 @@ user = root workers = 1 [pipeline:main] -pipeline = catch_errors healthcheck proxy-logging cache proxy-logging kerbauth proxy-server +pipeline = catch_errors healthcheck proxy-logging cache tempurl proxy-logging kerbauth proxy-server [app:proxy-server] use = egg:gluster_swift#proxy @@ -48,6 +48,10 @@ object_chunk_size = 65536 # amount of memory available on the system can accommodate increased values # for object_chunk_size. put_queue_depth = 10 +# The following parameter is used by object-expirer and needs to be same +# across all conf files! +auto_create_account_prefix = gs +expiring_objects_account_name = expiring [filter:catch_errors] use = egg:swift#catch_errors @@ -59,12 +63,15 @@ access_log_level = WARN [filter:healthcheck] use = egg:swift#healthcheck -[filter:kerbauth] -use = egg:gluster_swift#kerbauth -ext_authentication_url = http://client.rhelbox.com/cgi-bin/swift-auth - [filter:cache] use = egg:swift#memcache # Update this line to contain a comma separated list of memcache servers # shared by all nodes running the proxy-server service. memcache_servers = localhost:11211 + +[filter:tempurl] +use = egg:swift#tempurl + +[filter:kerbauth] +use = egg:gluster_swift#kerbauth +ext_authentication_url = http://client.rhelbox.com/cgi-bin/swift-auth diff --git a/test/functional_auth/swiftkerbauth/conf/swift.conf b/test/functional_auth/swiftkerbauth/conf/swift.conf deleted file mode 100644 index 0d25209..0000000 --- a/test/functional_auth/swiftkerbauth/conf/swift.conf +++ /dev/null @@ -1,84 +0,0 @@ -[DEFAULT] - - -[swift-hash] -# random unique string that can never change (DO NOT LOSE) -swift_hash_path_suffix = gluster - - -# The swift-constraints section sets the basic constraints on data -# saved in the swift cluster. - -[swift-constraints] -# max_file_size is the largest "normal" object that can be saved in -# the cluster. This is also the limit on the size of each segment of -# a "large" object when using the large object manifest support. -# This value is set in bytes. Setting it to lower than 1MiB will cause -# some tests to fail. -# Default is 1 TiB = 2**30*1024 - -max_file_size = 1099511627776 - -# max_meta_name_length is the max number of bytes in the utf8 encoding -# of the name portion of a metadata header. - -#max_meta_name_length = 128 - - -# max_meta_value_length is the max number of bytes in the utf8 encoding -# of a metadata value - -#max_meta_value_length = 256 - - -# max_meta_count is the max number of metadata keys that can be stored -# on a single account, container, or object - -#max_meta_count = 90 - - -# max_meta_overall_size is the max number of bytes in the utf8 encoding -# of the metadata (keys + values) - -#max_meta_overall_size = 4096 - - -# max_object_name_length is the max number of bytes in the utf8 encoding of an -# object name: Gluster FS can handle much longer file names, but the length -# between the slashes of the URL is handled below. Remember that most web -# clients can't handle anything greater than 2048, and those that do are -# rather clumsy. - -max_object_name_length = 2048 - -# max_object_name_component_length (GlusterFS) is the max number of bytes in -# the utf8 encoding of an object name component (the part between the -# slashes); this is a limit imposed by the underlying file system (for XFS it -# is 255 bytes). - -max_object_name_component_length = 255 - -# container_listing_limit is the default (and max) number of items -# returned for a container listing request - -#container_listing_limit = 10000 - - -# account_listing_limit is the default (and max) number of items returned -# for an account listing request - -#account_listing_limit = 10000 - - -# max_account_name_length is the max number of bytes in the utf8 encoding of -# an account name: Gluster FS Filename limit (XFS limit?), must be the same -# size as max_object_name_component_length above. - -max_account_name_length = 255 - - -# max_container_name_length is the max number of bytes in the utf8 encoding -# of a container name: Gluster FS Filename limit (XFS limit?), must be the same -# size as max_object_name_component_length above. - -max_container_name_length = 255 diff --git a/test/functional_auth/swiftkerbauth/conf/test.conf b/test/functional_auth/swiftkerbauth/conf/test.conf deleted file mode 100644 index 643f2d1..0000000 --- a/test/functional_auth/swiftkerbauth/conf/test.conf +++ /dev/null @@ -1,49 +0,0 @@ -[func_test] -# Swiftkerbauth configuration -auth_host = 127.0.0.1 -auth_port = 8080 -auth_prefix = /auth/ -auth_scheme = http:// -auth_mode = passive -auth_version = 1 -domain_name = RHELBOX.COM - -#All the accounts, users & passwords to be prepared on kerberos server. -# Primary functional test account (needs admin access to the account) -# Note: Account name to be prepared on kerberos server 'AUTH_accoun' -account = test -username = tester -password = testing - -# User on a second account (needs admin access to the account) -account2 = test2 -username2 = tester2 -password2 = testing2 - -# User on same account as first, but without admin access -username3 = tester3 -password3 = testing3 - -# Default constraints if not defined here, the test runner will try -# to set them from /etc/swift/swift.conf. If that file isn't found, -# the test runner will skip tests that depend on these values. -# Note that the cluster must have "sane" values for the test suite to pass. -#max_file_size = 5368709122 -#max_meta_name_length = 128 -#max_meta_value_length = 256 -#max_meta_count = 90 -#max_meta_overall_size = 4096 -#max_object_name_length = 1024 -#container_listing_limit = 10000 -#account_listing_limit = 10000 -#max_account_name_length = 256 -#max_container_name_length = 256 -normalized_urls = True - -collate = C - -[unit_test] -fake_syslog = False - -[probe_test] -# check_server_timeout = 30 diff --git a/test/functional_auth/swiftkerbauth/test_swkrbath_active.py b/test/functional_auth/swiftkerbauth/test_swkrbath_active.py deleted file mode 100644 index 86c79ef..0000000 --- a/test/functional_auth/swiftkerbauth/test_swkrbath_active.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/python - -# Copyright (c) 2010-2014 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re -import unittest -from nose import SkipTest -import commands -import os -from test import get_config -from swift.common.bufferedhttp import http_connect_raw as http_connect - -config = get_config('func_test') - -class Utils: - @classmethod - def SwiftKerbAuthPrep(self, - user=config['username'],domain=config['domain_name'],\ - passwd=config['password']): - username = '%s@%s' % (user, domain) - return commands.getstatusoutput('kinit %s <<< %s' % (username, passwd)) - - @classmethod - def SwiftKerbAuthCleanAll(self): - return commands.getstatusoutput('kdestroy') - - -class TestSwKrbAthActive(unittest.TestCase): - def setUp(self): - #Perform kinit in active mode. - (status, output) = Utils.SwiftKerbAuthPrep() - self.assertEqual(status, 0, \ - 'swkrbauth prep failed with valid credentials'+output) - self.auth_host = config['auth_host'] - self.auth_port = int(config['auth_port']) - self.auth_prefix = config.get('auth_prefix', '/auth/') - self.auth_version = str(config.get('auth_version', '1')) - self.account_name = config['account'] - self.username = config['username'] - self.password = config['password'] - self.auth_scheme = config['auth_scheme'] - - #Prepare auth_url. e.g. http://client.rhelbox.com:8080/auth/v1.0 - if self.auth_version == "1": - self.auth_path = '%sv1.0' % (self.auth_prefix) - else: - self.auth_path = self.auth_prefix - self.auth_netloc = "%s:%d" % (self.auth_host, self.auth_port) - auth_url = self.auth_scheme + self.auth_netloc + self.auth_path - - #Obtain the X-Auth-Token from kerberos server to use it in furhter - #testing - self.auth_token = None - (status, output) = commands.getstatusoutput('curl -v -u : --negotiate\ - --location-trusted %s' % (auth_url)) - self.assertEqual(status, 0, 'Token negotiation failed:' +output) - match = re.search('X-Auth-Token: AUTH.*', output) - if match: - self.auth_token = match.group(0).split(':')[1].strip() - else: - self.fail('No X-Auth-Token found, failed') - - def tearDown(self): - Utils.SwiftKerbAuthCleanAll() - - - def _get_auth_token(self): - return {'X-Auth-Token' : self.auth_token} - - def testGetAccounts(self): - #TODO: The test case is to perform GET on the account mentioned via - #configuration file. This is a sample test case. The whole test - #suite can be enhanced further to have further complicated test cases. - path = '/v1/AUTH_%s' % (config['account']) - - headers = self._get_auth_token() - conn = http_connect(config['auth_host'], config['auth_port'], 'GET', - path, headers) - resp = conn.getresponse() - self.assertTrue(resp.status == 204) diff --git a/test/functional_auth/tempauth/conf/account-server.conf b/test/functional_auth/tempauth/conf/account-server.conf deleted file mode 100644 index 4996367..0000000 --- a/test/functional_auth/tempauth/conf/account-server.conf +++ /dev/null @@ -1,32 +0,0 @@ -[DEFAULT] -devices = /mnt/gluster-object -# -# Once you are confident that your startup processes will always have your -# gluster volumes properly mounted *before* the account-server workers start, -# you can *consider* setting this value to "false" to reduce the per-request -# overhead it can incur. -# -# *** Keep false for Functional Tests *** -mount_check = false -bind_port = 6012 -# -# Override swift's default behaviour for fallocate. -disable_fallocate = true -# -# One or two workers should be sufficient for almost any installation of -# Gluster. -workers = 1 - -[pipeline:main] -pipeline = account-server - -[app:account-server] -use = egg:gluster_swift#account -user = root -log_facility = LOG_LOCAL2 -log_level = WARN -# -# After ensuring things are running in a stable manner, you can turn off -# normal request logging for the account server to unclutter the log -# files. Warnings and errors will still be logged. -log_requests = off diff --git a/test/functional_auth/tempauth/conf/container-server.conf b/test/functional_auth/tempauth/conf/container-server.conf deleted file mode 100644 index 122d97e..0000000 --- a/test/functional_auth/tempauth/conf/container-server.conf +++ /dev/null @@ -1,35 +0,0 @@ -[DEFAULT] -devices = /mnt/gluster-object -# -# Once you are confident that your startup processes will always have your -# gluster volumes properly mounted *before* the container-server workers -# start, you can *consider* setting this value to "false" to reduce the -# per-request overhead it can incur. -# -# *** Keep false for Functional Tests *** -mount_check = false -bind_port = 6011 -# -# Override swift's default behaviour for fallocate. -disable_fallocate = true -# -# One or two workers should be sufficient for almost any installation of -# Gluster. -workers = 1 - -[pipeline:main] -pipeline = container-server - -[app:container-server] -use = egg:gluster_swift#container -user = root -log_facility = LOG_LOCAL2 -log_level = WARN -# -# After ensuring things are running in a stable manner, you can turn off -# normal request logging for the container server to unclutter the log -# files. Warnings and errors will still be logged. -log_requests = off - -#enable object versioning for functional test -allow_versions = on diff --git a/test/functional_auth/tempauth/conf/fs.conf b/test/functional_auth/tempauth/conf/fs.conf deleted file mode 100644 index b06a854..0000000 --- a/test/functional_auth/tempauth/conf/fs.conf +++ /dev/null @@ -1,19 +0,0 @@ -[DEFAULT] -# -# IP address of a node in the GlusterFS server cluster hosting the -# volumes to be served via Swift API. -mount_ip = localhost - -# Performance optimization parameter. When turned off, the filesystem will -# see a reduced number of stat calls, resulting in substantially faster -# response time for GET and HEAD container requests on containers with large -# numbers of objects, at the expense of an accurate count of combined bytes -# used by all objects in the container. For most installations "off" works -# fine. -# -# *** Keep on for Functional Tests *** -accurate_size_in_listing = on - -# *** Keep on for Functional Tests *** -container_update_object_count = on -account_update_container_count = on diff --git a/test/functional_auth/tempauth/conf/object-expirer.conf b/test/functional_auth/tempauth/conf/object-expirer.conf deleted file mode 100644 index b75963c..0000000 --- a/test/functional_auth/tempauth/conf/object-expirer.conf +++ /dev/null @@ -1,17 +0,0 @@ -[DEFAULT] - -[object-expirer] -# auto_create_account_prefix = . - -[pipeline:main] -pipeline = catch_errors cache proxy-server - -[app:proxy-server] -use = egg:swift#proxy - -[filter:cache] -use = egg:swift#memcache -memcache_servers = 127.0.0.1:11211 - -[filter:catch_errors] -use = egg:swift#catch_errors diff --git a/test/functional_auth/tempauth/conf/object-server.conf b/test/functional_auth/tempauth/conf/object-server.conf deleted file mode 100644 index 3cb9ead..0000000 --- a/test/functional_auth/tempauth/conf/object-server.conf +++ /dev/null @@ -1,48 +0,0 @@ -[DEFAULT] -devices = /mnt/gluster-object -# -# Once you are confident that your startup processes will always have your -# gluster volumes properly mounted *before* the object-server workers start, -# you can *consider* setting this value to "false" to reduce the per-request -# overhead it can incur. -# -# *** Keep false for Functional Tests *** -mount_check = false -bind_port = 6010 -# -# Maximum number of clients one worker can process simultaneously (it will -# actually accept N + 1). Setting this to one (1) will only handle one request -# at a time, without accepting another request concurrently. By increasing the -# number of workers to a much higher value, one can prevent slow file system -# operations for one request from starving other requests. -max_clients = 1024 -# -# If not doing the above, setting this value initially to match the number of -# CPUs is a good starting point for determining the right value. -workers = 1 -# Override swift's default behaviour for fallocate. -disable_fallocate = true - -[pipeline:main] -pipeline = object-server - -[app:object-server] -use = egg:gluster_swift#object -user = root -log_facility = LOG_LOCAL2 -log_level = WARN -# -# For performance, after ensuring things are running in a stable manner, you -# can turn off normal request logging for the object server to reduce the -# per-request overhead and unclutter the log files. Warnings and errors will -# still be logged. -log_requests = off -# -# Adjust this value to match the stripe width of the underlying storage array -# (not the stripe element size). This will provide a reasonable starting point -# for tuning this value. -disk_chunk_size = 65536 -# -# Adjust this value match whatever is set for the disk_chunk_size initially. -# This will provide a reasonable starting point for tuning this value. -network_chunk_size = 65556 diff --git a/test/functional_auth/tempauth/conf/proxy-server.conf b/test/functional_auth/tempauth/conf/proxy-server.conf index 830fadf..554ce61 100644 --- a/test/functional_auth/tempauth/conf/proxy-server.conf +++ b/test/functional_auth/tempauth/conf/proxy-server.conf @@ -5,7 +5,7 @@ user = root workers = 1 [pipeline:main] -pipeline = catch_errors healthcheck proxy-logging cache tempauth proxy-logging proxy-server +pipeline = catch_errors healthcheck proxy-logging cache tempurl tempauth proxy-logging proxy-server [app:proxy-server] use = egg:gluster_swift#proxy @@ -48,6 +48,10 @@ object_chunk_size = 65536 # amount of memory available on the system can accommodate increased values # for object_chunk_size. put_queue_depth = 10 +# The following parameter is used by object-expirer and needs to be same +# across all conf files! +auto_create_account_prefix = gs +expiring_objects_account_name = expiring [filter:catch_errors] use = egg:swift#catch_errors @@ -58,15 +62,18 @@ use = egg:swift#proxy_logging [filter:healthcheck] use = egg:swift#healthcheck +[filter:cache] +use = egg:swift#memcache +# Update this line to contain a comma separated list of memcache servers +# shared by all nodes running the proxy-server service. +memcache_servers = localhost:11211 + +[filter:tempurl] +use = egg:swift#tempurl + [filter:tempauth] use = egg:swift#tempauth user_admin_admin = admin .admin .reseller_admin user_test_tester = testing .admin user_test2_tester2 = testing2 .admin user_test_tester3 = testing3 - -[filter:cache] -use = egg:swift#memcache -# Update this line to contain a comma separated list of memcache servers -# shared by all nodes running the proxy-server service. -memcache_servers = localhost:11211 diff --git a/test/functional_auth/tempauth/conf/swift.conf b/test/functional_auth/tempauth/conf/swift.conf deleted file mode 100644 index ce9a4d0..0000000 --- a/test/functional_auth/tempauth/conf/swift.conf +++ /dev/null @@ -1,85 +0,0 @@ -[DEFAULT] - - -[swift-hash] -# random unique string that can never change (DO NOT LOSE) -swift_hash_path_suffix = gluster - - -# The swift-constraints section sets the basic constraints on data -# saved in the swift cluster. - -[swift-constraints] - -# max_file_size is the largest "normal" object that can be saved in -# the cluster. This is also the limit on the size of each segment of -# a "large" object when using the large object manifest support. -# This value is set in bytes. Setting it to lower than 1MiB will cause -# some tests to fail. -# Default is 1 TiB = 2**30*1024 -max_file_size = 1099511627776 - - -# max_meta_name_length is the max number of bytes in the utf8 encoding -# of the name portion of a metadata header. - -#max_meta_name_length = 128 - - -# max_meta_value_length is the max number of bytes in the utf8 encoding -# of a metadata value - -#max_meta_value_length = 256 - - -# max_meta_count is the max number of metadata keys that can be stored -# on a single account, container, or object - -#max_meta_count = 90 - - -# max_meta_overall_size is the max number of bytes in the utf8 encoding -# of the metadata (keys + values) - -#max_meta_overall_size = 4096 - - -# max_object_name_length is the max number of bytes in the utf8 encoding of an -# object name: Gluster FS can handle much longer file names, but the length -# between the slashes of the URL is handled below. Remember that most web -# clients can't handle anything greater than 2048, and those that do are -# rather clumsy. - -max_object_name_length = 2048 - -# max_object_name_component_length (GlusterFS) is the max number of bytes in -# the utf8 encoding of an object name component (the part between the -# slashes); this is a limit imposed by the underlying file system (for XFS it -# is 255 bytes). - -max_object_name_component_length = 255 - -# container_listing_limit is the default (and max) number of items -# returned for a container listing request - -#container_listing_limit = 10000 - - -# account_listing_limit is the default (and max) number of items returned -# for an account listing request - -#account_listing_limit = 10000 - - -# max_account_name_length is the max number of bytes in the utf8 encoding of -# an account name: Gluster FS Filename limit (XFS limit?), must be the same -# size as max_object_name_component_length above. - -max_account_name_length = 255 - - -# max_container_name_length is the max number of bytes in the utf8 encoding -# of a container name: Gluster FS Filename limit (XFS limit?), must be the same -# size as max_object_name_component_length above. - -max_container_name_length = 255 diff --git a/test/object_expirer_functional/__init__.py b/test/object_expirer_functional/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/object_expirer_functional/test_object_expirer.py b/test/object_expirer_functional/test_object_expirer.py new file mode 100644 index 0000000..aaec75e --- /dev/null +++ b/test/object_expirer_functional/test_object_expirer.py @@ -0,0 +1,332 @@ +# Copyright (c) 2014 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time + +from swift.common.manager import Manager +from swift.common.internal_client import InternalClient + +from test.functional.tests import Base, config, Utils +from test.functional.swift_test_client import Account, Connection, \ + ResponseError + + +class TestObjectExpirerEnv: + @classmethod + def setUp(cls): + cls.conn = Connection(config) + cls.conn.authenticate() + cls.account = Account(cls.conn, + config.get('account', + config['username'])) + cls.account.delete_containers() + cls.container = cls.account.container(Utils.create_name()) + if not cls.container.create(): + raise ResponseError(cls.conn.response) + cls.file_size = 8 + cls.root_dir = os.path.join('/mnt/gluster-object', + cls.account.conn.storage_url.split('/')[2].split('_')[1]) + cls.client = InternalClient('/etc/swift/object-expirer.conf', + 'Test Object Expirer', 1) + cls.expirer = Manager(['object-expirer']) + + +class TestObjectExpirer(Base): + env = TestObjectExpirerEnv + set_up = False + + def test_object_expiry_X_Delete_At_PUT(self): + obj = self.env.container.file(Utils.create_name()) + x_delete_at = str(int(time.time()) + 2) + obj.write_random(self.env.file_size, + hdrs={'X-Delete-At': x_delete_at}) + + # Object is not expired. Should still be accessible. + obj.read() + self.assert_status(200) + + # Ensure X-Delete-At is saved as object metadata. + self.assertEqual(x_delete_at, str(obj.info()['x_delete_at'])) + + # Wait for object to be expired. + time.sleep(3) + + # Object has expired. Should no longer be accessible. + self.assertRaises(ResponseError, obj.read) + self.assert_status(404) + + # Object should still be present on filesystem. + self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # But, GET on container should list the expired object. + result = self.env.container.files() + self.assertTrue(obj.name in self.env.container.files()) + + # Check existence of corresponding tracker object in gsexpiring + # account. + enteredLoop = False + for c in self.env.client.iter_containers("gsexpiring"): + for o in self.env.client.iter_objects("gsexpiring", c['name']): + enteredLoop = True + l = o['name'].split('/') + self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name)) + self.assertEqual(l[1], self.env.container.name) + self.assertEqual(l[2], obj.name) + if not enteredLoop: + self.fail("Tracker object not found.") + + # Run expirer daemon once. + self.env.expirer.once() + + # Ensure object is physically deleted from filesystem. + self.assertFalse(os.path.exists(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # Ensure tracker object is consumed. + try: + self.env.client.iter_containers("gsexpiring").next() + except StopIteration: + pass + else: + self.fail("Tracker object persists!") + + # GET on container should no longer list the object. + self.assertFalse(obj.name in self.env.container.files()) + + def test_object_expiry_X_Delete_After_PUT(self): + obj = self.env.container.file(Utils.create_name()) + obj.write_random(self.env.file_size, + hdrs={'X-Delete-After': 2}) + + # Object is not expired. Should still be accessible. + obj.read() + self.assert_status(200) + + # Ensure X-Delete-At is saved as object metadata. + self.assertTrue(str(obj.info()['x_delete_at'])) + + # Wait for object to be expired. + time.sleep(3) + + # Object has expired. Should no longer be accessible. + self.assertRaises(ResponseError, obj.read) + self.assert_status(404) + + # Object should still be present on filesystem. + self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # But, GET on container should list the expired object. + result = self.env.container.files() + self.assertTrue(obj.name in self.env.container.files()) + + # Check existence of corresponding tracker object in gsexpiring + # account. + enteredLoop = False + for c in self.env.client.iter_containers("gsexpiring"): + for o in self.env.client.iter_objects("gsexpiring", c['name']): + enteredLoop = True + l = o['name'].split('/') + self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name)) + self.assertEqual(l[1], self.env.container.name) + self.assertEqual(l[2], obj.name) + if not enteredLoop: + self.fail("Tracker object not found.") + + # Run expirer daemon once. + self.env.expirer.once() + + # Ensure object is physically deleted from filesystem. + self.assertFalse(os.path.exists(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # Ensure tracker object is consumed. + try: + self.env.client.iter_containers("gsexpiring").next() + except StopIteration: + pass + else: + self.fail("Tracker object persists!") + + # GET on container should no longer list the object. + self.assertFalse(obj.name in self.env.container.files()) + + + def test_object_expiry_X_Delete_At_POST(self): + + # Create normal object + obj = self.env.container.file(Utils.create_name()) + obj.write_random(self.env.file_size) + obj.read() + self.assert_status(200) + + # Send POST on that object and set it to be expired. + x_delete_at = str(int(time.time()) + 2) + obj.sync_metadata(metadata={'X-Delete-At': x_delete_at}, + cfg={'x_delete_at': x_delete_at}) + + # Ensure X-Delete-At is saved as object metadata. + self.assertEqual(x_delete_at, str(obj.info()['x_delete_at'])) + + # Object is not expired. Should still be accessible. + obj.read() + self.assert_status(200) + + # Wait for object to be expired. + time.sleep(3) + + # Object has expired. Should no longer be accessible. + self.assertRaises(ResponseError, obj.read) + self.assert_status(404) + + # Object should still be present on filesystem. + self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # But, GET on container should list the expired object. + result = self.env.container.files() + self.assertTrue(obj.name in self.env.container.files()) + + # Check existence of corresponding tracker object in gsexpiring + # account. + + enteredLoop = False + for c in self.env.client.iter_containers("gsexpiring"): + for o in self.env.client.iter_objects("gsexpiring", c['name']): + enteredLoop = True + l = o['name'].split('/') + self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name)) + self.assertEqual(l[1], self.env.container.name) + self.assertEqual(l[2], obj.name) + if not enteredLoop: + self.fail("Tracker object not found.") + + # Run expirer daemon once. + self.env.expirer.once() + time.sleep(3) + + # Ensure object is physically deleted from filesystem. + self.assertFalse(os.path.exists(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # Ensure tracker object is consumed. + try: + self.env.client.iter_containers("gsexpiring").next() + except StopIteration: + pass + else: + self.fail("Tracker object persists!") + + # GET on container should no longer list the object. + self.assertFalse(obj.name in self.env.container.files()) + + + def test_object_expiry_X_Delete_After_POST(self): + + # Create normal object + obj = self.env.container.file(Utils.create_name()) + obj.write_random(self.env.file_size) + obj.read() + self.assert_status(200) + + # Send POST on that object and set it to be expired. + obj.sync_metadata(metadata={'X-Delete-After': 2}, + cfg={'x_delete_after': 2}) + + # Ensure X-Delete-At is saved as object metadata. + self.assertTrue(str(obj.info()['x_delete_at'])) + + # Object is not expired. Should still be accessible. + obj.read() + self.assert_status(200) + + # Wait for object to be expired. + time.sleep(3) + + # Object has expired. Should no longer be accessible. + self.assertRaises(ResponseError, obj.read) + self.assert_status(404) + + # Object should still be present on filesystem. + self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # But, GET on container should list the expired object. + result = self.env.container.files() + self.assertTrue(obj.name in self.env.container.files()) + + # Check existence of corresponding tracker object in gsexpiring + # account. + + enteredLoop = False + for c in self.env.client.iter_containers("gsexpiring"): + for o in self.env.client.iter_objects("gsexpiring", c['name']): + enteredLoop = True + l = o['name'].split('/') + self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name)) + self.assertEqual(l[1], self.env.container.name) + self.assertEqual(l[2], obj.name) + if not enteredLoop: + self.fail("Tracker object not found.") + + # Run expirer daemon once. + self.env.expirer.once() + time.sleep(3) + + # Ensure object is physically deleted from filesystem. + self.assertFalse(os.path.exists(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # Ensure tracker object is consumed. + try: + self.env.client.iter_containers("gsexpiring").next() + except StopIteration: + pass + else: + self.fail("Tracker object persists!") + + # GET on container should no longer list the object. + self.assertFalse(obj.name in self.env.container.files()) + + + def test_object_expiry_err(self): + obj = self.env.container.file(Utils.create_name()) + + # X-Delete-At is invalid or is in the past + for i in (-2, 'abc', str(int(time.time()) - 2), 5.8): + self.assertRaises(ResponseError, + obj.write_random, + self.env.file_size, + hdrs={'X-Delete-At': i}) + self.assert_status(400) + + # X-Delete-After is invalid. + for i in (-2, 'abc', 3.7): + self.assertRaises(ResponseError, + obj.write_random, + self.env.file_size, + hdrs={'X-Delete-After': i}) + self.assert_status(400) + diff --git a/test/unit/__init__.py b/test/unit/__init__.py index 847479a..a1bfef8 100644 --- a/test/unit/__init__.py +++ b/test/unit/__init__.py @@ -33,6 +33,7 @@ from hashlib import md5 from eventlet import sleep, Timeout import logging.handlers from httplib import HTTPException +from numbers import Number class FakeRing(object): @@ -248,6 +249,7 @@ class FakeLogger(logging.Logger): if 'facility' in kwargs: self.facility = kwargs['facility'] self.statsd_client = None + self.thread_locals = None def _clear(self): self.log_dict = defaultdict(list) @@ -465,8 +467,11 @@ def fake_http_connect(*code_iter, **kwargs): self.body = body self.headers = headers or {} self.timestamp = timestamp - if kwargs.get('slow') and isinstance(kwargs['slow'], list): - kwargs['slow'][0] -= 1 + if 'slow' in kwargs and isinstance(kwargs['slow'], list): + try: + self._next_sleep = kwargs['slow'].pop(0) + except IndexError: + self._next_sleep = None def getresponse(self): if kwargs.get('raise_exc'): @@ -482,6 +487,8 @@ def fake_http_connect(*code_iter, **kwargs): return FakeConn(507) if self.expect_status == -4: return FakeConn(201) + if self.expect_status == 412: + return FakeConn(412) return FakeConn(100) def getheaders(self): @@ -510,31 +517,39 @@ def fake_http_connect(*code_iter, **kwargs): headers['x-container-timestamp'] = '1' except StopIteration: pass - if self.am_slow(): + am_slow, value = self.get_slow() + if am_slow: headers['content-length'] = '4' headers.update(self.headers) return headers.items() - def am_slow(self): - if kwargs.get('slow') and isinstance(kwargs['slow'], list): - return kwargs['slow'][0] >= 0 - return bool(kwargs.get('slow')) + def get_slow(self): + if 'slow' in kwargs and isinstance(kwargs['slow'], list): + if self._next_sleep is not None: + return True, self._next_sleep + else: + return False, 0.01 + if kwargs.get('slow') and isinstance(kwargs['slow'], Number): + return True, kwargs['slow'] + return bool(kwargs.get('slow')), 0.1 def read(self, amt=None): - if self.am_slow(): + am_slow, value = self.get_slow() + if am_slow: if self.sent < 4: self.sent += 1 - sleep(0.1) + sleep(value) return ' ' rv = self.body[:amt] self.body = self.body[amt:] return rv def send(self, amt=None): - if self.am_slow(): + am_slow, value = self.get_slow() + if am_slow: if self.received < 4: self.received += 1 - sleep(0.1) + sleep(value) def getheader(self, name, default=None): return dict(self.getheaders()).get(name.lower(), default) @@ -584,4 +599,6 @@ def fake_http_connect(*code_iter, **kwargs): return FakeConn(status, etag, body=body, timestamp=timestamp, expect_status=expect_status, headers=headers) + connect.code_iter = code_iter + return connect diff --git a/test/unit/common/middleware/swiftkerbauth/__init__.py b/test/unit/common/middleware/swiftkerbauth/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/test/unit/common/middleware/swiftkerbauth/test_kerbauth.py b/test/unit/common/middleware/swiftkerbauth/test_kerbauth.py deleted file mode 100644 index 537b8d3..0000000 --- a/test/unit/common/middleware/swiftkerbauth/test_kerbauth.py +++ /dev/null @@ -1,478 +0,0 @@ -# Copyright (c) 2013 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import errno -import unittest -from time import time -from mock import patch, Mock -from test.unit import FakeMemcache -from swift.common.swob import Request, Response -from gluster.swift.common.middleware.swiftkerbauth import kerbauth as auth - -EXT_AUTHENTICATION_URL = "127.0.0.1" -REDIRECT_STATUS = 303 # HTTPSeeOther - - -def my_filter_factory(global_conf, **local_conf): - if 'ext_authentication_url' not in global_conf: - global_conf['ext_authentication_url'] = EXT_AUTHENTICATION_URL - conf = global_conf.copy() - conf.update(local_conf) - - def auth_filter(app): - return auth.KerbAuth(app, conf) - return auth_filter - -# Monkey patching filter_factory to always pass ext_authentication_url -# as a parameter. Absence of ext_authentication_url raises a RuntimeError - - -def patch_filter_factory(): - auth.filter_factory = my_filter_factory - - -def unpatch_filter_factory(): - reload(auth) - - -class FakeApp(object): - - def __init__(self, status_headers_body_iter=None, acl=None, sync_key=None): - self.calls = 0 - self.status_headers_body_iter = status_headers_body_iter - if not self.status_headers_body_iter: - self.status_headers_body_iter = iter([('404 Not Found', {}, '')]) - self.acl = acl - self.sync_key = sync_key - - def __call__(self, env, start_response): - self.calls += 1 - self.request = Request.blank('', environ=env) - if self.acl: - self.request.acl = self.acl - if self.sync_key: - self.request.environ['swift_sync_key'] = self.sync_key - if 'swift.authorize' in env: - resp = env['swift.authorize'](self.request) - if resp: - return resp(env, start_response) - status, headers, body = self.status_headers_body_iter.next() - return Response(status=status, headers=headers, - body=body)(env, start_response) - - -class TestKerbAuth(unittest.TestCase): - - # Patch auth.filter_factory() - patch_filter_factory() - - def setUp(self): - self.test_auth = \ - auth.filter_factory({'auth_method': 'active'})(FakeApp()) - self.test_auth_passive = \ - auth.filter_factory({'auth_method': 'passive'})(FakeApp()) - - def _make_request(self, path, **kwargs): - req = Request.blank(path, **kwargs) - req.environ['swift.cache'] = FakeMemcache() - return req - - def test_no_ext_authentication_url(self): - app = FakeApp() - try: - # Use original auth.filter_factory and NOT monkey patched version - unpatch_filter_factory() - auth.filter_factory({})(app) - except RuntimeError as e: - # Restore monkey patched version - patch_filter_factory() - self.assertTrue(e.args[0].startswith("Missing filter parameter " - "ext_authentication_url")) - - def test_reseller_prefix_init(self): - app = FakeApp() - ath = auth.filter_factory({})(app) - self.assertEquals(ath.reseller_prefix, 'AUTH_') - ath = auth.filter_factory({'reseller_prefix': 'TEST'})(app) - self.assertEquals(ath.reseller_prefix, 'TEST_') - ath = auth.filter_factory({'reseller_prefix': 'TEST_'})(app) - self.assertEquals(ath.reseller_prefix, 'TEST_') - - def test_auth_prefix_init(self): - app = FakeApp() - ath = auth.filter_factory({})(app) - self.assertEquals(ath.auth_prefix, '/auth/') - ath = auth.filter_factory({'auth_prefix': ''})(app) - self.assertEquals(ath.auth_prefix, '/auth/') - ath = auth.filter_factory({'auth_prefix': '/'})(app) - self.assertEquals(ath.auth_prefix, '/auth/') - ath = auth.filter_factory({'auth_prefix': '/test/'})(app) - self.assertEquals(ath.auth_prefix, '/test/') - ath = auth.filter_factory({'auth_prefix': '/test'})(app) - self.assertEquals(ath.auth_prefix, '/test/') - ath = auth.filter_factory({'auth_prefix': 'test/'})(app) - self.assertEquals(ath.auth_prefix, '/test/') - ath = auth.filter_factory({'auth_prefix': 'test'})(app) - self.assertEquals(ath.auth_prefix, '/test/') - - def test_top_level_redirect(self): - req = self._make_request('/') - resp = req.get_response(self.test_auth) - self.assertEquals(resp.status_int, REDIRECT_STATUS) - self.assertEquals(req.environ['swift.authorize'], - self.test_auth.denied_response) - - def test_passive_top_level_deny(self): - req = self._make_request('/') - resp = req.get_response(self.test_auth_passive) - self.assertEquals(resp.status_int, 401) - self.assertEquals(req.environ['swift.authorize'], - self.test_auth_passive.denied_response) - - def test_passive_deny_invalid_token(self): - req = self._make_request('/v1/AUTH_account', - headers={'X-Auth-Token': 'AUTH_t'}) - resp = req.get_response(self.test_auth_passive) - self.assertEquals(resp.status_int, 401) - - def test_override_asked_for_and_allowed(self): - self.test_auth = \ - auth.filter_factory({'allow_overrides': 'true'})(FakeApp()) - req = self._make_request('/v1/AUTH_account', - environ={'swift.authorize_override': True}) - resp = req.get_response(self.test_auth) - self.assertEquals(resp.status_int, 404) - self.assertTrue('swift.authorize' not in req.environ) - - def test_override_default_allowed(self): - req = self._make_request('/v1/AUTH_account', - environ={'swift.authorize_override': True}) - resp = req.get_response(self.test_auth) - self.assertEquals(resp.status_int, 404) - self.assertTrue('swift.authorize' not in req.environ) - - def test_options_call(self): - req = self._make_request('/v1/AUTH_cfa/c/o', - environ={'REQUEST_METHOD': 'OPTIONS'}) - resp = self.test_auth.authorize(req) - self.assertEquals(resp, None) - - def test_auth_deny_non_reseller_prefix_no_override(self): - fake_authorize = lambda x: Response(status='500 Fake') - req = self._make_request('/v1/BLAH_account', - headers={'X-Auth-Token': 'BLAH_t'}, - environ={'swift.authorize': fake_authorize} - ) - resp = req.get_response(self.test_auth) - self.assertEquals(resp.status_int, 500) - self.assertEquals(req.environ['swift.authorize'], fake_authorize) - - def test_authorize_acl_group_access(self): - req = self._make_request('/v1/AUTH_cfa') - req.remote_user = 'act:usr,act' - resp = self.test_auth.authorize(req) - self.assertEquals(resp.status_int, 403) - req = self._make_request('/v1/AUTH_cfa') - req.remote_user = 'act:usr,act' - req.acl = 'act' - self.assertEquals(self.test_auth.authorize(req), None) - req = self._make_request('/v1/AUTH_cfa') - req.remote_user = 'act:usr,act' - req.acl = 'act:usr' - self.assertEquals(self.test_auth.authorize(req), None) - req = self._make_request('/v1/AUTH_cfa') - req.remote_user = 'act:usr,act' - - def test_deny_cross_reseller(self): - # Tests that cross-reseller is denied, even if ACLs/group names match - req = self._make_request('/v1/OTHER_cfa') - req.remote_user = 'act:usr,act,AUTH_cfa' - req.acl = 'act' - resp = self.test_auth.authorize(req) - self.assertEquals(resp.status_int, 403) - - def test_authorize_acl_referer_after_user_groups(self): - req = self._make_request('/v1/AUTH_cfa/c') - req.remote_user = 'act:usr' - req.acl = '.r:*,act:usr' - self.assertEquals(self.test_auth.authorize(req), None) - - def test_detect_reseller_request(self): - req = self._make_request('/v1/AUTH_admin', - headers={'X-Auth-Token': 'AUTH_t'}) - cache_key = 'AUTH_/token/AUTH_t' - cache_entry = (time() + 3600, '.reseller_admin') - req.environ['swift.cache'].set(cache_key, cache_entry) - req.get_response(self.test_auth) - self.assertTrue(req.environ.get('reseller_request', False)) - - def test_regular_is_not_owner(self): - orig_authorize = self.test_auth.authorize - owner_values = [] - - def mitm_authorize(req): - rv = orig_authorize(req) - owner_values.append(req.environ.get('swift_owner', False)) - return rv - - self.test_auth.authorize = mitm_authorize - - req = self._make_request( - '/v1/AUTH_cfa/c', - headers={'X-Auth-Token': 'AUTH_t'}) - req.remote_user = 'act:usr' - self.test_auth.authorize(req) - self.assertEquals(owner_values, [False]) - - def test_no_memcache(self): - env = {'swift.cache': None} - try: - self.test_auth.get_groups(env, None) - except Exception as e: - self.assertTrue(e.args[0].startswith("Memcache required")) - - def test_handle_request(self): - req = self._make_request('/auth/v1.0') - resp = self.test_auth.handle_request(req) - self.assertEquals(resp.status_int, REDIRECT_STATUS) - - def test_handle_request_bad_request(self): - req = self._make_request('////') - resp = self.test_auth.handle_request(req) - self.assertEquals(resp.status_int, 404) - - def test_handle_request_no_handler(self): - req = self._make_request('/blah/blah/blah/blah') - resp = self.test_auth.handle_request(req) - self.assertEquals(resp.status_int, 400) - - def test_handle_get_token_bad_request(self): - req = self._make_request('/blah/blah') - resp = self.test_auth.handle_get_token(req) - self.assertEquals(resp.status_int, 400) - req = self._make_request('/////') - resp = self.test_auth.handle_get_token(req) - self.assertEquals(resp.status_int, 404) - - def test_passive_handle_get_token_no_user_or_key(self): - #No user and key - req = self._make_request('/auth/v1.0') - resp = self.test_auth_passive.handle_get_token(req) - self.assertEquals(resp.status_int, REDIRECT_STATUS) - #User given but no key - req = self._make_request('/auth/v1.0', - headers={'X-Auth-User': 'test:user'}) - resp = self.test_auth_passive.handle_get_token(req) - self.assertEquals(resp.status_int, 401) - - def test_passive_handle_get_token_account_in_req_path(self): - req = self._make_request('/v1/test/auth', - headers={'X-Auth-User': 'test:user', - 'X-Auth-Key': 'password'}) - _mock_run_kinit = Mock(return_value=0) - _mock_get_groups = Mock(return_value="user,auth_test") - with patch('gluster.swift.common.middleware.swiftkerbauth.kerbauth.run_kinit', _mock_run_kinit): - with patch('gluster.swift.common.middleware.swiftkerbauth.kerbauth.get_groups_from_username', - _mock_get_groups): - resp = self.test_auth_passive.handle_get_token(req) - _mock_run_kinit.assert_called_once_with('user', 'password') - self.assertEquals(_mock_get_groups.call_count, 2) - self.assertEquals(resp.status_int, 200) - self.assertTrue(resp.headers['X-Auth-Token'] is not None) - self.assertTrue(resp.headers['X-Storage-Token'] is not None) - self.assertTrue(resp.headers['X-Storage-Url'] is not None) - - def test_passive_handle_get_token_user_invalid_or_no__account(self): - #X-Auth-User not in acc:user format - req = self._make_request('/auth/v1.0', - headers={'X-Auth-User': 'user'}) - resp = self.test_auth_passive.handle_get_token(req) - self.assertEquals(resp.status_int, 401) - req = self._make_request('/v1/test/auth', - headers={'X-Auth-User': 'user'}) - resp = self.test_auth_passive.handle_get_token(req) - self.assertEquals(resp.status_int, 401) - # Account name mismatch - req = self._make_request('/v1/test/auth', - headers={'X-Auth-User': 'wrongacc:user'}) - resp = self.test_auth_passive.handle_get_token(req) - self.assertEquals(resp.status_int, 401) - - def test_passive_handle_get_token_no_kinit(self): - req = self._make_request('/auth/v1.0', - headers={'X-Auth-User': 'test:user', - 'X-Auth-Key': 'password'}) - _mock_run_kinit = Mock(side_effect=OSError(errno.ENOENT, - os.strerror(errno.ENOENT))) - with patch('gluster.swift.common.middleware.swiftkerbauth.kerbauth.run_kinit', _mock_run_kinit): - resp = self.test_auth_passive.handle_get_token(req) - self.assertEquals(resp.status_int, 500) - self.assertTrue("kinit command not found" in resp.body) - _mock_run_kinit.assert_called_once_with('user', 'password') - - def test_passive_handle_get_token_kinit_fail(self): - req = self._make_request('/auth/v1.0', - headers={'X-Auth-User': 'test:user', - 'X-Auth-Key': 'password'}) - _mock_run_kinit = Mock(return_value=1) - with patch('gluster.swift.common.middleware.swiftkerbauth.kerbauth.run_kinit', _mock_run_kinit): - resp = self.test_auth_passive.handle_get_token(req) - self.assertEquals(resp.status_int, 401) - _mock_run_kinit.assert_called_once_with('user', 'password') - - def test_passive_handle_get_token_kinit_success_token_not_present(self): - req = self._make_request('/auth/v1.0', - headers={'X-Auth-User': 'test:user', - 'X-Auth-Key': 'password'}) - _mock_run_kinit = Mock(return_value=0) - _mock_get_groups = Mock(return_value="user,auth_test") - with patch('gluster.swift.common.middleware.swiftkerbauth.kerbauth.run_kinit', _mock_run_kinit): - with patch('gluster.swift.common.middleware.swiftkerbauth.kerbauth.get_groups_from_username', - _mock_get_groups): - resp = self.test_auth_passive.handle_get_token(req) - _mock_run_kinit.assert_called_once_with('user', 'password') - self.assertEquals(_mock_get_groups.call_count, 2) - self.assertEquals(resp.status_int, 200) - self.assertTrue(resp.headers['X-Auth-Token'] is not None) - self.assertTrue(resp.headers['X-Storage-Token'] is not None) - self.assertTrue(resp.headers['X-Storage-Url'] is not None) - - def test_passive_handle_get_token_kinit_realm_and_memcache(self): - req = self._make_request('/auth/v1.0', - headers={'X-Auth-User': 'test:user', - 'X-Auth-Key': 'password'}) - req.environ['swift.cache'] = None - _auth_passive = \ - auth.filter_factory({'auth_method': 'passive', - 'realm_name': 'EXAMPLE.COM'})(FakeApp()) - _mock_run_kinit = Mock(return_value=0) - _mock_get_groups = Mock(return_value="user,auth_test") - with patch('gluster.swift.common.middleware.swiftkerbauth.kerbauth.run_kinit', _mock_run_kinit): - with patch('gluster.swift.common.middleware.swiftkerbauth.kerbauth.get_groups_from_username', - _mock_get_groups): - try: - _auth_passive.handle_get_token(req) - except Exception as e: - self.assertTrue(e.args[0].startswith("Memcache " - "required")) - else: - self.fail("Expected Exception - Memcache required") - _mock_run_kinit.assert_called_once_with('user@EXAMPLE.COM', 'password') - _mock_get_groups.assert_called_once_with('user') - - def test_passive_handle_get_token_user_in_any__account(self): - req = self._make_request('/auth/v1.0', - headers={'X-Auth-User': 'test:user', - 'X-Auth-Key': 'password'}) - _mock_run_kinit = Mock(return_value=0) - _mock_get_groups = Mock(return_value="user,auth_blah") - with patch('gluster.swift.common.middleware.swiftkerbauth.kerbauth.run_kinit', _mock_run_kinit): - with patch('gluster.swift.common.middleware.swiftkerbauth.kerbauth.get_groups_from_username', - _mock_get_groups): - resp = self.test_auth_passive.handle_get_token(req) - self.assertEquals(resp.status_int, 401) - _mock_run_kinit.assert_called_once_with('user', 'password') - _mock_get_groups.assert_called_once_with('user') - - def test_handle(self): - req = self._make_request('/auth/v1.0') - resp = req.get_response(self.test_auth) - self.assertEquals(resp.status_int, REDIRECT_STATUS) - - def test_authorize_invalid_req(self): - req = self._make_request('/') - resp = self.test_auth.authorize(req) - self.assertEquals(resp.status_int, 404) - - def test_authorize_set_swift_owner(self): - req = self._make_request('/v1/AUTH_test/c1/o1') - req.remote_user = 'test,auth_reseller_admin' - resp = self.test_auth.authorize(req) - self.assertEquals(req.environ['swift_owner'], True) - self.assertTrue(resp is None) - req = self._make_request('/v1/AUTH_test/c1/o1') - req.remote_user = 'test,auth_test' - resp = self.test_auth.authorize(req) - self.assertEquals(req.environ['swift_owner'], True) - self.assertTrue(resp is None) - - def test_authorize_swift_sync_key(self): - req = self._make_request( - '/v1/AUTH_cfa/c/o', - environ={'swift_sync_key': 'secret'}, - headers={'x-container-sync-key': 'secret', - 'x-timestamp': '123.456'}) - resp = self.test_auth.authorize(req) - self.assertTrue(resp is None) - - def test_authorize_acl_referrer_access(self): - req = self._make_request('/v1/AUTH_cfa/c') - req.remote_user = 'act:usr,act' - resp = self.test_auth.authorize(req) - self.assertEquals(resp.status_int, 403) - req = self._make_request('/v1/AUTH_cfa/c') - req.remote_user = 'act:usr,act' - req.acl = '.r:*,.rlistings' - self.assertEquals(self.test_auth.authorize(req), None) - req = self._make_request('/v1/AUTH_cfa/c') - req.remote_user = 'act:usr,act' - req.acl = '.r:*' # No listings allowed - resp = self.test_auth.authorize(req) - self.assertEquals(resp.status_int, 403) - req = self._make_request('/v1/AUTH_cfa/c') - req.remote_user = 'act:usr,act' - req.acl = '.r:.example.com,.rlistings' - resp = self.test_auth.authorize(req) - self.assertEquals(resp.status_int, 403) - req = self._make_request('/v1/AUTH_cfa/c') - req.remote_user = 'act:usr,act' - req.referer = 'http://www.example.com/index.html' - req.acl = '.r:.example.com,.rlistings' - self.assertEquals(self.test_auth.authorize(req), None) - req = self._make_request('/v1/AUTH_cfa/c') - resp = self.test_auth.authorize(req) - self.assertEquals(resp.status_int, REDIRECT_STATUS) - req = self._make_request('/v1/AUTH_cfa/c') - req.acl = '.r:*,.rlistings' - self.assertEquals(self.test_auth.authorize(req), None) - req = self._make_request('/v1/AUTH_cfa/c') - req.acl = '.r:*' # No listings allowed - resp = self.test_auth.authorize(req) - self.assertEquals(resp.status_int, REDIRECT_STATUS) - req = self._make_request('/v1/AUTH_cfa/c') - req.acl = '.r:.example.com,.rlistings' - resp = self.test_auth.authorize(req) - self.assertEquals(resp.status_int, REDIRECT_STATUS) - req = self._make_request('/v1/AUTH_cfa/c') - req.referer = 'http://www.example.com/index.html' - req.acl = '.r:.example.com,.rlistings' - self.assertEquals(self.test_auth.authorize(req), None) - - def test_handle_x_storage_token(self): - req = self._make_request( - '/auth/v1.0', - headers={'x-storage-token': 'blahblah', }) - resp = req.get_response(self.test_auth) - self.assertEquals(resp.status_int, REDIRECT_STATUS) - - def test_invalid_token(self): - req = self._make_request('/k1/test') - req.environ['HTTP_X_AUTH_TOKEN'] = 'AUTH_blahblahblah' - resp = req.get_response(self.test_auth) - self.assertEquals(resp.status_int, REDIRECT_STATUS) - -if __name__ == '__main__': - unittest.main() diff --git a/test/unit/common/middleware/swiftkerbauth/test_kerbauth_utils.py b/test/unit/common/middleware/swiftkerbauth/test_kerbauth_utils.py deleted file mode 100644 index 2a4e90b..0000000 --- a/test/unit/common/middleware/swiftkerbauth/test_kerbauth_utils.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright (c) 2013 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest -import re -from time import time -from test.unit import FakeMemcache -from gluster.swift.common.middleware.swiftkerbauth import kerbauth_utils as ku - - -class TestKerbUtils(unittest.TestCase): - - def test_get_remote_user(self): - env = {'REMOTE_USER': "auth_admin@EXAMPLE.COM"} - result = ku.get_remote_user(env) - self.assertEqual(result, "auth_admin") - - def test_get_remote_user_err(self): - env = {'REMOTE_USER': "auth_admin"} - try: - ku.get_remote_user(env) - except RuntimeError as err: - self.assertTrue(err.args[0].startswith("Malformed REMOTE_USER")) - else: - self.fail("Expected RuntimeError") - - def test_get_auth_data(self): - mc = FakeMemcache() - expiry = time() + 100 - ku.set_auth_data(mc, "root", "AUTH_tk", expiry, "root,admin") - (token, expires, groups) = ku.get_auth_data(mc, "root") - self.assertEqual(("AUTH_tk", expiry, "root,admin"), - (token, expires, groups)) - - def test_get_auth_data_err(self): - mc = FakeMemcache() - (token, expires, groups) = ku.get_auth_data(mc, "root") - self.assertEqual((token, expires, groups), (None, None, None)) - - expiry = time() - 1 - ku.set_auth_data(mc, "root", "AUTH_tk", expiry, "root,admin") - (token, expires, groups) = ku.get_auth_data(mc, "root") - self.assertEqual((token, expires, groups), (None, None, None)) - - def test_set_auth_data(self): - mc = FakeMemcache() - expiry = time() + 100 - ku.set_auth_data(mc, "root", "AUTH_tk", expiry, "root,admin") - - def test_generate_token(self): - token = ku.generate_token() - matches = re.match('AUTH_tk[a-f0-9]{32}', token) - self.assertTrue(matches is not None) - - def test_get_groups_from_username(self): - groups = ku.get_groups_from_username("root") - self.assertTrue("root" in groups) - - def test_get_groups_from_username_err(self): - try: - ku.get_groups_from_username("Zroot") - except RuntimeError as err: - self.assertTrue(err.args[0].startswith("Failure running id -G")) - else: - self.fail("Expected RuntimeError") diff --git a/test/unit/common/test_constraints.py b/test/unit/common/test_constraints.py index 6c78d75..e8ddd69 100644 --- a/test/unit/common/test_constraints.py +++ b/test/unit/common/test_constraints.py @@ -57,10 +57,10 @@ class TestConstraints(unittest.TestCase): cnt.set_object_name_component_length() self.assertEqual(len, cnt.get_object_name_component_length()) - with patch('swift.common.constraints.constraints_conf_int', - mock_constraints_conf_int): - cnt.set_object_name_component_length() - self.assertEqual(cnt.get_object_name_component_length(), 1000) + with patch('swift.common.constraints.constraints_conf_int', + mock_constraints_conf_int): + cnt.set_object_name_component_length() + self.assertEqual(cnt.get_object_name_component_length(), 1000) def test_validate_obj_name_component(self): max_obj_len = cnt.get_object_name_component_length() @@ -75,65 +75,6 @@ class TestConstraints(unittest.TestCase): self.assertTrue(cnt.validate_obj_name_component('..')) self.assertTrue(cnt.validate_obj_name_component('')) - def test_validate_headers(self): - req = Mock() - req.headers = [] - self.assertEqual(cnt.validate_headers(req), '') - req.headers = ['x-some-header'] - self.assertEqual(cnt.validate_headers(req), '') - #TODO: Although we now support x-delete-at and x-delete-after, - #retained this test case as we may add some other header to - #unsupported list in future - raise SkipTest - req.headers = ['x-delete-at', 'x-some-header'] - self.assertNotEqual(cnt.validate_headers(req), '') - req.headers = ['x-delete-after', 'x-some-header'] - self.assertNotEqual(cnt.validate_headers(req), '') - req.headers = ['x-delete-at', 'x-delete-after', 'x-some-header'] - self.assertNotEqual(cnt.validate_headers(req), '') - - def test_validate_headers_ignoring_config_set(self): - with patch('gluster.swift.common.constraints.' - 'Glusterfs._ignore_unsupported_headers', True): - req = Mock() - req.headers = [] - self.assertEqual(cnt.validate_headers(req), '') - req.headers = ['x-some-header'] - self.assertEqual(cnt.validate_headers(req), '') - #TODO: Although we now support x-delete-at and x-delete-after, - #retained this test case as we may add some other header to - #unsupported list in future - raise SkipTest - req.headers = ['x-delete-at', 'x-some-header'] - self.assertEqual(cnt.validate_headers(req), '') - req.headers = ['x-delete-after', 'x-some-header'] - self.assertEqual(cnt.validate_headers(req), '') - req.headers = ['x-delete-at', 'x-delete-after', 'x-some-header'] - self.assertEqual(cnt.validate_headers(req), '') - - def test_gluster_check_metadata(self): - mock_check_metadata = Mock() - with patch('gluster.swift.common.constraints.__check_metadata', - mock_check_metadata): - req = Mock() - req.headers = [] - cnt.gluster_check_metadata(req, 'object') - self.assertTrue(1, mock_check_metadata.call_count) - cnt.gluster_check_metadata(req, 'object', POST=False) - self.assertTrue(1, mock_check_metadata.call_count) - req.headers = ['x-some-header'] - self.assertEqual(cnt.gluster_check_metadata(req, 'object', POST=False), None) - #TODO: Although we now support x-delete-at and x-delete-after, - #retained this test case as we may add some other header to - #unsupported list in future - raise SkipTest - req.headers = ['x-delete-at', 'x-some-header'] - self.assertNotEqual(cnt.gluster_check_metadata(req, 'object', POST=False), None) - req.headers = ['x-delete-after', 'x-some-header'] - self.assertNotEqual(cnt.gluster_check_metadata(req, 'object', POST=False), None) - req.headers = ['x-delete-at', 'x-delete-after', 'x-some-header'] - self.assertNotEqual(cnt.gluster_check_metadata(req, 'object', POST=False), None) - def test_gluster_check_object_creation(self): with patch('gluster.swift.common.constraints.__check_object_creation', mock_check_object_creation): @@ -147,9 +88,3 @@ class TestConstraints(unittest.TestCase): req = Mock() req.headers = [] self.assertTrue(cnt.gluster_check_object_creation(req, 'dir/.')) - #TODO: Although we now support x-delete-at and x-delete-after, - #retained this test case as we may add some other header to - #unsupported list in future - raise SkipTest - req.headers = ['x-delete-at'] - self.assertTrue(cnt.gluster_check_object_creation(req, 'dir/z')) diff --git a/test/unit/common/test_diskdir.py b/test/unit/common/test_diskdir.py index f32c3ad..ebc554a 100644 --- a/test/unit/common/test_diskdir.py +++ b/test/unit/common/test_diskdir.py @@ -408,7 +408,7 @@ class TestContainerBroker(unittest.TestCase): self.assertEqual(os.path.basename(broker.db_file), 'db_file.db') broker.initialize(self.initial_ts) self.assertTrue(os.path.isdir(self.container)) - self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP]) + self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP][0]) self.assertFalse(broker.is_deleted()) def test_creation_existing(self): @@ -419,7 +419,7 @@ class TestContainerBroker(unittest.TestCase): self.assertEqual(os.path.basename(broker.db_file), 'db_file.db') broker.initialize(self.initial_ts) self.assertTrue(os.path.isdir(self.container)) - self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP]) + self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP][0]) self.assertFalse(broker.is_deleted()) def test_creation_existing_bad_metadata(self): @@ -432,7 +432,7 @@ class TestContainerBroker(unittest.TestCase): self.assertEqual(os.path.basename(broker.db_file), 'db_file.db') broker.initialize(self.initial_ts) self.assertTrue(os.path.isdir(self.container)) - self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP]) + self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP][0]) self.assertFalse(broker.is_deleted()) def test_empty(self): @@ -954,7 +954,7 @@ class TestContainerBroker(unittest.TestCase): self.assertEqual(os.path.basename(broker.db_file), 'db_file.db') broker.initialize(self.initial_ts) self.assertTrue(os.path.isdir(self.container)) - self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP]) + self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP][0]) self.assertFalse(broker.is_deleted()) broker.delete_db(normalize_timestamp(time())) self.assertTrue(broker.is_deleted()) @@ -1005,7 +1005,7 @@ class TestAccountBroker(unittest.TestCase): self.assertEqual(os.path.basename(broker.db_file), 'db_file.db') broker.initialize(self.initial_ts) self.assertTrue(os.path.isdir(self.drive_fullpath)) - self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP]) + self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP][0]) self.assertFalse(broker.is_deleted()) def test_creation_bad_metadata(self): @@ -1016,7 +1016,7 @@ class TestAccountBroker(unittest.TestCase): self.assertEqual(os.path.basename(broker.db_file), 'db_file.db') broker.initialize(self.initial_ts) self.assertTrue(os.path.isdir(self.drive_fullpath)) - self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP]) + self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP][0]) self.assertFalse(broker.is_deleted()) def test_empty(self): @@ -1219,7 +1219,7 @@ class TestAccountBroker(unittest.TestCase): self.assertEqual(os.path.basename(broker.db_file), 'db_file.db') broker.initialize(self.initial_ts) self.assertTrue(os.path.isdir(self.drive_fullpath)) - self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP]) + self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP][0]) self.assertFalse(broker.is_deleted()) broker.delete_db(normalize_timestamp(time())) # Deleting the "db" should be a NOOP diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index dd03bd8..55bd0cf 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -25,9 +25,10 @@ import hashlib import tarfile import shutil from collections import defaultdict -from mock import patch +from mock import patch, Mock from gluster.swift.common import utils, Glusterfs -from gluster.swift.common.exceptions import GlusterFileSystemOSError +from gluster.swift.common.exceptions import GlusterFileSystemOSError,\ + GlusterFileSystemIOError from swift.common.exceptions import DiskFileNoSpace # @@ -712,6 +713,18 @@ class TestUtils(unittest.TestCase): ret = utils.validate_object(md) assert ret + def test_validate_object_with_stat(self): + md = {utils.X_TIMESTAMP: 'na', + utils.X_CONTENT_TYPE: 'na', + utils.X_ETAG: 'bad', + utils.X_CONTENT_LENGTH: '12345', + utils.X_TYPE: utils.OBJECT, + utils.X_OBJECT_TYPE: 'na'} + fake_stat = Mock(st_size=12346) + self.assertFalse(utils.validate_object(md, fake_stat)) + fake_stat = Mock(st_size=12345) + self.assertTrue(utils.validate_object(md, fake_stat)) + class TestUtilsDirObjects(unittest.TestCase): @@ -794,7 +807,8 @@ class TestUtilsDirObjects(unittest.TestCase): def _mock_rm(path): print "_mock_rm-metadata_enoent(%s)" % path shutil.rmtree(path) - raise OSError(errno.ENOENT, os.strerror(errno.ENOENT)) + raise GlusterFileSystemIOError(errno.ENOENT, + os.strerror(errno.ENOENT)) # Remove the files for f in self.files: @@ -805,8 +819,8 @@ class TestUtilsDirObjects(unittest.TestCase): try: try: self.assertTrue(utils.rmobjdir(self.rootdir)) - except OSError: - self.fail("Unexpected OSError") + except IOError: + self.fail("Unexpected IOError") else: pass finally: diff --git a/test/unit/obj/test_diskfile.py b/test/unit/obj/test_diskfile.py index f8c26db..1fe0904 100644 --- a/test/unit/obj/test_diskfile.py +++ b/test/unit/obj/test_diskfile.py @@ -223,7 +223,7 @@ class TestDiskFile(unittest.TestCase): ini_md = { 'X-Type': 'Object', 'X-Object-Type': 'file', - 'Content-Length': 5, + 'Content-Length': 4, 'ETag': 'etag', 'X-Timestamp': 'ts', 'Content-Type': 'application/loctet-stream'} @@ -283,9 +283,8 @@ class TestDiskFile(unittest.TestCase): with gdf.open(): assert gdf._is_dir assert gdf._data_file == the_dir - assert gdf._metadata == exp_md - def _create_and_get_diskfile(self, dev, par, acc, con, obj): + def _create_and_get_diskfile(self, dev, par, acc, con, obj, fsize=256): # FIXME: assumes account === volume the_path = os.path.join(self.td, dev, con) the_file = os.path.join(the_path, obj) @@ -293,7 +292,7 @@ class TestDiskFile(unittest.TestCase): base_dir = os.path.dirname(the_file) os.makedirs(base_dir) with open(the_file, "wb") as fd: - fd.write("y" * 256) + fd.write("y" * fsize) gdf = self._get_diskfile(dev, par, acc, con, obj) assert gdf._obj == base_obj assert not gdf._is_dir @@ -353,6 +352,26 @@ class TestDiskFile(unittest.TestCase): assert len(chunks) == 1, repr(chunks) assert called[0] == 1, called + def test_reader_larger_file(self): + closed = [False] + fd = [-1] + + def mock_close(*args, **kwargs): + closed[0] = True + os.close(fd[0]) + + with mock.patch("gluster.swift.obj.diskfile.do_close", mock_close): + gdf = self._create_and_get_diskfile("vol0", "p57", "ufo47", "bar", "z", fsize=1024*1024*2) + with gdf.open(): + assert gdf._fd is not None + assert gdf._data_file == os.path.join(self.td, "vol0", "bar", "z") + reader = gdf.reader() + assert reader._fd is not None + fd[0] = reader._fd + chunks = [ck for ck in reader] + assert reader._fd is None + assert closed[0] + def test_reader_dir_object(self): called = [False] @@ -926,3 +945,31 @@ class TestDiskFile(unittest.TestCase): dw.write("123") os.unlink(saved_tmppath) assert not os.path.exists(saved_tmppath) + + def test_unlink_not_called_after_rename(self): + the_obj_path = os.path.join("b", "a") + the_file = os.path.join(the_obj_path, "z") + gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", the_file) + + body = '1234\n' + etag = md5(body).hexdigest() + metadata = { + 'X-Timestamp': '1234', + 'Content-Type': 'file', + 'ETag': etag, + 'Content-Length': '5', + } + + _mock_do_unlink = Mock() # Shouldn't be called + with patch("gluster.swift.obj.diskfile.do_unlink", _mock_do_unlink): + with gdf.create() as dw: + assert dw._tmppath is not None + tmppath = dw._tmppath + dw.write(body) + dw.put(metadata) + # do_unlink is not called if dw._tmppath is set to None + assert dw._tmppath is None + self.assertFalse(_mock_do_unlink.called) + + assert os.path.exists(gdf._data_file) # Real file exists + assert not os.path.exists(tmppath) # Temp file does not exist diff --git a/test/unit/obj/test_expirer.py b/test/unit/obj/test_expirer.py index 4329eef..236775e 100644 --- a/test/unit/obj/test_expirer.py +++ b/test/unit/obj/test_expirer.py @@ -46,7 +46,7 @@ class TestObjectExpirer(TestCase): self.old_loadapp = internal_client.loadapp self.old_sleep = internal_client.sleep - internal_client.loadapp = lambda x: None + internal_client.loadapp = lambda *a, **kw: None internal_client.sleep = not_sleep def teardown(self): @@ -618,7 +618,7 @@ class TestObjectExpirer(TestCase): start_response('204 No Content', [('Content-Length', '0')]) return [] - internal_client.loadapp = lambda x: fake_app + internal_client.loadapp = lambda *a, **kw: fake_app x = expirer.ObjectExpirer({}) ts = '1234' @@ -635,7 +635,7 @@ class TestObjectExpirer(TestCase): start_response('204 No Content', [('Content-Length', '0')]) return [] - internal_client.loadapp = lambda x: fake_app + internal_client.loadapp = lambda *a, **kw: fake_app x = expirer.ObjectExpirer({}) ts = '1234' @@ -649,7 +649,7 @@ class TestObjectExpirer(TestCase): start_response('404 Not Found', [('Content-Length', '0')]) return [] - internal_client.loadapp = lambda x: fake_app + internal_client.loadapp = lambda *a, **kw: fake_app x = expirer.ObjectExpirer({}) x.delete_actual_object('/path/to/object', '1234') @@ -661,7 +661,7 @@ class TestObjectExpirer(TestCase): [('Content-Length', '0')]) return [] - internal_client.loadapp = lambda x: fake_app + internal_client.loadapp = lambda *a, **kw: fake_app x = expirer.ObjectExpirer({}) x.delete_actual_object('/path/to/object', '1234') @@ -674,7 +674,7 @@ class TestObjectExpirer(TestCase): [('Content-Length', '0')]) return [] - internal_client.loadapp = lambda x: fake_app + internal_client.loadapp = lambda *a, **kw: fake_app x = expirer.ObjectExpirer({}) exc = None @@ -692,7 +692,7 @@ class TestObjectExpirer(TestCase): x = expirer.ObjectExpirer({}) x.swift.make_request = mock.MagicMock() x.delete_actual_object(name, timestamp) - x.swift.make_request.assert_called_once() + self.assertTrue(x.swift.make_request.called) self.assertEqual(x.swift.make_request.call_args[0][1], '/v1/' + urllib.quote(name)) diff --git a/test/unit/proxy/controllers/test_obj.py b/test/unit/proxy/controllers/test_obj.py index aada616..4942691 100755 --- a/test/unit/proxy/controllers/test_obj.py +++ b/test/unit/proxy/controllers/test_obj.py @@ -22,7 +22,7 @@ import mock import swift from swift.proxy import server as proxy_server from swift.common.swob import HTTPException -from test.unit import FakeRing, FakeMemcache, fake_http_connect +from test.unit import FakeRing, FakeMemcache, fake_http_connect, debug_logger @contextmanager @@ -90,26 +90,96 @@ class TestObjControllerWriteAffinity(unittest.TestCase): class TestObjController(unittest.TestCase): + def setUp(self): + logger = debug_logger('proxy-server') + logger.thread_locals = ('txn1', '127.0.0.2') + self.app = proxy_server.Application( + None, FakeMemcache(), account_ring=FakeRing(), + container_ring=FakeRing(), object_ring=FakeRing(), + logger=logger) + self.controller = proxy_server.ObjectController(self.app, + 'a', 'c', 'o') + self.controller.container_info = mock.MagicMock(return_value={ + 'partition': 1, + 'nodes': [ + {'ip': '127.0.0.1', 'port': '1', 'device': 'sda'}, + {'ip': '127.0.0.1', 'port': '2', 'device': 'sda'}, + {'ip': '127.0.0.1', 'port': '3', 'device': 'sda'}, + ], + 'write_acl': None, + 'read_acl': None, + 'sync_key': None, + 'versions': None}) + + def test_PUT_simple(self): + req = swift.common.swob.Request.blank('/v1/a/c/o') + req.headers['content-length'] = '0' + with set_http_connect(201, 201, 201): + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 201) + + def test_PUT_if_none_match(self): + req = swift.common.swob.Request.blank('/v1/a/c/o') + req.headers['if-none-match'] = '*' + req.headers['content-length'] = '0' + with set_http_connect(201, 201, 201): + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 201) + + def test_PUT_if_none_match_denied(self): + req = swift.common.swob.Request.blank('/v1/a/c/o') + req.headers['if-none-match'] = '*' + req.headers['content-length'] = '0' + with set_http_connect(201, (412, 412), 201): + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 412) + + def test_PUT_if_none_match_not_star(self): + req = swift.common.swob.Request.blank('/v1/a/c/o') + req.headers['if-none-match'] = 'somethingelse' + req.headers['content-length'] = '0' + with set_http_connect(201, 201, 201): + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 400) + + def test_GET_simple(self): + req = swift.common.swob.Request.blank('/v1/a/c/o') + with set_http_connect(200): + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 200) + + def test_DELETE_simple(self): + req = swift.common.swob.Request.blank('/v1/a/c/o') + with set_http_connect(204, 204, 204): + resp = self.controller.DELETE(req) + self.assertEquals(resp.status_int, 204) + + def test_POST_simple(self): + req = swift.common.swob.Request.blank('/v1/a/c/o') + with set_http_connect(200, 200, 200, 201, 201, 201): + resp = self.controller.POST(req) + self.assertEquals(resp.status_int, 202) + + def test_COPY_simple(self): + req = swift.common.swob.Request.blank('/v1/a/c/o') + with set_http_connect(200, 200, 200, 201, 201, 201): + resp = self.controller.POST(req) + self.assertEquals(resp.status_int, 202) + + def test_HEAD_simple(self): + req = swift.common.swob.Request.blank('/v1/a/c/o') + with set_http_connect(200, 200, 200, 201, 201, 201): + resp = self.controller.POST(req) + self.assertEquals(resp.status_int, 202) def test_PUT_log_info(self): # mock out enough to get to the area of the code we want to test with mock.patch('swift.proxy.controllers.obj.check_object_creation', mock.MagicMock(return_value=None)): - app = mock.MagicMock() - app.container_ring.get_nodes.return_value = (1, [2]) - app.object_ring.get_nodes.return_value = (1, [2]) - controller = proxy_server.ObjectController(app, 'a', 'c', 'o') - controller.container_info = mock.MagicMock(return_value={ - 'partition': 1, - 'nodes': [{}], - 'write_acl': None, - 'sync_key': None, - 'versions': None}) - # and now test that we add the header to log_info req = swift.common.swob.Request.blank('/v1/a/c/o') req.headers['x-copy-from'] = 'somewhere' try: - controller.PUT(req) + self.controller.PUT(req) except HTTPException: pass self.assertEquals( @@ -119,7 +189,7 @@ class TestObjController(unittest.TestCase): req.method = 'POST' req.headers['x-copy-from'] = 'elsewhere' try: - controller.PUT(req) + self.controller.PUT(req) except HTTPException: pass self.assertEquals(req.environ.get('swift.log_info'), None) diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 0cb2278..1a59016 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -30,6 +30,7 @@ from urllib import quote from hashlib import md5 from tempfile import mkdtemp import weakref +import re import mock from eventlet import sleep, spawn, wsgi, listen @@ -60,7 +61,8 @@ from swift.proxy.controllers.base import get_container_memcache_key, \ get_account_memcache_key, cors_validation import swift.proxy.controllers from swift.common.request_helpers import get_sys_meta_prefix -from swift.common.swob import Request, Response, HTTPUnauthorized +from swift.common.swob import Request, Response, HTTPUnauthorized, \ + HTTPException # mocks logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) @@ -245,6 +247,7 @@ def set_http_connect(*args, **kwargs): swift.proxy.controllers.obj.http_connect = new_connect swift.proxy.controllers.account.http_connect = new_connect swift.proxy.controllers.container.http_connect = new_connect + return new_connect # tests @@ -692,10 +695,10 @@ class TestObjectController(unittest.TestCase): def setUp(self): self.app = proxy_server.Application(None, FakeMemcache(), + logger=debug_logger('proxy-ut'), account_ring=FakeRing(), container_ring=FakeRing(), object_ring=FakeRing()) - monkey_patch_mimetools() def tearDown(self): self.app.account_ring.set_replicas(3) @@ -802,6 +805,7 @@ class TestObjectController(unittest.TestCase): self.app.update_request(req) self.app.memcache.store = {} res = controller.PUT(req) + self.assertEqual(test_errors, []) self.assertTrue(res.status.startswith('201 ')) def test_PUT_respects_write_affinity(self): @@ -1609,7 +1613,7 @@ class TestObjectController(unittest.TestCase): dev['ip'] = '127.0.0.1' dev['port'] = 1 - class SlowBody(): + class SlowBody(object): def __init__(self): self.sent = 0 @@ -1658,7 +1662,7 @@ class TestObjectController(unittest.TestCase): dev['ip'] = '127.0.0.1' dev['port'] = 1 - class SlowBody(): + class SlowBody(object): def __init__(self): self.sent = 0 @@ -1693,7 +1697,7 @@ class TestObjectController(unittest.TestCase): dev['port'] = 1 req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) self.app.update_request(req) - set_http_connect(200, 200, 200, slow=True) + set_http_connect(200, 200, 200, slow=0.1) req.sent_size = 0 resp = req.get_response(self.app) got_exc = False @@ -1703,7 +1707,7 @@ class TestObjectController(unittest.TestCase): got_exc = True self.assert_(not got_exc) self.app.recoverable_node_timeout = 0.1 - set_http_connect(200, 200, 200, slow=True) + set_http_connect(200, 200, 200, slow=1.0) resp = req.get_response(self.app) got_exc = False try: @@ -1718,16 +1722,17 @@ class TestObjectController(unittest.TestCase): self.app.update_request(req) self.app.recoverable_node_timeout = 0.1 - set_http_connect(200, 200, 200, slow=[3]) + set_http_connect(200, 200, 200, slow=[1.0, 1.0, 1.0]) resp = req.get_response(self.app) got_exc = False try: - resp.body + self.assertEquals('', resp.body) except ChunkReadTimeout: got_exc = True self.assert_(got_exc) - set_http_connect(200, 200, 200, body='lalala', slow=[2]) + set_http_connect(200, 200, 200, body='lalala', + slow=[1.0, 1.0]) resp = req.get_response(self.app) got_exc = False try: @@ -1736,8 +1741,8 @@ class TestObjectController(unittest.TestCase): got_exc = True self.assert_(not got_exc) - set_http_connect(200, 200, 200, body='lalala', slow=[2], - etags=['a', 'a', 'a']) + set_http_connect(200, 200, 200, body='lalala', + slow=[1.0, 1.0], etags=['a', 'a', 'a']) resp = req.get_response(self.app) got_exc = False try: @@ -1746,8 +1751,8 @@ class TestObjectController(unittest.TestCase): got_exc = True self.assert_(not got_exc) - set_http_connect(200, 200, 200, body='lalala', slow=[2], - etags=['a', 'b', 'a']) + set_http_connect(200, 200, 200, body='lalala', + slow=[1.0, 1.0], etags=['a', 'b', 'a']) resp = req.get_response(self.app) got_exc = False try: @@ -1757,8 +1762,8 @@ class TestObjectController(unittest.TestCase): self.assert_(not got_exc) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) - set_http_connect(200, 200, 200, body='lalala', slow=[2], - etags=['a', 'b', 'b']) + set_http_connect(200, 200, 200, body='lalala', + slow=[1.0, 1.0], etags=['a', 'b', 'b']) resp = req.get_response(self.app) got_exc = False try: @@ -1787,17 +1792,17 @@ class TestObjectController(unittest.TestCase): 'Content-Type': 'text/plain'}, body=' ') self.app.update_request(req) - set_http_connect(200, 200, 201, 201, 201, slow=True) + set_http_connect(200, 200, 201, 201, 201, slow=0.1) resp = req.get_response(self.app) self.assertEquals(resp.status_int, 201) self.app.node_timeout = 0.1 - set_http_connect(201, 201, 201, slow=True) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '4', 'Content-Type': 'text/plain'}, body=' ') self.app.update_request(req) + set_http_connect(201, 201, 201, slow=1.0) resp = req.get_response(self.app) self.assertEquals(resp.status_int, 503) @@ -2227,307 +2232,322 @@ class TestObjectController(unittest.TestCase): resp = controller.PUT(req) self.assertEquals(resp.status_int, 400) - def test_copy_from(self): + @contextmanager + def controller_context(self, req, *args, **kwargs): + _v, account, container, obj = utils.split_path(req.path, 4, 4, True) + controller = proxy_server.ObjectController(self.app, account, + container, obj) + self.app.update_request(req) + self.app.memcache.store = {} with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') - # initial source object PUT - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0'}) - self.app.update_request(req) - set_http_connect(200, 200, 201, 201, 201) - # acct cont obj obj obj + new_connect = set_http_connect(*args, **kwargs) + yield controller + unused_status_list = [] + while True: + try: + unused_status_list.append(new_connect.code_iter.next()) + except StopIteration: + break + if unused_status_list: + raise self.fail('UN-USED STATUS CODES: %r' % + unused_status_list) + + def test_basic_put_with_x_copy_from(self): + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': 'c/o'}) + status_list = (200, 200, 200, 200, 200, 201, 201, 201) + # acct cont objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: resp = controller.PUT(req) - self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers['x-copied-from'], 'c/o') - # basic copy - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': 'c/o'}) - self.app.update_request(req) - set_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acct cont objc objc objc obj obj obj - self.app.memcache.store = {} + def test_basic_put_with_x_copy_from_across_container(self): + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': 'c2/o'}) + status_list = (200, 200, 200, 200, 200, 200, 201, 201, 201) + # acct cont conc objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: resp = controller.PUT(req) - self.assertEquals(resp.status_int, 201) - self.assertEquals(resp.headers['x-copied-from'], 'c/o') + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers['x-copied-from'], 'c2/o') - # non-zero content length - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '5', - 'X-Copy-From': 'c/o'}) - self.app.update_request(req) - set_http_connect(200, 200, 200, 200, 200, 200, 200) - # acct cont acct cont objc objc objc - self.app.memcache.store = {} + def test_copy_non_zero_content_length(self): + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '5', + 'X-Copy-From': 'c/o'}) + status_list = (200, 200) + # acct cont + with self.controller_context(req, *status_list) as controller: resp = controller.PUT(req) - self.assertEquals(resp.status_int, 400) + self.assertEquals(resp.status_int, 400) - # extra source path parsing - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': 'c/o/o2'}) - req.account = 'a' - set_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acct cont objc objc objc obj obj obj - self.app.memcache.store = {} + def test_copy_with_slashes_in_x_copy_from(self): + # extra source path parsing + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': 'c/o/o2'}) + status_list = (200, 200, 200, 200, 200, 201, 201, 201) + # acct cont objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: resp = controller.PUT(req) - self.assertEquals(resp.status_int, 201) - self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2') + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2') - # space in soure path - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': 'c/o%20o2'}) - req.account = 'a' - set_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acct cont objc objc objc obj obj obj - self.app.memcache.store = {} + def test_copy_with_spaces_in_x_copy_from(self): + # space in soure path + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': 'c/o%20o2'}) + status_list = (200, 200, 200, 200, 200, 201, 201, 201) + # acct cont objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: resp = controller.PUT(req) - self.assertEquals(resp.status_int, 201) - self.assertEquals(resp.headers['x-copied-from'], 'c/o%20o2') + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers['x-copied-from'], 'c/o%20o2') - # repeat tests with leading / - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o'}) - self.app.update_request(req) - set_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acct cont objc objc objc obj obj obj - self.app.memcache.store = {} + def test_copy_with_leading_slash_in_x_copy_from(self): + # repeat tests with leading / + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o'}) + status_list = (200, 200, 200, 200, 200, 201, 201, 201) + # acct cont objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: resp = controller.PUT(req) - self.assertEquals(resp.status_int, 201) - self.assertEquals(resp.headers['x-copied-from'], 'c/o') + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers['x-copied-from'], 'c/o') - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o/o2'}) - req.account = 'a' - set_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acct cont objc objc objc obj obj obj - self.app.memcache.store = {} + def test_copy_with_leading_slash_and_slashes_in_x_copy_from(self): + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o/o2'}) + status_list = (200, 200, 200, 200, 200, 201, 201, 201) + # acct cont objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: resp = controller.PUT(req) - self.assertEquals(resp.status_int, 201) - self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2') + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2') - # negative tests + def test_copy_with_no_object_in_x_copy_from(self): + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c'}) + status_list = (200, 200) + # acct cont + with self.controller_context(req, *status_list) as controller: + try: + controller.PUT(req) + except HTTPException as resp: + self.assertEquals(resp.status_int // 100, 4) # client error + else: + raise self.fail('Invalid X-Copy-From did not raise ' + 'client error') - # invalid x-copy-from path - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c'}) - self.app.update_request(req) - self.app.memcache.store = {} + def test_copy_server_error_reading_source(self): + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o'}) + status_list = (200, 200, 503, 503, 503) + # acct cont objc objc objc + with self.controller_context(req, *status_list) as controller: resp = controller.PUT(req) - self.assertEquals(resp.status_int // 100, 4) # client error + self.assertEquals(resp.status_int, 503) - # server error - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o'}) - self.app.update_request(req) - set_http_connect(200, 200, 503, 503, 503) - # acct cont objc objc objc - self.app.memcache.store = {} + def test_copy_not_found_reading_source(self): + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o'}) + # not found + status_list = (200, 200, 404, 404, 404) + # acct cont objc objc objc + with self.controller_context(req, *status_list) as controller: resp = controller.PUT(req) - self.assertEquals(resp.status_int, 503) + self.assertEquals(resp.status_int, 404) - # not found - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o'}) - self.app.update_request(req) - set_http_connect(200, 200, 404, 404, 404) - # acct cont objc objc objc - self.app.memcache.store = {} + def test_copy_with_some_missing_sources(self): + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o'}) + status_list = (200, 200, 404, 404, 200, 201, 201, 201) + # acct cont objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: resp = controller.PUT(req) - self.assertEquals(resp.status_int, 404) + self.assertEquals(resp.status_int, 201) - # some missing containers - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o'}) - self.app.update_request(req) - set_http_connect(200, 200, 404, 404, 200, 201, 201, 201) - # acct cont objc objc objc obj obj obj - self.app.memcache.store = {} + def test_copy_with_object_metadata(self): + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o', + 'X-Object-Meta-Ours': 'okay'}) + # test object metadata + status_list = (200, 200, 200, 200, 200, 201, 201, 201) + # acct cont objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: resp = controller.PUT(req) - self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers.get('x-object-meta-test'), 'testing') + self.assertEquals(resp.headers.get('x-object-meta-ours'), 'okay') + self.assertEquals(resp.headers.get('x-delete-at'), '9876543210') - # test object meta data - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o', - 'X-Object-Meta-Ours': 'okay'}) - self.app.update_request(req) - set_http_connect(200, 200, 200, 200, 200, 201, 201, 201) - # acct cont objc objc objc obj obj obj - self.app.memcache.store = {} - resp = controller.PUT(req) - self.assertEquals(resp.status_int, 201) - self.assertEquals(resp.headers.get('x-object-meta-test'), - 'testing') - self.assertEquals(resp.headers.get('x-object-meta-ours'), 'okay') - self.assertEquals(resp.headers.get('x-delete-at'), '9876543210') + def test_copy_source_larger_than_max_file_size(self): + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o'}) - # copy-from object is too large to fit in target object - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o'}) - self.app.update_request(req) + # copy-from object is too large to fit in target object + class LargeResponseBody(object): - class LargeResponseBody(object): + def __len__(self): + return MAX_FILE_SIZE + 1 - def __len__(self): - return MAX_FILE_SIZE + 1 + def __getitem__(self, key): + return '' - def __getitem__(self, key): - return '' + copy_from_obj_body = LargeResponseBody() + status_list = (200, 200, 200, 200, 200) + # acct cont objc objc objc + kwargs = dict(body=copy_from_obj_body) + with self.controller_context(req, *status_list, + **kwargs) as controller: + self.app.update_request(req) - copy_from_obj_body = LargeResponseBody() - set_http_connect(200, 200, 200, 200, 200, 201, 201, 201, - body=copy_from_obj_body) self.app.memcache.store = {} resp = controller.PUT(req) self.assertEquals(resp.status_int, 413) - def test_COPY(self): - with save_globals(): - controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0'}) - req.account = 'a' - set_http_connect(200, 200, 201, 201, 201) - # acct cont obj obj obj - resp = controller.PUT(req) - self.assertEquals(resp.status_int, 201) - - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': 'c/o'}) - req.account = 'a' - set_http_connect(200, 200, 200, 200, 200, 201, 201, 201, 200, 200) - # acct cont acct cont objc objc objc obj obj obj - self.app.memcache.store = {} + def test_basic_COPY(self): + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': 'c/o2'}) + status_list = (200, 200, 200, 200, 200, 201, 201, 201) + # acct cont objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: resp = controller.COPY(req) - self.assertEquals(resp.status_int, 201) - self.assertEquals(resp.headers['x-copied-from'], 'c/o') + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers['x-copied-from'], 'c/o') - req = Request.blank('/v1/a/c/o/o2', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': 'c/o'}) - req.account = 'a' - controller.object_name = 'o/o2' - set_http_connect(200, 200, 200, 200, 200, 201, 201, 201, 200, 200) - # acct cont acct cont objc objc objc obj obj obj - self.app.memcache.store = {} + def test_COPY_across_containers(self): + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': 'c2/o'}) + status_list = (200, 200, 200, 200, 200, 200, 201, 201, 201) + # acct cont c2 objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: resp = controller.COPY(req) - self.assertEquals(resp.status_int, 201) - self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2') - - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c/o'}) - req.account = 'a' - controller.object_name = 'o' - set_http_connect(200, 200, 200, 200, 200, 201, 201, 201, 200, 200) - # acct cont acct cont objc objc objc obj obj obj - self.app.memcache.store = {} + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers['x-copied-from'], 'c/o') + + def test_COPY_source_with_slashes_in_name(self): + req = Request.blank('/v1/a/c/o/o2', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': 'c/o'}) + status_list = (200, 200, 200, 200, 200, 201, 201, 201) + # acct cont objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: resp = controller.COPY(req) - self.assertEquals(resp.status_int, 201) - self.assertEquals(resp.headers['x-copied-from'], 'c/o') + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2') - req = Request.blank('/v1/a/c/o/o2', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c/o'}) - req.account = 'a' - controller.object_name = 'o/o2' - set_http_connect(200, 200, 200, 200, 200, 201, 201, 201, 200, 200) - # acct cont acct cont objc objc objc obj obj obj - self.app.memcache.store = {} + def test_COPY_destination_leading_slash(self): + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c/o'}) + status_list = (200, 200, 200, 200, 200, 201, 201, 201) + # acct cont objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: resp = controller.COPY(req) - self.assertEquals(resp.status_int, 201) - self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2') + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers['x-copied-from'], 'c/o') + + def test_COPY_source_with_slashes_destination_leading_slash(self): + req = Request.blank('/v1/a/c/o/o2', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c/o'}) + status_list = (200, 200, 200, 200, 200, 201, 201, 201) + # acct cont objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: + resp = controller.COPY(req) + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2') - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': 'c_o'}) - req.account = 'a' - controller.object_name = 'o' - set_http_connect(200, 200) - # acct cont - self.app.memcache.store = {} + def test_COPY_no_object_in_destination(self): + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': 'c_o'}) + status_list = [] # no requests needed + with self.controller_context(req, *status_list) as controller: resp = controller.COPY(req) - self.assertEquals(resp.status_int, 412) + self.assertEquals(resp.status_int, 412) - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c/o'}) - req.account = 'a' - controller.object_name = 'o' - set_http_connect(200, 200, 503, 503, 503) - # acct cont objc objc objc - self.app.memcache.store = {} + def test_COPY_server_error_reading_source(self): + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c/o'}) + status_list = (200, 200, 503, 503, 503) + # acct cont objc objc objc + with self.controller_context(req, *status_list) as controller: resp = controller.COPY(req) - self.assertEquals(resp.status_int, 503) + self.assertEquals(resp.status_int, 503) - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c/o'}) - req.account = 'a' - controller.object_name = 'o' - set_http_connect(200, 200, 404, 404, 404) - # acct cont objc objc objc - self.app.memcache.store = {} + def test_COPY_not_found_reading_source(self): + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c/o'}) + status_list = (200, 200, 404, 404, 404) + # acct cont objc objc objc + with self.controller_context(req, *status_list) as controller: resp = controller.COPY(req) - self.assertEquals(resp.status_int, 404) + self.assertEquals(resp.status_int, 404) - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c/o'}) - req.account = 'a' - controller.object_name = 'o' - set_http_connect(200, 200, 404, 404, 200, 201, 201, 201) - # acct cont objc objc objc obj obj obj - self.app.memcache.store = {} + def test_COPY_with_some_missing_sources(self): + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c/o'}) + status_list = (200, 200, 404, 404, 200, 201, 201, 201) + # acct cont objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: resp = controller.COPY(req) - self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.status_int, 201) - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c/o', - 'X-Object-Meta-Ours': 'okay'}) - req.account = 'a' - controller.object_name = 'o' - set_http_connect(200, 200, 200, 200, 200, 201, 201, 201) - # acct cont objc objc objc obj obj obj - self.app.memcache.store = {} + def test_COPY_with_metadata(self): + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c/o', + 'X-Object-Meta-Ours': 'okay'}) + status_list = (200, 200, 200, 200, 200, 201, 201, 201) + # acct cont objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: resp = controller.COPY(req) - self.assertEquals(resp.status_int, 201) - self.assertEquals(resp.headers.get('x-object-meta-test'), - 'testing') - self.assertEquals(resp.headers.get('x-object-meta-ours'), 'okay') - self.assertEquals(resp.headers.get('x-delete-at'), '9876543210') + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers.get('x-object-meta-test'), + 'testing') + self.assertEquals(resp.headers.get('x-object-meta-ours'), 'okay') + self.assertEquals(resp.headers.get('x-delete-at'), '9876543210') - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c/o'}) - self.app.update_request(req) + def test_COPY_source_larger_than_max_file_size(self): + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c/o'}) - class LargeResponseBody(object): + class LargeResponseBody(object): - def __len__(self): - return MAX_FILE_SIZE + 1 + def __len__(self): + return MAX_FILE_SIZE + 1 - def __getitem__(self, key): - return '' + def __getitem__(self, key): + return '' - copy_from_obj_body = LargeResponseBody() - set_http_connect(200, 200, 200, 200, 200, 201, 201, 201, - body=copy_from_obj_body) - self.app.memcache.store = {} + copy_from_obj_body = LargeResponseBody() + status_list = (200, 200, 200, 200, 200) + # acct cont objc objc objc + kwargs = dict(body=copy_from_obj_body) + with self.controller_context(req, *status_list, + **kwargs) as controller: resp = controller.COPY(req) - self.assertEquals(resp.status_int, 413) + self.assertEquals(resp.status_int, 413) def test_COPY_newest(self): with save_globals(): @@ -2574,7 +2594,7 @@ class TestObjectController(unittest.TestCase): def test_chunked_put(self): - class ChunkedFile(): + class ChunkedFile(object): def __init__(self, bytes): self.bytes = bytes @@ -3824,9 +3844,7 @@ class TestObjectController(unittest.TestCase): req.content_length = 0 resp = controller.OPTIONS(req) self.assertEquals(200, resp.status_int) - self.assertEquals( - 'https://bar.baz', - resp.headers['access-control-allow-origin']) + self.assertEquals('*', resp.headers['access-control-allow-origin']) for verb in 'OPTIONS COPY GET POST PUT DELETE HEAD'.split(): self.assertTrue( verb in resp.headers['access-control-allow-methods']) @@ -3842,10 +3860,11 @@ class TestObjectController(unittest.TestCase): def stubContainerInfo(*args): return { 'cors': { - 'allow_origin': 'http://foo.bar' + 'allow_origin': 'http://not.foo.bar' } } controller.container_info = stubContainerInfo + controller.app.strict_cors_mode = False def objectGET(controller, req): return Response(headers={ @@ -3876,6 +3895,50 @@ class TestObjectController(unittest.TestCase): 'x-trans-id', 'x-object-meta-color']) self.assertEquals(expected_exposed, exposed) + controller.app.strict_cors_mode = True + req = Request.blank( + '/v1/a/c/o.jpg', + {'REQUEST_METHOD': 'GET'}, + headers={'Origin': 'http://foo.bar'}) + + resp = cors_validation(objectGET)(controller, req) + + self.assertEquals(200, resp.status_int) + self.assertTrue('access-control-allow-origin' not in resp.headers) + + def test_CORS_valid_with_obj_headers(self): + with save_globals(): + controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') + + def stubContainerInfo(*args): + return { + 'cors': { + 'allow_origin': 'http://foo.bar' + } + } + controller.container_info = stubContainerInfo + + def objectGET(controller, req): + return Response(headers={ + 'X-Object-Meta-Color': 'red', + 'X-Super-Secret': 'hush', + 'Access-Control-Allow-Origin': 'http://obj.origin', + 'Access-Control-Expose-Headers': 'x-trans-id' + }) + + req = Request.blank( + '/v1/a/c/o.jpg', + {'REQUEST_METHOD': 'GET'}, + headers={'Origin': 'http://foo.bar'}) + + resp = cors_validation(objectGET)(controller, req) + + self.assertEquals(200, resp.status_int) + self.assertEquals('http://obj.origin', + resp.headers['access-control-allow-origin']) + self.assertEquals('x-trans-id', + resp.headers['access-control-expose-headers']) + def _gather_x_container_headers(self, controller_call, req, *connect_args, **kwargs): header_list = kwargs.pop('header_list', ['X-Container-Device', @@ -4092,7 +4155,8 @@ class TestContainerController(unittest.TestCase): self.app = proxy_server.Application(None, FakeMemcache(), account_ring=FakeRing(), container_ring=FakeRing(), - object_ring=FakeRing()) + object_ring=FakeRing(), + logger=FakeLogger()) def test_transfer_headers(self): src_headers = {'x-remove-versions-location': 'x', @@ -4843,9 +4907,7 @@ class TestContainerController(unittest.TestCase): req.content_length = 0 resp = controller.OPTIONS(req) self.assertEquals(200, resp.status_int) - self.assertEquals( - 'https://bar.baz', - resp.headers['access-control-allow-origin']) + self.assertEquals('*', resp.headers['access-control-allow-origin']) for verb in 'OPTIONS GET POST PUT DELETE HEAD'.split(): self.assertTrue( verb in resp.headers['access-control-allow-methods']) @@ -5016,11 +5078,60 @@ class TestContainerController(unittest.TestCase): 'X-Account-Device': 'sdc'} ]) + def test_PUT_backed_x_timestamp_header(self): + timestamps = [] + + def capture_timestamps(*args, **kwargs): + headers = kwargs['headers'] + timestamps.append(headers.get('X-Timestamp')) + + req = Request.blank('/v1/a/c', method='PUT', headers={'': ''}) + with save_globals(): + new_connect = set_http_connect(200, # account existance check + 201, 201, 201, + give_connect=capture_timestamps) + resp = self.app.handle_request(req) + + # sanity + self.assertRaises(StopIteration, new_connect.code_iter.next) + self.assertEqual(2, resp.status_int // 100) + + timestamps.pop(0) # account existance check + self.assertEqual(3, len(timestamps)) + for timestamp in timestamps: + self.assertEqual(timestamp, timestamps[0]) + self.assert_(re.match('[0-9]{10}\.[0-9]{5}', timestamp)) + + def test_DELETE_backed_x_timestamp_header(self): + timestamps = [] + + def capture_timestamps(*args, **kwargs): + headers = kwargs['headers'] + timestamps.append(headers.get('X-Timestamp')) + + req = Request.blank('/v1/a/c', method='DELETE', headers={'': ''}) + self.app.update_request(req) + with save_globals(): + new_connect = set_http_connect(200, # account existance check + 201, 201, 201, + give_connect=capture_timestamps) + resp = self.app.handle_request(req) + + # sanity + self.assertRaises(StopIteration, new_connect.code_iter.next) + self.assertEqual(2, resp.status_int // 100) + + timestamps.pop(0) # account existance check + self.assertEqual(3, len(timestamps)) + for timestamp in timestamps: + self.assertEqual(timestamp, timestamps[0]) + self.assert_(re.match('[0-9]{10}\.[0-9]{5}', timestamp)) + def test_node_read_timeout_retry_to_container(self): with save_globals(): req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'GET'}) self.app.node_timeout = 0.1 - set_http_connect(200, 200, 200, body='abcdef', slow=[2]) + set_http_connect(200, 200, 200, body='abcdef', slow=[1.0, 1.0]) resp = req.get_response(self.app) got_exc = False try: @@ -5547,6 +5658,143 @@ class TestAccountControllerFakeGetResponse(unittest.TestCase): resp = req.get_response(self.app) self.assertEqual(400, resp.status_int) + def test_account_acl_header_access(self): + acl = { + 'admin': ['AUTH_alice'], + 'read-write': ['AUTH_bob'], + 'read-only': ['AUTH_carol'], + } + prefix = get_sys_meta_prefix('account') + privileged_headers = {(prefix + 'core-access-control'): format_acl( + version=2, acl_dict=acl)} + + app = proxy_server.Application( + None, FakeMemcache(), account_ring=FakeRing(), + container_ring=FakeRing(), object_ring=FakeRing()) + + with save_globals(): + # Mock account server will provide privileged information (ACLs) + set_http_connect(200, 200, 200, headers=privileged_headers) + req = Request.blank('/v1/a', environ={'REQUEST_METHOD': 'GET'}) + resp = app.handle_request(req) + + # Not a swift_owner -- ACLs should NOT be in response + header = 'X-Account-Access-Control' + self.assert_(header not in resp.headers, '%r was in %r' % ( + header, resp.headers)) + + # Same setup -- mock acct server will provide ACLs + set_http_connect(200, 200, 200, headers=privileged_headers) + req = Request.blank('/v1/a', environ={'REQUEST_METHOD': 'GET', + 'swift_owner': True}) + resp = app.handle_request(req) + + # For a swift_owner, the ACLs *should* be in response + self.assert_(header in resp.headers, '%r not in %r' % ( + header, resp.headers)) + + def test_account_acls_through_delegation(self): + + # Define a way to grab the requests sent out from the AccountController + # to the Account Server, and a way to inject responses we'd like the + # Account Server to return. + resps_to_send = [] + + @contextmanager + def patch_account_controller_method(verb): + old_method = getattr(proxy_server.AccountController, verb) + new_method = lambda self, req, *_, **__: resps_to_send.pop(0) + try: + setattr(proxy_server.AccountController, verb, new_method) + yield + finally: + setattr(proxy_server.AccountController, verb, old_method) + + def make_test_request(http_method, swift_owner=True): + env = { + 'REQUEST_METHOD': http_method, + 'swift_owner': swift_owner, + } + acl = { + 'admin': ['foo'], + 'read-write': ['bar'], + 'read-only': ['bas'], + } + headers = {} if http_method in ('GET', 'HEAD') else { + 'x-account-access-control': format_acl(version=2, acl_dict=acl) + } + + return Request.blank('/v1/a', environ=env, headers=headers) + + # Our AccountController will invoke methods to communicate with the + # Account Server, and they will return responses like these: + def make_canned_response(http_method): + acl = { + 'admin': ['foo'], + 'read-write': ['bar'], + 'read-only': ['bas'], + } + headers = {'x-account-sysmeta-core-access-control': format_acl( + version=2, acl_dict=acl)} + canned_resp = Response(headers=headers) + canned_resp.environ = { + 'PATH_INFO': '/acct', + 'REQUEST_METHOD': http_method, + } + resps_to_send.append(canned_resp) + + app = proxy_server.Application( + None, FakeMemcache(), account_ring=FakeRing(), + container_ring=FakeRing(), object_ring=FakeRing()) + app.allow_account_management = True + + ext_header = 'x-account-access-control' + with patch_account_controller_method('GETorHEAD_base'): + # GET/HEAD requests should remap sysmeta headers from acct server + for verb in ('GET', 'HEAD'): + make_canned_response(verb) + req = make_test_request(verb) + resp = app.handle_request(req) + h = parse_acl(version=2, data=resp.headers.get(ext_header)) + self.assertEqual(h['admin'], ['foo']) + self.assertEqual(h['read-write'], ['bar']) + self.assertEqual(h['read-only'], ['bas']) + + # swift_owner = False: GET/HEAD shouldn't return sensitive info + make_canned_response(verb) + req = make_test_request(verb, swift_owner=False) + resp = app.handle_request(req) + h = resp.headers + self.assertEqual(None, h.get(ext_header)) + + # swift_owner unset: GET/HEAD shouldn't return sensitive info + make_canned_response(verb) + req = make_test_request(verb, swift_owner=False) + del req.environ['swift_owner'] + resp = app.handle_request(req) + h = resp.headers + self.assertEqual(None, h.get(ext_header)) + + # Verify that PUT/POST requests remap sysmeta headers from acct server + with patch_account_controller_method('make_requests'): + make_canned_response('PUT') + req = make_test_request('PUT') + resp = app.handle_request(req) + + h = parse_acl(version=2, data=resp.headers.get(ext_header)) + self.assertEqual(h['admin'], ['foo']) + self.assertEqual(h['read-write'], ['bar']) + self.assertEqual(h['read-only'], ['bas']) + + make_canned_response('POST') + req = make_test_request('POST') + resp = app.handle_request(req) + + h = parse_acl(version=2, data=resp.headers.get(ext_header)) + self.assertEqual(h['admin'], ['foo']) + self.assertEqual(h['read-write'], ['bar']) + self.assertEqual(h['read-only'], ['bas']) + class FakeObjectController(object): -- cgit