diff options
Diffstat (limited to 'test')
49 files changed, 3451 insertions, 2080 deletions
diff --git a/test/functional/gluster_swift_tests.py b/test/functional/gluster_swift_tests.py index 2768f9d..b4514c9 100644 --- a/test/functional/gluster_swift_tests.py +++ b/test/functional/gluster_swift_tests.py @@ -58,44 +58,13 @@ class TestFile(Base): data_read = file.read() self.assertEquals(data,data_read) - def testInvalidHeadersPUT(self): - #TODO: Although we now support x-delete-at and x-delete-after, - #retained this test case as we may add some other header to - #unsupported list in future - raise SkipTest() - file = self.env.container.file(Utils.create_name()) - self.assertRaises(ResponseError, - file.write_random, - self.env.file_size, - hdrs={'X-Delete-At': '9876545321'}) - self.assert_status(400) - self.assertRaises(ResponseError, - file.write_random, - self.env.file_size, - hdrs={'X-Delete-After': '60'}) - self.assert_status(400) - - def testInvalidHeadersPOST(self): - #TODO: Although we now support x-delete-at and x-delete-after, - #retained this test case as we may add some other header to - #unsupported list in future - raise SkipTest() - file = self.env.container.file(Utils.create_name()) - file.write_random(self.env.file_size) - headers = file.make_headers(cfg={}) - headers.update({ 'X-Delete-At' : '987654321'}) - # Need to call conn.make_request instead of file.sync_metadata - # because sync_metadata calls make_headers. make_headers() - # overwrites any headers in file.metadata as 'user' metadata - # by appending 'X-Object-Meta-' to any of the headers - # in file.metadata. - file.conn.make_request('POST', file.path, hdrs=headers, cfg={}) - self.assertEqual(400, file.conn.response.status) - - headers = file.make_headers(cfg={}) - headers.update({ 'X-Delete-After' : '60'}) - file.conn.make_request('POST', file.path, hdrs=headers, cfg={}) - self.assertEqual(400, file.conn.response.status) + def test_PUT_large_object(self): + file_item = self.env.container.file(Utils.create_name()) + data = File.random_data(1024 * 1024 * 2) + self.assertTrue(file_item.write(data)) + self.assert_status(201) + self.assertTrue(data == file_item.read()) + self.assert_status(200) class TestFileUTF8(Base2, TestFile): @@ -375,3 +344,49 @@ class TestMultiProtocolAccess(Base): md5_returned = hashlib.md5(data_read_from_mountP).hexdigest() self.assertEquals(md5_returned,file_info['etag']) fhOnMountPoint.close() + + def testObjectMetadataWhenFileModified(self): + data = "I'm whatever Gotham needs me to be " + data_hash = hashlib.md5(data).hexdigest() + # Create an object through object interface + object_name = Utils.create_name() + object_item = self.env.container.file(object_name) + object_item.write(data) + # Make sure GET works + self.assertEqual(data, object_item.read()) + self.assert_status(200) + # Check Etag is right + self.assertEqual(data_hash, object_item.info()['etag']) + self.assert_status(200) + + # Extend/append more data to file from filesystem interface + file_path = os.path.join(self.env.root_dir, + self.env.container.name, + object_name) + more_data = "- Batman" + with open(file_path, 'a') as f: + f.write(more_data) + total_data = data + more_data + total_data_hash = hashlib.md5(total_data).hexdigest() + # Make sure GET works + self.assertEqual(total_data, object_item.read()) + self.assert_status(200) + # Check Etag and content-length is right + metadata = object_item.info() + self.assert_status(200) + self.assertEqual(total_data_hash, metadata['etag']) + self.assertEqual(len(total_data), int(metadata['content_length'])) + + # Re-write the file to be shorter + new_data = "I am Batman" + new_data_hash = hashlib.md5(new_data).hexdigest() + with open(file_path, 'w') as f: + f.write(new_data) + # Make sure GET works + self.assertEqual(new_data, object_item.read()) + self.assert_status(200) + # Check Etag and content-length is right + metadata = object_item.info() + self.assert_status(200) + self.assertEqual(new_data_hash, metadata['etag']) + self.assertEqual(len(new_data), int(metadata['content_length'])) diff --git a/test/functional/swift_test_client.py b/test/functional/swift_test_client.py index a7c7c96..27e025b 100644 --- a/test/functional/swift_test_client.py +++ b/test/functional/swift_test_client.py @@ -144,6 +144,7 @@ class Connection(object): auth_scheme = 'https://' if self.auth_ssl else 'http://' auth_netloc = "%s:%d" % (self.auth_host, self.auth_port) auth_url = auth_scheme + auth_netloc + auth_path + (storage_url, storage_token) = get_auth( auth_url, auth_user, self.password, snet=False, tenant_name=self.account, auth_version=self.auth_version, @@ -166,17 +167,29 @@ class Connection(object): self.storage_host = x[2].split(':')[0] if ':' in x[2]: self.storage_port = int(x[2].split(':')[1]) - # Make sure storage_url and the storage_token are - # strings and not unicode, since + # Make sure storage_url is a string and not unicode, since # keystoneclient (called by swiftclient) returns them in # unicode and this would cause troubles when doing # no_safe_quote query. self.storage_url = str('/%s/%s' % (x[3], x[4])) - self.storage_token = str(storage_token) + + self.storage_token = storage_token self.http_connect() return self.storage_url, self.storage_token + def cluster_info(self): + """ + Retrieve the data in /info, or {} on 404 + """ + status = self.make_request('GET', '/info', + cfg={'absolute_path': True}) + if status == 404: + return {} + if not 200 <= status <= 299: + raise ResponseError(self.response, 'GET', '/info') + return json.loads(self.response.read()) + def http_connect(self): self.connection = self.conn_class(self.storage_host, port=self.storage_port) @@ -207,8 +220,8 @@ class Connection(object): def make_request(self, method, path=[], data='', hdrs={}, parms={}, cfg={}): - if not cfg.get('verbatim_path'): - # Set verbatim_path=True to make a request to exactly the given + if not cfg.get('absolute_path'): + # Set absolute_path=True to make a request to exactly the given # path, not storage path + given path. Useful for # non-account/container/object requests. path = self.make_path(path, cfg=cfg) @@ -305,7 +318,7 @@ class Connection(object): return self.response.status -class Base: +class Base(object): def __str__(self): return self.name @@ -339,6 +352,16 @@ class Account(Base): self.conn = conn self.name = str(name) + def update_metadata(self, metadata={}, cfg={}): + headers = dict(("X-Account-Meta-%s" % k, v) + for k, v in metadata.items()) + + self.conn.make_request('POST', self.path, hdrs=headers, cfg=cfg) + if not 200 <= self.conn.response.status <= 299: + raise ResponseError(self.conn.response, 'POST', + self.conn.make_path(self.path)) + return True + def container(self, container_name): return Container(self.conn, self.name, container_name) @@ -532,6 +555,11 @@ class File(Base): else: headers['Content-Type'] = 'application/octet-stream' + if cfg.get('x_delete_at'): + headers['X-Delete-At'] = cfg.get('x_delete_at') + if cfg.get('x_delete_after'): + headers['X-Delete-After'] = cfg.get('x_delete_after') + for key in self.metadata: headers['X-Object-Meta-' + key] = self.metadata[key] @@ -588,7 +616,11 @@ class File(Base): ['last_modified', 'last-modified'], ['etag', 'etag']] - header_fields = self.header_fields(fields) + optional_fields = [['x_delete_at', 'x-delete-at'], + ['x_delete_after', 'x-delete-after']] + + header_fields = self.header_fields(fields, + optional_fields=optional_fields) header_fields['etag'] = header_fields['etag'].strip('"') return header_fields @@ -705,7 +737,6 @@ class File(Base): cfg.get('set_content_length') else: headers['Content-Length'] = 0 - self.conn.make_request('POST', self.path, hdrs=headers, cfg=cfg) if self.conn.response.status not in (201, 202): diff --git a/test/functional/swift_testing.py b/test/functional/swift_testing.py index f05cb48..2a1e1fa 100644 --- a/test/functional/swift_testing.py +++ b/test/functional/swift_testing.py @@ -19,10 +19,13 @@ import socket import sys from time import sleep from urlparse import urlparse +import functools +from nose import SkipTest from test import get_config from swiftclient import get_auth, http_connection +from test.functional.swift_test_client import Connection conf = get_config('func_test') web_front_end = conf.get('web_front_end', 'integral') @@ -184,3 +187,45 @@ def check_response(conn): resp.read() raise InternalServerError() return resp + +cluster_info = {} + + +def get_cluster_info(): + conn = Connection(conf) + conn.authenticate() + global cluster_info + cluster_info = conn.cluster_info() + + +def reset_acl(): + def post(url, token, parsed, conn): + conn.request('POST', parsed.path, '', { + 'X-Auth-Token': token, + 'X-Account-Access-Control': '{}' + }) + return check_response(conn) + resp = retry(post, use_account=1) + resp.read() + + +def requires_acls(f): + @functools.wraps(f) + def wrapper(*args, **kwargs): + if skip: + raise SkipTest + if not cluster_info: + get_cluster_info() + # Determine whether this cluster has account ACLs; if not, skip test + if not cluster_info.get('tempauth', {}).get('account_acls'): + raise SkipTest + if 'keystoneauth' in cluster_info: + # remove when keystoneauth supports account acls + raise SkipTest + reset_acl() + try: + rv = f(*args, **kwargs) + finally: + reset_acl() + return rv + return wrapper diff --git a/test/functional/test_account.py b/test/functional/test_account.py index d456090..1cc61bc 100755 --- a/test/functional/test_account.py +++ b/test/functional/test_account.py @@ -17,19 +17,57 @@ import unittest import json +from uuid import uuid4 from nose import SkipTest +from string import letters from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \ MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH from swift.common.middleware.acl import format_acl -from test.functional.swift_test_client import Connection -from test import get_config -from swift_testing import check_response, retry, skip, web_front_end +from swift_testing import (check_response, retry, skip, skip2, skip3, + web_front_end, requires_acls) import swift_testing +from test.functional.tests import load_constraint class TestAccount(unittest.TestCase): + def setUp(self): + self.max_meta_count = load_constraint('max_meta_count') + self.max_meta_name_length = load_constraint('max_meta_name_length') + self.max_meta_overall_size = load_constraint('max_meta_overall_size') + self.max_meta_value_length = load_constraint('max_meta_value_length') + + def head(url, token, parsed, conn): + conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(head) + self.existing_metadata = set([ + k for k, v in resp.getheaders() if + k.lower().startswith('x-account-meta')]) + + def tearDown(self): + def head(url, token, parsed, conn): + conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(head) + resp.read() + new_metadata = set( + [k for k, v in resp.getheaders() if + k.lower().startswith('x-account-meta')]) + + def clear_meta(url, token, parsed, conn, remove_metadata_keys): + headers = {'X-Auth-Token': token} + headers.update((k, '') for k in remove_metadata_keys) + conn.request('POST', parsed.path, '', headers) + return check_response(conn) + extra_metadata = list(self.existing_metadata ^ new_metadata) + for i in range(0, len(extra_metadata), 90): + batch = extra_metadata[i:i + 90] + resp = retry(clear_meta, batch) + resp.read() + self.assertEqual(resp.status // 100, 2) + def test_metadata(self): if skip: raise SkipTest @@ -49,49 +87,338 @@ class TestAccount(unittest.TestCase): resp = retry(post, '') resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader('x-account-meta-test'), None) + self.assertEqual(resp.getheader('x-account-meta-test'), None) resp = retry(get) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader('x-account-meta-test'), None) + self.assertEqual(resp.getheader('x-account-meta-test'), None) resp = retry(post, 'Value') resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader('x-account-meta-test'), 'Value') + self.assertEqual(resp.getheader('x-account-meta-test'), 'Value') resp = retry(get) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader('x-account-meta-test'), 'Value') + self.assertEqual(resp.getheader('x-account-meta-test'), 'Value') - def test_tempauth_account_acls(self): - if skip: + def test_invalid_acls(self): + def post(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + # needs to be an acceptable header size + num_keys = 8 + max_key_size = load_constraint('max_header_size') / num_keys + acl = {'admin': [c * max_key_size for c in letters[:num_keys]]} + headers = {'x-account-access-control': format_acl( + version=2, acl_dict=acl)} + resp = retry(post, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 400) + + # and again a touch smaller + acl = {'admin': [c * max_key_size for c in letters[:num_keys - 1]]} + headers = {'x-account-access-control': format_acl( + version=2, acl_dict=acl)} + resp = retry(post, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + @requires_acls + def test_invalid_acl_keys(self): + def post(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + # needs to be json + resp = retry(post, headers={'X-Account-Access-Control': 'invalid'}, + use_account=1) + resp.read() + self.assertEqual(resp.status, 400) + + acl_user = swift_testing.swift_test_user[1] + acl = {'admin': [acl_user], 'invalid_key': 'invalid_value'} + headers = {'x-account-access-control': format_acl( + version=2, acl_dict=acl)} + + resp = retry(post, headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 400) + self.assertEqual(resp.getheader('X-Account-Access-Control'), None) + + @requires_acls + def test_invalid_acl_values(self): + def post(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + acl = {'admin': 'invalid_value'} + headers = {'x-account-access-control': format_acl( + version=2, acl_dict=acl)} + + resp = retry(post, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 400) + self.assertEqual(resp.getheader('X-Account-Access-Control'), None) + + @requires_acls + def test_read_only_acl(self): + if skip3: raise SkipTest - # Determine whether this cluster has account ACLs; if not, skip test - conn = Connection(get_config('func_test')) - conn.authenticate() - status = conn.make_request( - 'GET', '/info', cfg={'verbatim_path': True}) - if status // 100 != 2: - # Can't tell if account ACLs are enabled; skip tests proactively. + def get(url, token, parsed, conn): + conn.request('GET', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + + def post(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + # cannot read account + resp = retry(get, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # grant read access + acl_user = swift_testing.swift_test_user[2] + acl = {'read-only': [acl_user]} + headers = {'x-account-access-control': format_acl( + version=2, acl_dict=acl)} + resp = retry(post, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # read-only can read account headers + resp = retry(get, use_account=3) + resp.read() + self.assert_(resp.status in (200, 204)) + # but not acls + self.assertEqual(resp.getheader('X-Account-Access-Control'), None) + + # read-only can not write metadata + headers = {'x-account-meta-test': 'value'} + resp = retry(post, headers=headers, use_account=3) + resp.read() + self.assertEqual(resp.status, 403) + + # but they can read it + headers = {'x-account-meta-test': 'value'} + resp = retry(post, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + resp = retry(get, use_account=3) + resp.read() + self.assert_(resp.status in (200, 204)) + self.assertEqual(resp.getheader('X-Account-Meta-Test'), 'value') + + @requires_acls + def test_read_write_acl(self): + if skip3: + raise SkipTest + + def get(url, token, parsed, conn): + conn.request('GET', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + + def post(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + # cannot read account + resp = retry(get, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # grant read-write access + acl_user = swift_testing.swift_test_user[2] + acl = {'read-write': [acl_user]} + headers = {'x-account-access-control': format_acl( + version=2, acl_dict=acl)} + resp = retry(post, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # read-write can read account headers + resp = retry(get, use_account=3) + resp.read() + self.assert_(resp.status in (200, 204)) + # but not acls + self.assertEqual(resp.getheader('X-Account-Access-Control'), None) + + # read-write can not write account metadata + headers = {'x-account-meta-test': 'value'} + resp = retry(post, headers=headers, use_account=3) + resp.read() + self.assertEqual(resp.status, 403) + + @requires_acls + def test_admin_acl(self): + if skip3: + raise SkipTest + + def get(url, token, parsed, conn): + conn.request('GET', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + + def post(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + # cannot read account + resp = retry(get, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # grant admin access + acl_user = swift_testing.swift_test_user[2] + acl = {'admin': [acl_user]} + acl_json_str = format_acl(version=2, acl_dict=acl) + headers = {'x-account-access-control': acl_json_str} + resp = retry(post, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # admin can read account headers + resp = retry(get, use_account=3) + resp.read() + self.assert_(resp.status in (200, 204)) + # including acls + self.assertEqual(resp.getheader('X-Account-Access-Control'), + acl_json_str) + + # admin can write account metadata + value = str(uuid4()) + headers = {'x-account-meta-test': value} + resp = retry(post, headers=headers, use_account=3) + resp.read() + self.assertEqual(resp.status, 204) + resp = retry(get, use_account=3) + resp.read() + self.assert_(resp.status in (200, 204)) + self.assertEqual(resp.getheader('X-Account-Meta-Test'), value) + + # admin can even revoke their own access + headers = {'x-account-access-control': '{}'} + resp = retry(post, headers=headers, use_account=3) + resp.read() + self.assertEqual(resp.status, 204) + + # and again, cannot read account + resp = retry(get, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + @requires_acls + def test_protected_tempurl(self): + if skip3: + raise SkipTest + + def get(url, token, parsed, conn): + conn.request('GET', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + + def post(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + # add a account metadata, and temp-url-key to account + value = str(uuid4()) + headers = { + 'x-account-meta-temp-url-key': 'secret', + 'x-account-meta-test': value, + } + resp = retry(post, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # grant read-only access to tester3 + acl_user = swift_testing.swift_test_user[2] + acl = {'read-only': [acl_user]} + acl_json_str = format_acl(version=2, acl_dict=acl) + headers = {'x-account-access-control': acl_json_str} + resp = retry(post, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # read-only tester3 can read account metadata + resp = retry(get, use_account=3) + resp.read() + self.assert_(resp.status in (200, 204), + 'Expected status in (200, 204), got %s' % resp.status) + self.assertEqual(resp.getheader('X-Account-Meta-Test'), value) + # but not temp-url-key + self.assertEqual(resp.getheader('X-Account-Meta-Temp-Url-Key'), None) + + # grant read-write access to tester3 + acl_user = swift_testing.swift_test_user[2] + acl = {'read-write': [acl_user]} + acl_json_str = format_acl(version=2, acl_dict=acl) + headers = {'x-account-access-control': acl_json_str} + resp = retry(post, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # read-write tester3 can read account metadata + resp = retry(get, use_account=3) + resp.read() + self.assert_(resp.status in (200, 204), + 'Expected status in (200, 204), got %s' % resp.status) + self.assertEqual(resp.getheader('X-Account-Meta-Test'), value) + # but not temp-url-key + self.assertEqual(resp.getheader('X-Account-Meta-Temp-Url-Key'), None) + + # grant admin access to tester3 + acl_user = swift_testing.swift_test_user[2] + acl = {'admin': [acl_user]} + acl_json_str = format_acl(version=2, acl_dict=acl) + headers = {'x-account-access-control': acl_json_str} + resp = retry(post, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # admin tester3 can read account metadata + resp = retry(get, use_account=3) + resp.read() + self.assert_(resp.status in (200, 204), + 'Expected status in (200, 204), got %s' % resp.status) + self.assertEqual(resp.getheader('X-Account-Meta-Test'), value) + # including temp-url-key + self.assertEqual(resp.getheader('X-Account-Meta-Temp-Url-Key'), + 'secret') + + # admin tester3 can even change temp-url-key + secret = str(uuid4()) + headers = { + 'x-account-meta-temp-url-key': secret, + } + resp = retry(post, headers=headers, use_account=3) + resp.read() + self.assertEqual(resp.status, 204) + resp = retry(get, use_account=3) + resp.read() + self.assert_(resp.status in (200, 204), + 'Expected status in (200, 204), got %s' % resp.status) + self.assertEqual(resp.getheader('X-Account-Meta-Temp-Url-Key'), + secret) + + @requires_acls + def test_account_acls(self): + if skip2: raise SkipTest - else: - cluster_info = json.loads(conn.response.read()) - if not cluster_info.get('tempauth', {}).get('account_acls'): - raise SkipTest - if 'keystoneauth' in cluster_info: - # Unfortunate hack -- tempauth (with account ACLs) is expected - # to play nice with Keystone (without account ACLs), but Zuul - # functest framework doesn't give us an easy way to get a - # tempauth user. - raise SkipTest def post(url, token, parsed, conn, headers): new_headers = dict({'X-Auth-Token': token}, **headers) @@ -212,6 +539,137 @@ class TestAccount(unittest.TestCase): use_account=1) resp.read() + @requires_acls + def test_swift_account_acls(self): + if skip: + raise SkipTest + + def post(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + def head(url, token, parsed, conn): + conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + + def get(url, token, parsed, conn): + conn.request('GET', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + + try: + # User1 can POST to their own account + resp = retry(post, headers={'X-Account-Access-Control': '{}'}) + resp.read() + self.assertEqual(resp.status, 204) + self.assertEqual(resp.getheader('X-Account-Access-Control'), None) + + # User1 can GET their own empty account + resp = retry(get) + resp.read() + self.assertEqual(resp.status // 100, 2) + self.assertEqual(resp.getheader('X-Account-Access-Control'), None) + + # User1 can POST non-empty data + acl_json = '{"admin":["bob"]}' + resp = retry(post, headers={'X-Account-Access-Control': acl_json}) + resp.read() + self.assertEqual(resp.status, 204) + + # User1 can GET the non-empty data + resp = retry(get) + resp.read() + self.assertEqual(resp.status // 100, 2) + self.assertEqual(resp.getheader('X-Account-Access-Control'), + acl_json) + + # POST non-JSON ACL should fail + resp = retry(post, headers={'X-Account-Access-Control': 'yuck'}) + resp.read() + # resp.status will be 400 if tempauth or some other ACL-aware + # auth middleware rejects it, or 200 (but silently swallowed by + # core Swift) if ACL-unaware auth middleware approves it. + + # A subsequent GET should show the old, valid data, not the garbage + resp = retry(get) + resp.read() + self.assertEqual(resp.status // 100, 2) + self.assertEqual(resp.getheader('X-Account-Access-Control'), + acl_json) + + finally: + # Make sure to clean up even if tests fail -- User2 should not + # have access to User1's account in other functional tests! + resp = retry(post, headers={'X-Account-Access-Control': '{}'}) + resp.read() + + def test_swift_prohibits_garbage_account_acls(self): + if skip: + raise SkipTest + + def post(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + def get(url, token, parsed, conn): + conn.request('GET', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + + try: + # User1 can POST to their own account + resp = retry(post, headers={'X-Account-Access-Control': '{}'}) + resp.read() + self.assertEqual(resp.status, 204) + self.assertEqual(resp.getheader('X-Account-Access-Control'), None) + + # User1 can GET their own empty account + resp = retry(get) + resp.read() + self.assertEqual(resp.status // 100, 2) + self.assertEqual(resp.getheader('X-Account-Access-Control'), None) + + # User1 can POST non-empty data + acl_json = '{"admin":["bob"]}' + resp = retry(post, headers={'X-Account-Access-Control': acl_json}) + resp.read() + self.assertEqual(resp.status, 204) + # If this request is handled by ACL-aware auth middleware, then the + # ACL will be persisted. If it is handled by ACL-unaware auth + # middleware, then the header will be thrown out. But the request + # should return successfully in any case. + + # User1 can GET the non-empty data + resp = retry(get) + resp.read() + self.assertEqual(resp.status // 100, 2) + # ACL will be set if some ACL-aware auth middleware (e.g. tempauth) + # propagates it to sysmeta; if no ACL-aware auth middleware does, + # then X-Account-Access-Control will still be empty. + + # POST non-JSON ACL should fail + resp = retry(post, headers={'X-Account-Access-Control': 'yuck'}) + resp.read() + # resp.status will be 400 if tempauth or some other ACL-aware + # auth middleware rejects it, or 200 (but silently swallowed by + # core Swift) if ACL-unaware auth middleware approves it. + + # A subsequent GET should either show the old, valid data (if + # ACL-aware auth middleware is propagating it) or show nothing + # (if no auth middleware in the pipeline is ACL-aware), but should + # never return the garbage ACL. + resp = retry(get) + resp.read() + self.assertEqual(resp.status // 100, 2) + self.assertNotEqual(resp.getheader('X-Account-Access-Control'), + 'yuck') + + finally: + # Make sure to clean up even if tests fail -- User2 should not + # have access to User1's account in other functional tests! + resp = retry(post, headers={'X-Account-Access-Control': '{}'}) + resp.read() + def test_unicode_metadata(self): if skip: raise SkipTest @@ -233,24 +691,24 @@ class TestAccount(unittest.TestCase): resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader(uni_key.encode('utf-8')), '1') + self.assertEqual(resp.getheader(uni_key.encode('utf-8')), '1') resp = retry(post, 'X-Account-Meta-uni', uni_value) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader('X-Account-Meta-uni'), - uni_value.encode('utf-8')) + self.assertEqual(resp.getheader('X-Account-Meta-uni'), + uni_value.encode('utf-8')) if (web_front_end == 'integral'): resp = retry(post, uni_key, uni_value) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader(uni_key.encode('utf-8')), - uni_value.encode('utf-8')) + self.assertEqual(resp.getheader(uni_key.encode('utf-8')), + uni_value.encode('utf-8')) def test_multi_metadata(self): if skip: @@ -267,19 +725,19 @@ class TestAccount(unittest.TestCase): resp = retry(post, 'X-Account-Meta-One', '1') resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader('x-account-meta-one'), '1') + self.assertEqual(resp.getheader('x-account-meta-one'), '1') resp = retry(post, 'X-Account-Meta-Two', '2') resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader('x-account-meta-one'), '1') - self.assertEquals(resp.getheader('x-account-meta-two'), '2') + self.assertEqual(resp.getheader('x-account-meta-one'), '1') + self.assertEqual(resp.getheader('x-account-meta-two'), '2') def test_bad_metadata(self): if skip: @@ -294,35 +752,65 @@ class TestAccount(unittest.TestCase): resp = retry(post, {'X-Account-Meta-' + ('k' * MAX_META_NAME_LENGTH): 'v'}) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry( post, {'X-Account-Meta-' + ('k' * (MAX_META_NAME_LENGTH + 1)): 'v'}) resp.read() - self.assertEquals(resp.status, 400) + self.assertEqual(resp.status, 400) resp = retry(post, {'X-Account-Meta-Too-Long': 'k' * MAX_META_VALUE_LENGTH}) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry( post, {'X-Account-Meta-Too-Long': 'k' * (MAX_META_VALUE_LENGTH + 1)}) resp.read() - self.assertEquals(resp.status, 400) + self.assertEqual(resp.status, 400) + + def test_bad_metadata2(self): + if skip: + raise SkipTest + + def post(url, token, parsed, conn, extra_headers): + headers = {'X-Auth-Token': token} + headers.update(extra_headers) + conn.request('POST', parsed.path, '', headers) + return check_response(conn) + + # TODO: Find the test that adds these and remove them. + headers = {'x-remove-account-meta-temp-url-key': 'remove', + 'x-remove-account-meta-temp-url-key-2': 'remove'} + resp = retry(post, headers) headers = {} for x in xrange(MAX_META_COUNT): headers['X-Account-Meta-%d' % x] = 'v' resp = retry(post, headers) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) headers = {} for x in xrange(MAX_META_COUNT + 1): headers['X-Account-Meta-%d' % x] = 'v' resp = retry(post, headers) resp.read() - self.assertEquals(resp.status, 400) + self.assertEqual(resp.status, 400) + + def test_bad_metadata3(self): + if skip: + raise SkipTest + + def post(url, token, parsed, conn, extra_headers): + headers = {'X-Auth-Token': token} + headers.update(extra_headers) + conn.request('POST', parsed.path, '', headers) + return check_response(conn) + + # TODO: Find the test that adds these and remove them. + headers = {'x-remove-account-meta-temp-url-key': 'remove', + 'x-remove-account-meta-temp-url-key-2': 'remove'} + resp = retry(post, headers) headers = {} header_value = 'k' * MAX_META_VALUE_LENGTH @@ -337,12 +825,12 @@ class TestAccount(unittest.TestCase): 'v' * (MAX_META_OVERALL_SIZE - size - 1) resp = retry(post, headers) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) headers['X-Account-Meta-k'] = \ 'v' * (MAX_META_OVERALL_SIZE - size) resp = retry(post, headers) resp.read() - self.assertEquals(resp.status, 400) + self.assertEqual(resp.status, 400) if __name__ == '__main__': diff --git a/test/functional/test_container.py b/test/functional/test_container.py index 15f7fc1..91702e9 100755 --- a/test/functional/test_container.py +++ b/test/functional/test_container.py @@ -24,7 +24,7 @@ from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \ MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH from swift_testing import check_response, retry, skip, skip2, skip3, \ - swift_test_perm, web_front_end + swift_test_perm, web_front_end, requires_acls, swift_test_user class TestContainer(unittest.TestCase): @@ -41,7 +41,7 @@ class TestContainer(unittest.TestCase): resp = retry(put) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) def tearDown(self): if skip: @@ -68,7 +68,7 @@ class TestContainer(unittest.TestCase): for obj in objs: resp = retry(delete, obj) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) def delete(url, token, parsed, conn): conn.request('DELETE', parsed.path + '/' + self.name, '', @@ -77,7 +77,7 @@ class TestContainer(unittest.TestCase): resp = retry(delete) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) def test_multi_metadata(self): if skip: @@ -95,19 +95,19 @@ class TestContainer(unittest.TestCase): resp = retry(post, 'X-Container-Meta-One', '1') resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader('x-container-meta-one'), '1') + self.assertEqual(resp.getheader('x-container-meta-one'), '1') resp = retry(post, 'X-Container-Meta-Two', '2') resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader('x-container-meta-one'), '1') - self.assertEquals(resp.getheader('x-container-meta-two'), '2') + self.assertEqual(resp.getheader('x-container-meta-one'), '1') + self.assertEqual(resp.getheader('x-container-meta-two'), '2') def test_unicode_metadata(self): if skip: @@ -128,28 +128,28 @@ class TestContainer(unittest.TestCase): if (web_front_end == 'integral'): resp = retry(post, uni_key, '1') resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader(uni_key.encode('utf-8')), '1') + self.assertEqual(resp.getheader(uni_key.encode('utf-8')), '1') resp = retry(post, 'X-Container-Meta-uni', uni_value) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader('X-Container-Meta-uni'), - uni_value.encode('utf-8')) + self.assertEqual(resp.getheader('X-Container-Meta-uni'), + uni_value.encode('utf-8')) if (web_front_end == 'integral'): resp = retry(post, uni_key, uni_value) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader(uni_key.encode('utf-8')), - uni_value.encode('utf-8')) + self.assertEqual(resp.getheader(uni_key.encode('utf-8')), + uni_value.encode('utf-8')) def test_PUT_metadata(self): if skip: @@ -179,34 +179,34 @@ class TestContainer(unittest.TestCase): name = uuid4().hex resp = retry(put, name, 'Value') resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) resp = retry(head, name) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader('x-container-meta-test'), 'Value') + self.assertEqual(resp.getheader('x-container-meta-test'), 'Value') resp = retry(get, name) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader('x-container-meta-test'), 'Value') + self.assertEqual(resp.getheader('x-container-meta-test'), 'Value') resp = retry(delete, name) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) name = uuid4().hex resp = retry(put, name, '') resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) resp = retry(head, name) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader('x-container-meta-test'), None) + self.assertEqual(resp.getheader('x-container-meta-test'), None) resp = retry(get, name) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader('x-container-meta-test'), None) + self.assertEqual(resp.getheader('x-container-meta-test'), None) resp = retry(delete, name) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) def test_POST_metadata(self): if skip: @@ -231,22 +231,22 @@ class TestContainer(unittest.TestCase): resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader('x-container-meta-test'), None) + self.assertEqual(resp.getheader('x-container-meta-test'), None) resp = retry(get) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader('x-container-meta-test'), None) + self.assertEqual(resp.getheader('x-container-meta-test'), None) resp = retry(post, 'Value') resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader('x-container-meta-test'), 'Value') + self.assertEqual(resp.getheader('x-container-meta-test'), 'Value') resp = retry(get) resp.read() self.assert_(resp.status in (200, 204), resp.status) - self.assertEquals(resp.getheader('x-container-meta-test'), 'Value') + self.assertEqual(resp.getheader('x-container-meta-test'), 'Value') def test_PUT_bad_metadata(self): if skip: @@ -268,38 +268,38 @@ class TestContainer(unittest.TestCase): put, name, {'X-Container-Meta-' + ('k' * MAX_META_NAME_LENGTH): 'v'}) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) resp = retry(delete, name) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) name = uuid4().hex resp = retry( put, name, {'X-Container-Meta-' + ('k' * (MAX_META_NAME_LENGTH + 1)): 'v'}) resp.read() - self.assertEquals(resp.status, 400) + self.assertEqual(resp.status, 400) resp = retry(delete, name) resp.read() - self.assertEquals(resp.status, 404) + self.assertEqual(resp.status, 404) name = uuid4().hex resp = retry( put, name, {'X-Container-Meta-Too-Long': 'k' * MAX_META_VALUE_LENGTH}) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) resp = retry(delete, name) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) name = uuid4().hex resp = retry( put, name, {'X-Container-Meta-Too-Long': 'k' * (MAX_META_VALUE_LENGTH + 1)}) resp.read() - self.assertEquals(resp.status, 400) + self.assertEqual(resp.status, 400) resp = retry(delete, name) resp.read() - self.assertEquals(resp.status, 404) + self.assertEqual(resp.status, 404) name = uuid4().hex headers = {} @@ -307,20 +307,20 @@ class TestContainer(unittest.TestCase): headers['X-Container-Meta-%d' % x] = 'v' resp = retry(put, name, headers) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) resp = retry(delete, name) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) name = uuid4().hex headers = {} for x in xrange(MAX_META_COUNT + 1): headers['X-Container-Meta-%d' % x] = 'v' resp = retry(put, name, headers) resp.read() - self.assertEquals(resp.status, 400) + self.assertEqual(resp.status, 400) resp = retry(delete, name) resp.read() - self.assertEquals(resp.status, 404) + self.assertEqual(resp.status, 404) name = uuid4().hex headers = {} @@ -336,19 +336,19 @@ class TestContainer(unittest.TestCase): 'v' * (MAX_META_OVERALL_SIZE - size - 1) resp = retry(put, name, headers) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) resp = retry(delete, name) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) name = uuid4().hex headers['X-Container-Meta-k'] = \ 'v' * (MAX_META_OVERALL_SIZE - size) resp = retry(put, name, headers) resp.read() - self.assertEquals(resp.status, 400) + self.assertEqual(resp.status, 400) resp = retry(delete, name) resp.read() - self.assertEquals(resp.status, 404) + self.assertEqual(resp.status, 404) def test_POST_bad_metadata(self): if skip: @@ -364,36 +364,56 @@ class TestContainer(unittest.TestCase): post, {'X-Container-Meta-' + ('k' * MAX_META_NAME_LENGTH): 'v'}) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry( post, {'X-Container-Meta-' + ('k' * (MAX_META_NAME_LENGTH + 1)): 'v'}) resp.read() - self.assertEquals(resp.status, 400) + self.assertEqual(resp.status, 400) resp = retry( post, {'X-Container-Meta-Too-Long': 'k' * MAX_META_VALUE_LENGTH}) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry( post, {'X-Container-Meta-Too-Long': 'k' * (MAX_META_VALUE_LENGTH + 1)}) resp.read() - self.assertEquals(resp.status, 400) + self.assertEqual(resp.status, 400) + + def test_POST_bad_metadata2(self): + if skip: + raise SkipTest + + def post(url, token, parsed, conn, extra_headers): + headers = {'X-Auth-Token': token} + headers.update(extra_headers) + conn.request('POST', parsed.path + '/' + self.name, '', headers) + return check_response(conn) headers = {} for x in xrange(MAX_META_COUNT): headers['X-Container-Meta-%d' % x] = 'v' resp = retry(post, headers) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) headers = {} for x in xrange(MAX_META_COUNT + 1): headers['X-Container-Meta-%d' % x] = 'v' resp = retry(post, headers) resp.read() - self.assertEquals(resp.status, 400) + self.assertEqual(resp.status, 400) + + def test_POST_bad_metadata3(self): + if skip: + raise SkipTest + + def post(url, token, parsed, conn, extra_headers): + headers = {'X-Auth-Token': token} + headers.update(extra_headers) + conn.request('POST', parsed.path + '/' + self.name, '', headers) + return check_response(conn) headers = {} header_value = 'k' * MAX_META_VALUE_LENGTH @@ -408,12 +428,12 @@ class TestContainer(unittest.TestCase): 'v' * (MAX_META_OVERALL_SIZE - size - 1) resp = retry(post, headers) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) headers['X-Container-Meta-k'] = \ 'v' * (MAX_META_OVERALL_SIZE - size) resp = retry(post, headers) resp.read() - self.assertEquals(resp.status, 400) + self.assertEqual(resp.status, 400) def test_public_container(self): if skip: @@ -437,10 +457,10 @@ class TestContainer(unittest.TestCase): resp = retry(post) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry(get) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) def post(url, token, parsed, conn): conn.request('POST', parsed.path + '/' + self.name, '', @@ -449,7 +469,7 @@ class TestContainer(unittest.TestCase): resp = retry(post) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) try: resp = retry(get) raise Exception('Should not have been able to GET') @@ -479,7 +499,7 @@ class TestContainer(unittest.TestCase): resp = retry(get2, use_account=2) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # Make the container accessible by the second account def post(url, token, parsed, conn): @@ -491,11 +511,11 @@ class TestContainer(unittest.TestCase): resp = retry(post) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # Ensure we can now use the container with the second account resp = retry(get2, use_account=2) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # Make the container private again def post(url, token, parsed, conn): @@ -506,11 +526,11 @@ class TestContainer(unittest.TestCase): resp = retry(post) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # Ensure we can't access the container with the second account again resp = retry(get2, use_account=2) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) def test_cross_account_public_container(self): if skip or skip2: @@ -535,7 +555,7 @@ class TestContainer(unittest.TestCase): resp = retry(get2, use_account=2) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # Make the container completely public def post(url, token, parsed, conn): @@ -546,11 +566,11 @@ class TestContainer(unittest.TestCase): resp = retry(post) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # Ensure we can now read the container with the second account resp = retry(get2, use_account=2) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # But we shouldn't be able to write with the second account def put2(url, token, parsed, conn): @@ -560,7 +580,7 @@ class TestContainer(unittest.TestCase): resp = retry(put2, use_account=2) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # Now make the container also writeable by the second account def post(url, token, parsed, conn): @@ -571,15 +591,15 @@ class TestContainer(unittest.TestCase): resp = retry(post) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # Ensure we can still read the container with the second account resp = retry(get2, use_account=2) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # And that we can now write with the second account resp = retry(put2, use_account=2) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) def test_nonadmin_user(self): if skip or skip3: @@ -604,7 +624,7 @@ class TestContainer(unittest.TestCase): resp = retry(get3, use_account=3) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # Make the container accessible by the third account def post(url, token, parsed, conn): @@ -615,11 +635,11 @@ class TestContainer(unittest.TestCase): resp = retry(post) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # Ensure we can now read the container with the third account resp = retry(get3, use_account=3) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # But we shouldn't be able to write with the third account def put3(url, token, parsed, conn): @@ -629,7 +649,7 @@ class TestContainer(unittest.TestCase): resp = retry(put3, use_account=3) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # Now make the container also writeable by the third account def post(url, token, parsed, conn): @@ -640,15 +660,666 @@ class TestContainer(unittest.TestCase): resp = retry(post) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # Ensure we can still read the container with the third account resp = retry(get3, use_account=3) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # And that we can now write with the third account resp = retry(put3, use_account=3) resp.read() + self.assertEqual(resp.status, 201) + + @requires_acls + def test_read_only_acl_listings(self): + if skip3: + raise SkipTest + + def get(url, token, parsed, conn): + conn.request('GET', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + + def post_account(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + def put(url, token, parsed, conn, name): + conn.request('PUT', parsed.path + '/%s' % name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + # cannot list containers + resp = retry(get, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # grant read-only access + acl_user = swift_test_user[2] + acl = {'read-only': [acl_user]} + headers = {'x-account-access-control': json.dumps(acl)} + resp = retry(post_account, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # read-only can list containers + resp = retry(get, use_account=3) + listing = resp.read() + self.assertEquals(resp.status, 200) + self.assert_(self.name in listing) + + # read-only can not create containers + new_container_name = str(uuid4()) + resp = retry(put, new_container_name, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # but it can see newly created ones + resp = retry(put, new_container_name, use_account=1) + resp.read() self.assertEquals(resp.status, 201) + resp = retry(get, use_account=3) + listing = resp.read() + self.assertEquals(resp.status, 200) + self.assert_(new_container_name in listing) + + @requires_acls + def test_read_only_acl_metadata(self): + if skip3: + raise SkipTest + + def get(url, token, parsed, conn, name): + conn.request('GET', parsed.path + '/%s' % name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + def post_account(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + def post(url, token, parsed, conn, name, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path + '/%s' % name, '', new_headers) + return check_response(conn) + + # add some metadata + value = str(uuid4()) + headers = {'x-container-meta-test': value} + resp = retry(post, self.name, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + resp = retry(get, self.name, use_account=1) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) + + # cannot see metadata + resp = retry(get, self.name, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # grant read-only access + acl_user = swift_test_user[2] + acl = {'read-only': [acl_user]} + headers = {'x-account-access-control': json.dumps(acl)} + resp = retry(post_account, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # read-only can NOT write container metadata + new_value = str(uuid4()) + headers = {'x-container-meta-test': new_value} + resp = retry(post, self.name, headers=headers, use_account=3) + resp.read() + self.assertEqual(resp.status, 403) + + # read-only can read container metadata + resp = retry(get, self.name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) + + @requires_acls + def test_read_write_acl_listings(self): + if skip3: + raise SkipTest + + def get(url, token, parsed, conn): + conn.request('GET', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + + def post(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + def put(url, token, parsed, conn, name): + conn.request('PUT', parsed.path + '/%s' % name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + def delete(url, token, parsed, conn, name): + conn.request('DELETE', parsed.path + '/%s' % name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + # cannot list containers + resp = retry(get, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # grant read-write access + acl_user = swift_test_user[2] + acl = {'read-write': [acl_user]} + headers = {'x-account-access-control': json.dumps(acl)} + resp = retry(post, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # can list containers + resp = retry(get, use_account=3) + listing = resp.read() + self.assertEquals(resp.status, 200) + self.assert_(self.name in listing) + + # can create new containers + new_container_name = str(uuid4()) + resp = retry(put, new_container_name, use_account=3) + resp.read() + self.assertEquals(resp.status, 201) + resp = retry(get, use_account=3) + listing = resp.read() + self.assertEquals(resp.status, 200) + self.assert_(new_container_name in listing) + + # can also delete them + resp = retry(delete, new_container_name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(get, use_account=3) + listing = resp.read() + self.assertEquals(resp.status, 200) + self.assert_(new_container_name not in listing) + + # even if they didn't create them + empty_container_name = str(uuid4()) + resp = retry(put, empty_container_name, use_account=1) + resp.read() + self.assertEquals(resp.status, 201) + resp = retry(delete, empty_container_name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + + @requires_acls + def test_read_write_acl_metadata(self): + if skip3: + raise SkipTest + + def get(url, token, parsed, conn, name): + conn.request('GET', parsed.path + '/%s' % name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + def post_account(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + def post(url, token, parsed, conn, name, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path + '/%s' % name, '', new_headers) + return check_response(conn) + + # add some metadata + value = str(uuid4()) + headers = {'x-container-meta-test': value} + resp = retry(post, self.name, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + resp = retry(get, self.name, use_account=1) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) + + # cannot see metadata + resp = retry(get, self.name, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # grant read-write access + acl_user = swift_test_user[2] + acl = {'read-write': [acl_user]} + headers = {'x-account-access-control': json.dumps(acl)} + resp = retry(post_account, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # read-write can read container metadata + resp = retry(get, self.name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) + + # read-write can also write container metadata + new_value = str(uuid4()) + headers = {'x-container-meta-test': new_value} + resp = retry(post, self.name, headers=headers, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(get, self.name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Meta-Test'), new_value) + + # and remove it + headers = {'x-remove-container-meta-test': 'true'} + resp = retry(post, self.name, headers=headers, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(get, self.name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Meta-Test'), None) + + @requires_acls + def test_admin_acl_listing(self): + if skip3: + raise SkipTest + + def get(url, token, parsed, conn): + conn.request('GET', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + + def post(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + def put(url, token, parsed, conn, name): + conn.request('PUT', parsed.path + '/%s' % name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + def delete(url, token, parsed, conn, name): + conn.request('DELETE', parsed.path + '/%s' % name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + # cannot list containers + resp = retry(get, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # grant admin access + acl_user = swift_test_user[2] + acl = {'admin': [acl_user]} + headers = {'x-account-access-control': json.dumps(acl)} + resp = retry(post, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # can list containers + resp = retry(get, use_account=3) + listing = resp.read() + self.assertEquals(resp.status, 200) + self.assert_(self.name in listing) + + # can create new containers + new_container_name = str(uuid4()) + resp = retry(put, new_container_name, use_account=3) + resp.read() + self.assertEquals(resp.status, 201) + resp = retry(get, use_account=3) + listing = resp.read() + self.assertEquals(resp.status, 200) + self.assert_(new_container_name in listing) + + # can also delete them + resp = retry(delete, new_container_name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(get, use_account=3) + listing = resp.read() + self.assertEquals(resp.status, 200) + self.assert_(new_container_name not in listing) + + # even if they didn't create them + empty_container_name = str(uuid4()) + resp = retry(put, empty_container_name, use_account=1) + resp.read() + self.assertEquals(resp.status, 201) + resp = retry(delete, empty_container_name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + + @requires_acls + def test_admin_acl_metadata(self): + if skip3: + raise SkipTest + + def get(url, token, parsed, conn, name): + conn.request('GET', parsed.path + '/%s' % name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + def post_account(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + def post(url, token, parsed, conn, name, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path + '/%s' % name, '', new_headers) + return check_response(conn) + + # add some metadata + value = str(uuid4()) + headers = {'x-container-meta-test': value} + resp = retry(post, self.name, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + resp = retry(get, self.name, use_account=1) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) + + # cannot see metadata + resp = retry(get, self.name, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # grant access + acl_user = swift_test_user[2] + acl = {'admin': [acl_user]} + headers = {'x-account-access-control': json.dumps(acl)} + resp = retry(post_account, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # can read container metadata + resp = retry(get, self.name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) + + # can also write container metadata + new_value = str(uuid4()) + headers = {'x-container-meta-test': new_value} + resp = retry(post, self.name, headers=headers, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(get, self.name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Meta-Test'), new_value) + + # and remove it + headers = {'x-remove-container-meta-test': 'true'} + resp = retry(post, self.name, headers=headers, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(get, self.name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Meta-Test'), None) + + @requires_acls + def test_protected_container_sync(self): + if skip3: + raise SkipTest + + def get(url, token, parsed, conn, name): + conn.request('GET', parsed.path + '/%s' % name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + def post_account(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + def post(url, token, parsed, conn, name, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path + '/%s' % name, '', new_headers) + return check_response(conn) + + # add some metadata + value = str(uuid4()) + headers = { + 'x-container-sync-key': 'secret', + 'x-container-meta-test': value, + } + resp = retry(post, self.name, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + resp = retry(get, self.name, use_account=1) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Sync-Key'), 'secret') + self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) + + # grant read-only access + acl_user = swift_test_user[2] + acl = {'read-only': [acl_user]} + headers = {'x-account-access-control': json.dumps(acl)} + resp = retry(post_account, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # can read container metadata + resp = retry(get, self.name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) + # but not sync-key + self.assertEqual(resp.getheader('X-Container-Sync-Key'), None) + + # and can not write + headers = {'x-container-sync-key': str(uuid4())} + resp = retry(post, self.name, headers=headers, use_account=3) + resp.read() + self.assertEqual(resp.status, 403) + + # grant read-write access + acl_user = swift_test_user[2] + acl = {'read-write': [acl_user]} + headers = {'x-account-access-control': json.dumps(acl)} + resp = retry(post_account, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # can read container metadata + resp = retry(get, self.name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) + # but not sync-key + self.assertEqual(resp.getheader('X-Container-Sync-Key'), None) + + # sanity check sync-key w/ account1 + resp = retry(get, self.name, use_account=1) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Sync-Key'), 'secret') + + # and can write + new_value = str(uuid4()) + headers = { + 'x-container-sync-key': str(uuid4()), + 'x-container-meta-test': new_value, + } + resp = retry(post, self.name, headers=headers, use_account=3) + resp.read() + self.assertEqual(resp.status, 204) + resp = retry(get, self.name, use_account=1) # validate w/ account1 + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Meta-Test'), new_value) + # but can not write sync-key + self.assertEqual(resp.getheader('X-Container-Sync-Key'), 'secret') + + # grant admin access + acl_user = swift_test_user[2] + acl = {'admin': [acl_user]} + headers = {'x-account-access-control': json.dumps(acl)} + resp = retry(post_account, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # admin can read container metadata + resp = retry(get, self.name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Meta-Test'), new_value) + # and ALSO sync-key + self.assertEqual(resp.getheader('X-Container-Sync-Key'), 'secret') + + # admin tester3 can even change sync-key + new_secret = str(uuid4()) + headers = {'x-container-sync-key': new_secret} + resp = retry(post, self.name, headers=headers, use_account=3) + resp.read() + self.assertEqual(resp.status, 204) + resp = retry(get, self.name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Sync-Key'), new_secret) + + @requires_acls + def test_protected_container_acl(self): + if skip3: + raise SkipTest + + def get(url, token, parsed, conn, name): + conn.request('GET', parsed.path + '/%s' % name, '', + {'X-Auth-Token': token}) + return check_response(conn) + + def post_account(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + def post(url, token, parsed, conn, name, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path + '/%s' % name, '', new_headers) + return check_response(conn) + + # add some container acls + value = str(uuid4()) + headers = { + 'x-container-read': 'jdoe', + 'x-container-write': 'jdoe', + 'x-container-meta-test': value, + } + resp = retry(post, self.name, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + resp = retry(get, self.name, use_account=1) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Read'), 'jdoe') + self.assertEqual(resp.getheader('X-Container-Write'), 'jdoe') + self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) + + # grant read-only access + acl_user = swift_test_user[2] + acl = {'read-only': [acl_user]} + headers = {'x-account-access-control': json.dumps(acl)} + resp = retry(post_account, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # can read container metadata + resp = retry(get, self.name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) + # but not container acl + self.assertEqual(resp.getheader('X-Container-Read'), None) + self.assertEqual(resp.getheader('X-Container-Write'), None) + + # and can not write + headers = { + 'x-container-read': 'frank', + 'x-container-write': 'frank', + } + resp = retry(post, self.name, headers=headers, use_account=3) + resp.read() + self.assertEqual(resp.status, 403) + + # grant read-write access + acl_user = swift_test_user[2] + acl = {'read-write': [acl_user]} + headers = {'x-account-access-control': json.dumps(acl)} + resp = retry(post_account, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # can read container metadata + resp = retry(get, self.name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) + # but not container acl + self.assertEqual(resp.getheader('X-Container-Read'), None) + self.assertEqual(resp.getheader('X-Container-Write'), None) + + # sanity check container acls with account1 + resp = retry(get, self.name, use_account=1) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Read'), 'jdoe') + self.assertEqual(resp.getheader('X-Container-Write'), 'jdoe') + + # and can write + new_value = str(uuid4()) + headers = { + 'x-container-read': 'frank', + 'x-container-write': 'frank', + 'x-container-meta-test': new_value, + } + resp = retry(post, self.name, headers=headers, use_account=3) + resp.read() + self.assertEqual(resp.status, 204) + resp = retry(get, self.name, use_account=1) # validate w/ account1 + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Meta-Test'), new_value) + # but can not write container acls + self.assertEqual(resp.getheader('X-Container-Read'), 'jdoe') + self.assertEqual(resp.getheader('X-Container-Write'), 'jdoe') + + # grant admin access + acl_user = swift_test_user[2] + acl = {'admin': [acl_user]} + headers = {'x-account-access-control': json.dumps(acl)} + resp = retry(post_account, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # admin can read container metadata + resp = retry(get, self.name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Meta-Test'), new_value) + # and ALSO container acls + self.assertEqual(resp.getheader('X-Container-Read'), 'jdoe') + self.assertEqual(resp.getheader('X-Container-Write'), 'jdoe') + + # admin tester3 can even change container acls + new_value = str(uuid4()) + headers = { + 'x-container-read': '.r:*', + } + resp = retry(post, self.name, headers=headers, use_account=3) + resp.read() + self.assertEqual(resp.status, 204) + resp = retry(get, self.name, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEqual(resp.getheader('X-Container-Read'), '.r:*') def test_long_name_content_type(self): if skip: @@ -662,9 +1333,9 @@ class TestContainer(unittest.TestCase): resp = retry(put) resp.read() - self.assertEquals(resp.status, 400) - self.assertEquals(resp.getheader('Content-Type'), - 'text/html; charset=UTF-8') + self.assertEqual(resp.status, 400) + self.assertEqual(resp.getheader('Content-Type'), + 'text/html; charset=UTF-8') def test_null_name(self): if skip: @@ -677,10 +1348,10 @@ class TestContainer(unittest.TestCase): resp = retry(put) if (web_front_end == 'apache2'): - self.assertEquals(resp.status, 404) + self.assertEqual(resp.status, 404) else: - self.assertEquals(resp.read(), 'Invalid UTF8 or contains NULL') - self.assertEquals(resp.status, 412) + self.assertEqual(resp.read(), 'Invalid UTF8 or contains NULL') + self.assertEqual(resp.status, 412) if __name__ == '__main__': diff --git a/test/functional/test_object.py b/test/functional/test_object.py index dad8635..675de30 100755 --- a/test/functional/test_object.py +++ b/test/functional/test_object.py @@ -19,8 +19,10 @@ import unittest from nose import SkipTest from uuid import uuid4 +from swift.common.utils import json + from swift_testing import check_response, retry, skip, skip3, \ - swift_test_perm, web_front_end + swift_test_perm, web_front_end, requires_acls, swift_test_user class TestObject(unittest.TestCase): @@ -36,7 +38,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(put) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) self.obj = uuid4().hex def put(url, token, parsed, conn): @@ -46,7 +48,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(put) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) def tearDown(self): if skip: @@ -66,13 +68,13 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(list) object_listing = resp.read() - self.assertEquals(resp.status, 200) + self.assertEqual(resp.status, 200) # iterate over object listing and delete all objects for obj in object_listing.splitlines(): resp = retry(delete, obj) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # delete the container def delete(url, token, parsed, conn): @@ -81,7 +83,33 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(delete) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) + + def test_if_none_match(self): + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s/%s' % ( + parsed.path, self.container, 'if_none_match_test'), '', + {'X-Auth-Token': token, + 'Content-Length': '0', + 'If-None-Match': '*'}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 201) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 412) + + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s/%s' % ( + parsed.path, self.container, 'if_none_match_test'), '', + {'X-Auth-Token': token, + 'Content-Length': '0', + 'If-None-Match': 'somethingelse'}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 400) def test_copy_object(self): if skip: @@ -98,8 +126,8 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(get_source) source_contents = resp.read() - self.assertEquals(resp.status, 200) - self.assertEquals(source_contents, 'test') + self.assertEqual(resp.status, 200) + self.assertEqual(source_contents, 'test') # copy source to dest with X-Copy-From def put(url, token, parsed, conn): @@ -110,7 +138,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(put) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) # contents of dest should be the same as source def get_dest(url, token, parsed, conn): @@ -120,8 +148,8 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(get_dest) dest_contents = resp.read() - self.assertEquals(resp.status, 200) - self.assertEquals(dest_contents, source_contents) + self.assertEqual(resp.status, 200) + self.assertEqual(dest_contents, source_contents) # delete the copy def delete(url, token, parsed, conn): @@ -130,11 +158,11 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(delete) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # verify dest does not exist resp = retry(get_dest) resp.read() - self.assertEquals(resp.status, 404) + self.assertEqual(resp.status, 404) # copy source to dest with COPY def copy(url, token, parsed, conn): @@ -144,18 +172,18 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(copy) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) # contents of dest should be the same as source resp = retry(get_dest) dest_contents = resp.read() - self.assertEquals(resp.status, 200) - self.assertEquals(dest_contents, source_contents) + self.assertEqual(resp.status, 200) + self.assertEqual(dest_contents, source_contents) # delete the copy resp = retry(delete) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) def test_public_object(self): if skip: @@ -178,10 +206,10 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(post) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry(get) resp.read() - self.assertEquals(resp.status, 200) + self.assertEqual(resp.status, 200) def post(url, token, parsed, conn): conn.request('POST', parsed.path + '/' + self.container, '', @@ -189,7 +217,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(post) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) try: resp = retry(get) raise Exception('Should not have been able to GET') @@ -208,7 +236,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(get, use_account=3) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # create a shared container writable by account3 shared_container = uuid4().hex @@ -222,7 +250,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(put) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) # verify third account can not copy from private container def copy(url, token, parsed, conn): @@ -234,7 +262,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(copy, use_account=3) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # verify third account can write "obj1" to shared container def put(url, token, parsed, conn): @@ -244,7 +272,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(put, use_account=3) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) # verify third account can copy "obj1" to shared container def copy2(url, token, parsed, conn): @@ -255,7 +283,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(copy2, use_account=3) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) # verify third account STILL can not copy from private container def copy3(url, token, parsed, conn): @@ -267,7 +295,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(copy3, use_account=3) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # clean up "obj1" def delete(url, token, parsed, conn): @@ -277,7 +305,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(delete) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # clean up shared_container def delete(url, token, parsed, conn): @@ -287,8 +315,251 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(delete) resp.read() + self.assertEqual(resp.status, 204) + + @requires_acls + def test_read_only(self): + if skip3: + raise SkipTest + + def get_listing(url, token, parsed, conn): + conn.request('GET', '%s/%s' % (parsed.path, self.container), '', + {'X-Auth-Token': token}) + return check_response(conn) + + def post_account(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + def get(url, token, parsed, conn, name): + conn.request('GET', '%s/%s/%s' % ( + parsed.path, self.container, name), '', + {'X-Auth-Token': token}) + return check_response(conn) + + def put(url, token, parsed, conn, name): + conn.request('PUT', '%s/%s/%s' % ( + parsed.path, self.container, name), 'test', + {'X-Auth-Token': token}) + return check_response(conn) + + def delete(url, token, parsed, conn, name): + conn.request('PUT', '%s/%s/%s' % ( + parsed.path, self.container, name), '', + {'X-Auth-Token': token}) + return check_response(conn) + + # cannot list objects + resp = retry(get_listing, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # cannot get object + resp = retry(get, self.obj, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # grant read-only access + acl_user = swift_test_user[2] + acl = {'read-only': [acl_user]} + headers = {'x-account-access-control': json.dumps(acl)} + resp = retry(post_account, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # can list objects + resp = retry(get_listing, use_account=3) + listing = resp.read() + self.assertEquals(resp.status, 200) + self.assert_(self.obj in listing) + + # can get object + resp = retry(get, self.obj, use_account=3) + body = resp.read() + self.assertEquals(resp.status, 200) + self.assertEquals(body, 'test') + + # can not put an object + obj_name = str(uuid4()) + resp = retry(put, obj_name, use_account=3) + body = resp.read() + self.assertEquals(resp.status, 403) + + # can not delete an object + resp = retry(delete, self.obj, use_account=3) + body = resp.read() + self.assertEquals(resp.status, 403) + + # sanity with account1 + resp = retry(get_listing, use_account=3) + listing = resp.read() + self.assertEquals(resp.status, 200) + self.assert_(obj_name not in listing) + self.assert_(self.obj in listing) + + @requires_acls + def test_read_write(self): + if skip3: + raise SkipTest + + def get_listing(url, token, parsed, conn): + conn.request('GET', '%s/%s' % (parsed.path, self.container), '', + {'X-Auth-Token': token}) + return check_response(conn) + + def post_account(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + def get(url, token, parsed, conn, name): + conn.request('GET', '%s/%s/%s' % ( + parsed.path, self.container, name), '', + {'X-Auth-Token': token}) + return check_response(conn) + + def put(url, token, parsed, conn, name): + conn.request('PUT', '%s/%s/%s' % ( + parsed.path, self.container, name), 'test', + {'X-Auth-Token': token}) + return check_response(conn) + + def delete(url, token, parsed, conn, name): + conn.request('DELETE', '%s/%s/%s' % ( + parsed.path, self.container, name), '', + {'X-Auth-Token': token}) + return check_response(conn) + + # cannot list objects + resp = retry(get_listing, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # cannot get object + resp = retry(get, self.obj, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # grant read-write access + acl_user = swift_test_user[2] + acl = {'read-write': [acl_user]} + headers = {'x-account-access-control': json.dumps(acl)} + resp = retry(post_account, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # can list objects + resp = retry(get_listing, use_account=3) + listing = resp.read() + self.assertEquals(resp.status, 200) + self.assert_(self.obj in listing) + + # can get object + resp = retry(get, self.obj, use_account=3) + body = resp.read() + self.assertEquals(resp.status, 200) + self.assertEquals(body, 'test') + + # can put an object + obj_name = str(uuid4()) + resp = retry(put, obj_name, use_account=3) + body = resp.read() + self.assertEquals(resp.status, 201) + + # can delete an object + resp = retry(delete, self.obj, use_account=3) + body = resp.read() self.assertEquals(resp.status, 204) + # sanity with account1 + resp = retry(get_listing, use_account=3) + listing = resp.read() + self.assertEquals(resp.status, 200) + self.assert_(obj_name in listing) + self.assert_(self.obj not in listing) + + @requires_acls + def test_admin(self): + if skip3: + raise SkipTest + + def get_listing(url, token, parsed, conn): + conn.request('GET', '%s/%s' % (parsed.path, self.container), '', + {'X-Auth-Token': token}) + return check_response(conn) + + def post_account(url, token, parsed, conn, headers): + new_headers = dict({'X-Auth-Token': token}, **headers) + conn.request('POST', parsed.path, '', new_headers) + return check_response(conn) + + def get(url, token, parsed, conn, name): + conn.request('GET', '%s/%s/%s' % ( + parsed.path, self.container, name), '', + {'X-Auth-Token': token}) + return check_response(conn) + + def put(url, token, parsed, conn, name): + conn.request('PUT', '%s/%s/%s' % ( + parsed.path, self.container, name), 'test', + {'X-Auth-Token': token}) + return check_response(conn) + + def delete(url, token, parsed, conn, name): + conn.request('DELETE', '%s/%s/%s' % ( + parsed.path, self.container, name), '', + {'X-Auth-Token': token}) + return check_response(conn) + + # cannot list objects + resp = retry(get_listing, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # cannot get object + resp = retry(get, self.obj, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # grant admin access + acl_user = swift_test_user[2] + acl = {'admin': [acl_user]} + headers = {'x-account-access-control': json.dumps(acl)} + resp = retry(post_account, headers=headers, use_account=1) + resp.read() + self.assertEqual(resp.status, 204) + + # can list objects + resp = retry(get_listing, use_account=3) + listing = resp.read() + self.assertEquals(resp.status, 200) + self.assert_(self.obj in listing) + + # can get object + resp = retry(get, self.obj, use_account=3) + body = resp.read() + self.assertEquals(resp.status, 200) + self.assertEquals(body, 'test') + + # can put an object + obj_name = str(uuid4()) + resp = retry(put, obj_name, use_account=3) + body = resp.read() + self.assertEquals(resp.status, 201) + + # can delete an object + resp = retry(delete, self.obj, use_account=3) + body = resp.read() + self.assertEquals(resp.status, 204) + + # sanity with account1 + resp = retry(get_listing, use_account=3) + listing = resp.read() + self.assertEquals(resp.status, 200) + self.assert_(obj_name in listing) + self.assert_(self.obj not in listing) + def test_manifest(self): if skip: raise SkipTest @@ -306,7 +577,7 @@ class TestObject(unittest.TestCase): for objnum in xrange(len(segments1)): resp = retry(put, objnum) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) # Upload the manifest def put(url, token, parsed, conn): @@ -318,7 +589,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(put) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) # Get the manifest (should get all the segments as the body) def get(url, token, parsed, conn): @@ -326,9 +597,9 @@ class TestObject(unittest.TestCase): parsed.path, self.container), '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(get) - self.assertEquals(resp.read(), ''.join(segments1)) - self.assertEquals(resp.status, 200) - self.assertEquals(resp.getheader('content-type'), 'text/jibberish') + self.assertEqual(resp.read(), ''.join(segments1)) + self.assertEqual(resp.status, 200) + self.assertEqual(resp.getheader('content-type'), 'text/jibberish') # Get with a range at the start of the second segment def get(url, token, parsed, conn): @@ -337,8 +608,8 @@ class TestObject(unittest.TestCase): 'X-Auth-Token': token, 'Range': 'bytes=3-'}) return check_response(conn) resp = retry(get) - self.assertEquals(resp.read(), ''.join(segments1[1:])) - self.assertEquals(resp.status, 206) + self.assertEqual(resp.read(), ''.join(segments1[1:])) + self.assertEqual(resp.status, 206) # Get with a range in the middle of the second segment def get(url, token, parsed, conn): @@ -347,8 +618,8 @@ class TestObject(unittest.TestCase): 'X-Auth-Token': token, 'Range': 'bytes=5-'}) return check_response(conn) resp = retry(get) - self.assertEquals(resp.read(), ''.join(segments1)[5:]) - self.assertEquals(resp.status, 206) + self.assertEqual(resp.read(), ''.join(segments1)[5:]) + self.assertEqual(resp.status, 206) # Get with a full start and stop range def get(url, token, parsed, conn): @@ -357,8 +628,8 @@ class TestObject(unittest.TestCase): 'X-Auth-Token': token, 'Range': 'bytes=5-10'}) return check_response(conn) resp = retry(get) - self.assertEquals(resp.read(), ''.join(segments1)[5:11]) - self.assertEquals(resp.status, 206) + self.assertEqual(resp.read(), ''.join(segments1)[5:11]) + self.assertEqual(resp.status, 206) # Upload the second set of segments def put(url, token, parsed, conn, objnum): @@ -369,7 +640,7 @@ class TestObject(unittest.TestCase): for objnum in xrange(len(segments2)): resp = retry(put, objnum) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) # Get the manifest (should still be the first segments of course) def get(url, token, parsed, conn): @@ -377,8 +648,8 @@ class TestObject(unittest.TestCase): parsed.path, self.container), '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(get) - self.assertEquals(resp.read(), ''.join(segments1)) - self.assertEquals(resp.status, 200) + self.assertEqual(resp.read(), ''.join(segments1)) + self.assertEqual(resp.status, 200) # Update the manifest def put(url, token, parsed, conn): @@ -390,7 +661,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(put) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) # Get the manifest (should be the second set of segments now) def get(url, token, parsed, conn): @@ -398,8 +669,8 @@ class TestObject(unittest.TestCase): parsed.path, self.container), '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(get) - self.assertEquals(resp.read(), ''.join(segments2)) - self.assertEquals(resp.status, 200) + self.assertEqual(resp.read(), ''.join(segments2)) + self.assertEqual(resp.status, 200) if not skip3: @@ -410,7 +681,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(get, use_account=3) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # Grant access to the third account def post(url, token, parsed, conn): @@ -420,7 +691,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(post) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # The third account should be able to get the manifest now def get(url, token, parsed, conn): @@ -428,8 +699,8 @@ class TestObject(unittest.TestCase): parsed.path, self.container), '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(get, use_account=3) - self.assertEquals(resp.read(), ''.join(segments2)) - self.assertEquals(resp.status, 200) + self.assertEqual(resp.read(), ''.join(segments2)) + self.assertEqual(resp.status, 200) # Create another container for the third set of segments acontainer = uuid4().hex @@ -440,7 +711,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(put) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) # Upload the third set of segments in the other container def put(url, token, parsed, conn, objnum): @@ -451,7 +722,7 @@ class TestObject(unittest.TestCase): for objnum in xrange(len(segments3)): resp = retry(put, objnum) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) # Update the manifest def put(url, token, parsed, conn): @@ -463,7 +734,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(put) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) # Get the manifest to ensure it's the third set of segments def get(url, token, parsed, conn): @@ -471,8 +742,8 @@ class TestObject(unittest.TestCase): parsed.path, self.container), '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(get) - self.assertEquals(resp.read(), ''.join(segments3)) - self.assertEquals(resp.status, 200) + self.assertEqual(resp.read(), ''.join(segments3)) + self.assertEqual(resp.status, 200) if not skip3: @@ -486,7 +757,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(get, use_account=3) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # Grant access to the third account def post(url, token, parsed, conn): @@ -496,7 +767,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(post) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # The third account should be able to get the manifest now def get(url, token, parsed, conn): @@ -504,8 +775,8 @@ class TestObject(unittest.TestCase): parsed.path, self.container), '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(get, use_account=3) - self.assertEquals(resp.read(), ''.join(segments3)) - self.assertEquals(resp.status, 200) + self.assertEqual(resp.read(), ''.join(segments3)) + self.assertEqual(resp.status, 200) # Delete the manifest def delete(url, token, parsed, conn, objnum): @@ -515,7 +786,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(delete, objnum) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # Delete the third set of segments def delete(url, token, parsed, conn, objnum): @@ -526,7 +797,7 @@ class TestObject(unittest.TestCase): for objnum in xrange(len(segments3)): resp = retry(delete, objnum) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # Delete the second set of segments def delete(url, token, parsed, conn, objnum): @@ -537,7 +808,7 @@ class TestObject(unittest.TestCase): for objnum in xrange(len(segments2)): resp = retry(delete, objnum) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # Delete the first set of segments def delete(url, token, parsed, conn, objnum): @@ -548,7 +819,7 @@ class TestObject(unittest.TestCase): for objnum in xrange(len(segments1)): resp = retry(delete, objnum) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # Delete the extra container def delete(url, token, parsed, conn): @@ -557,7 +828,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(delete) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) def test_delete_content_type(self): if skip: @@ -569,7 +840,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(put) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) def delete(url, token, parsed, conn): conn.request('DELETE', '%s/%s/hi' % (parsed.path, self.container), @@ -577,9 +848,9 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(delete) resp.read() - self.assertEquals(resp.status, 204) - self.assertEquals(resp.getheader('Content-Type'), - 'text/html; charset=UTF-8') + self.assertEqual(resp.status, 204) + self.assertEqual(resp.getheader('Content-Type'), + 'text/html; charset=UTF-8') def test_delete_if_delete_at_bad(self): if skip: @@ -592,7 +863,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(put) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) def delete(url, token, parsed, conn): conn.request('DELETE', '%s/%s/hi' % (parsed.path, self.container), @@ -601,7 +872,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(delete) resp.read() - self.assertEquals(resp.status, 400) + self.assertEqual(resp.status, 400) def test_null_name(self): if skip: @@ -614,10 +885,121 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(put) if (web_front_end == 'apache2'): - self.assertEquals(resp.status, 404) + self.assertEqual(resp.status, 404) else: - self.assertEquals(resp.read(), 'Invalid UTF8 or contains NULL') - self.assertEquals(resp.status, 412) + self.assertEqual(resp.read(), 'Invalid UTF8 or contains NULL') + self.assertEqual(resp.status, 412) + + def test_cors(self): + if skip: + raise SkipTest + + def is_strict_mode(url, token, parsed, conn): + conn.request('GET', '/info') + resp = conn.getresponse() + if resp.status // 100 == 2: + info = json.loads(resp.read()) + return info.get('swift', {}).get('strict_cors_mode', False) + return False + + def put_cors_cont(url, token, parsed, conn, orig): + conn.request( + 'PUT', '%s/%s' % (parsed.path, self.container), + '', {'X-Auth-Token': token, + 'X-Container-Meta-Access-Control-Allow-Origin': orig}) + return check_response(conn) + + def put_obj(url, token, parsed, conn, obj): + conn.request( + 'PUT', '%s/%s/%s' % (parsed.path, self.container, obj), + 'test', {'X-Auth-Token': token}) + return check_response(conn) + + def check_cors(url, token, parsed, conn, + method, obj, headers): + if method != 'OPTIONS': + headers['X-Auth-Token'] = token + conn.request( + method, '%s/%s/%s' % (parsed.path, self.container, obj), + '', headers) + return conn.getresponse() + + strict_cors = retry(is_strict_mode) + + resp = retry(put_cors_cont, '*') + resp.read() + self.assertEquals(resp.status // 100, 2) + + resp = retry(put_obj, 'cat') + resp.read() + self.assertEquals(resp.status // 100, 2) + + resp = retry(check_cors, + 'OPTIONS', 'cat', {'Origin': 'http://m.com'}) + self.assertEquals(resp.status, 401) + + resp = retry(check_cors, + 'OPTIONS', 'cat', + {'Origin': 'http://m.com', + 'Access-Control-Request-Method': 'GET'}) + + self.assertEquals(resp.status, 200) + resp.read() + headers = dict((k.lower(), v) for k, v in resp.getheaders()) + self.assertEquals(headers.get('access-control-allow-origin'), + '*') + + resp = retry(check_cors, + 'GET', 'cat', {'Origin': 'http://m.com'}) + self.assertEquals(resp.status, 200) + headers = dict((k.lower(), v) for k, v in resp.getheaders()) + self.assertEquals(headers.get('access-control-allow-origin'), + '*') + + resp = retry(check_cors, + 'GET', 'cat', {'Origin': 'http://m.com', + 'X-Web-Mode': 'True'}) + self.assertEquals(resp.status, 200) + headers = dict((k.lower(), v) for k, v in resp.getheaders()) + self.assertEquals(headers.get('access-control-allow-origin'), + '*') + + #################### + + resp = retry(put_cors_cont, 'http://secret.com') + resp.read() + self.assertEquals(resp.status // 100, 2) + + resp = retry(check_cors, + 'OPTIONS', 'cat', + {'Origin': 'http://m.com', + 'Access-Control-Request-Method': 'GET'}) + resp.read() + self.assertEquals(resp.status, 401) + + if strict_cors: + resp = retry(check_cors, + 'GET', 'cat', {'Origin': 'http://m.com'}) + resp.read() + self.assertEquals(resp.status, 200) + headers = dict((k.lower(), v) for k, v in resp.getheaders()) + self.assertTrue('access-control-allow-origin' not in headers) + + resp = retry(check_cors, + 'GET', 'cat', {'Origin': 'http://secret.com'}) + resp.read() + self.assertEquals(resp.status, 200) + headers = dict((k.lower(), v) for k, v in resp.getheaders()) + self.assertEquals(headers.get('access-control-allow-origin'), + 'http://secret.com') + else: + resp = retry(check_cors, + 'GET', 'cat', {'Origin': 'http://m.com'}) + resp.read() + self.assertEquals(resp.status, 200) + headers = dict((k.lower(), v) for k, v in resp.getheaders()) + self.assertEquals(headers.get('access-control-allow-origin'), + 'http://m.com') if __name__ == '__main__': diff --git a/test/functional/tests.py b/test/functional/tests.py index 0d9a9ef..ad87d7e 100644 --- a/test/functional/tests.py +++ b/test/functional/tests.py @@ -19,14 +19,16 @@ from datetime import datetime import os import hashlib +import hmac import json import locale import random import StringIO import time import threading -import uuid import unittest +import urllib +import uuid from nose import SkipTest from ConfigParser import ConfigParser @@ -36,7 +38,7 @@ from test.functional.swift_test_client import Account, Connection, File, \ from swift.common.constraints import MAX_FILE_SIZE, MAX_META_NAME_LENGTH, \ MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE, \ MAX_OBJECT_NAME_LENGTH, CONTAINER_LISTING_LIMIT, ACCOUNT_LISTING_LIMIT, \ - MAX_ACCOUNT_NAME_LENGTH, MAX_CONTAINER_NAME_LENGTH + MAX_ACCOUNT_NAME_LENGTH, MAX_CONTAINER_NAME_LENGTH, MAX_HEADER_SIZE from gluster.swift.common.constraints import \ set_object_name_component_length, get_object_name_component_length @@ -50,7 +52,8 @@ default_constraints = dict(( ('container_listing_limit', CONTAINER_LISTING_LIMIT), ('account_listing_limit', ACCOUNT_LISTING_LIMIT), ('max_account_name_length', MAX_ACCOUNT_NAME_LENGTH), - ('max_container_name_length', MAX_CONTAINER_NAME_LENGTH))) + ('max_container_name_length', MAX_CONTAINER_NAME_LENGTH), + ('max_header_size', MAX_HEADER_SIZE))) constraints_conf = ConfigParser() conf_exists = constraints_conf.read('/etc/swift/swift.conf') # Constraints are set first from the test config, then from @@ -285,7 +288,7 @@ class TestAccount(Base): if try_count < 5: time.sleep(1) - self.assertEquals(info['container_count'], len(self.env.containers)) + self.assertEqual(info['container_count'], len(self.env.containers)) self.assert_status(204) def testContainerSerializedInfo(self): @@ -309,11 +312,11 @@ class TestAccount(Base): headers = dict(self.env.conn.response.getheaders()) if format_type == 'json': - self.assertEquals(headers['content-type'], - 'application/json; charset=utf-8') + self.assertEqual(headers['content-type'], + 'application/json; charset=utf-8') elif format_type == 'xml': - self.assertEquals(headers['content-type'], - 'application/xml; charset=utf-8') + self.assertEqual(headers['content-type'], + 'application/xml; charset=utf-8') def testListingLimit(self): limit = load_constraint('account_listing_limit') @@ -337,7 +340,7 @@ class TestAccount(Base): if isinstance(b[0], dict): b = [x['name'] for x in b] - self.assertEquals(a, b) + self.assertEqual(a, b) def testInvalidAuthToken(self): hdrs = {'X-Auth-Token': 'bogus_auth_token'} @@ -347,12 +350,12 @@ class TestAccount(Base): def testLastContainerMarker(self): for format_type in [None, 'json', 'xml']: containers = self.env.account.containers({'format': format_type}) - self.assertEquals(len(containers), len(self.env.containers)) + self.assertEqual(len(containers), len(self.env.containers)) self.assert_status(200) containers = self.env.account.containers( parms={'format': format_type, 'marker': containers[-1]}) - self.assertEquals(len(containers), 0) + self.assertEqual(len(containers), 0) if format_type is None: self.assert_status(204) else: @@ -380,8 +383,8 @@ class TestAccount(Base): parms={'format': format_type}) if isinstance(containers[0], dict): containers = [x['name'] for x in containers] - self.assertEquals(sorted(containers, cmp=locale.strcoll), - containers) + self.assertEqual(sorted(containers, cmp=locale.strcoll), + containers) class TestAccountUTF8(Base2, TestAccount): @@ -518,13 +521,13 @@ class TestContainer(Base): for format_type in [None, 'json', 'xml']: for prefix in prefixs: files = cont.files(parms={'prefix': prefix}) - self.assertEquals(files, sorted(prefix_files[prefix])) + self.assertEqual(files, sorted(prefix_files[prefix])) for format_type in [None, 'json', 'xml']: for prefix in prefixs: files = cont.files(parms={'limit': limit_count, 'prefix': prefix}) - self.assertEquals(len(files), limit_count) + self.assertEqual(len(files), limit_count) for file_item in files: self.assert_(file_item.startswith(prefix)) @@ -548,7 +551,7 @@ class TestContainer(Base): container = self.env.account.container(valid_utf8) self.assert_(container.create(cfg={'no_path_quote': True})) self.assert_(container.name in self.env.account.containers()) - self.assertEquals(container.files(), []) + self.assertEqual(container.files(), []) self.assert_(container.delete()) container = self.env.account.container(invalid_utf8) @@ -614,12 +617,12 @@ class TestContainer(Base): def testLastFileMarker(self): for format_type in [None, 'json', 'xml']: files = self.env.container.files({'format': format_type}) - self.assertEquals(len(files), len(self.env.files)) + self.assertEqual(len(files), len(self.env.files)) self.assert_status(200) files = self.env.container.files( parms={'format': format_type, 'marker': files[-1]}) - self.assertEquals(len(files), 0) + self.assertEqual(len(files), 0) if format_type is None: self.assert_status(204) @@ -665,14 +668,14 @@ class TestContainer(Base): files = self.env.container.files(parms={'format': format_type}) if isinstance(files[0], dict): files = [x['name'] for x in files] - self.assertEquals(sorted(files, cmp=locale.strcoll), files) + self.assertEqual(sorted(files, cmp=locale.strcoll), files) def testContainerInfo(self): info = self.env.container.info() self.assert_status(204) - self.assertEquals(info['object_count'], self.env.file_count) - self.assertEquals(info['bytes_used'], - self.env.file_count * self.env.file_size) + self.assertEqual(info['object_count'], self.env.file_count) + self.assertEqual(info['bytes_used'], + self.env.file_count * self.env.file_size) def testContainerInfoOnContainerThatDoesNotExist(self): container = self.env.account.container(Utils.create_name()) @@ -683,7 +686,7 @@ class TestContainer(Base): for format_type in [None, 'json', 'xml']: files = self.env.container.files(parms={'format': format_type, 'limit': 2}) - self.assertEquals(len(files), 2) + self.assertEqual(len(files), 2) def testTooLongName(self): cont = self.env.account.container('x' * 257) @@ -838,7 +841,7 @@ class TestContainerPaths(Base): if isinstance(files[0], dict): files = [str(x['name']) for x in files] - self.assertEquals(files, self.env.stored_files) + self.assertEqual(files, self.env.stored_files) for format_type in ('json', 'xml'): for file_item in self.env.container.files(parms={'format': @@ -846,13 +849,13 @@ class TestContainerPaths(Base): self.assert_(int(file_item['bytes']) >= 0) self.assert_('last_modified' in file_item) if file_item['name'].endswith('/'): - self.assertEquals(file_item['content_type'], - 'application/directory') + self.assertEqual(file_item['content_type'], + 'application/directory') def testStructure(self): def assert_listing(path, file_list): files = self.env.container.files(parms={'path': path}) - self.assertEquals(sorted(file_list, cmp=locale.strcoll), files) + self.assertEqual(sorted(file_list, cmp=locale.strcoll), files) if not normalized_urls: assert_listing('/', ['/dir1/', '/dir2/', '/file1', '/file A']) assert_listing('/dir1', @@ -1176,7 +1179,7 @@ class TestFile(Base): for i in container.files(parms={'format': 'json'}): file_types_read[i['name'].split('.')[1]] = i['content_type'] - self.assertEquals(file_types, file_types_read) + self.assertEqual(file_types, file_types_read) def testRangedGets(self): file_length = 10000 @@ -1201,7 +1204,7 @@ class TestFile(Base): self.assertRaises(ResponseError, file_item.read, hdrs=hdrs) self.assert_status(416) else: - self.assertEquals(file_item.read(hdrs=hdrs), data[-i:]) + self.assertEqual(file_item.read(hdrs=hdrs), data[-i:]) range_string = 'bytes=%d-' % (i) hdrs = {'Range': range_string} @@ -1350,9 +1353,9 @@ class TestFile(Base): info = file_item.info() self.assert_status(200) - self.assertEquals(info['content_length'], self.env.file_size) - self.assertEquals(info['etag'], md5) - self.assertEquals(info['content_type'], content_type) + self.assertEqual(info['content_length'], self.env.file_size) + self.assertEqual(info['etag'], md5) + self.assertEqual(info['content_type'], content_type) self.assert_('last_modified' in info) def testDeleteOfFileThatDoesNotExist(self): @@ -1395,7 +1398,7 @@ class TestFile(Base): file_item = self.env.container.file(file_item.name) self.assert_(file_item.initialize()) self.assert_status(200) - self.assertEquals(file_item.metadata, metadata) + self.assertEqual(file_item.metadata, metadata) def testGetContentType(self): file_name = Utils.create_name() @@ -1408,7 +1411,7 @@ class TestFile(Base): file_item = self.env.container.file(file_name) file_item.read() - self.assertEquals(content_type, file_item.content_type) + self.assertEqual(content_type, file_item.content_type) def testGetOnFileThatDoesNotExist(self): # in container that exists @@ -1449,7 +1452,7 @@ class TestFile(Base): file_item = self.env.container.file(file_item.name) self.assert_(file_item.initialize()) self.assert_status(200) - self.assertEquals(file_item.metadata, metadata) + self.assertEqual(file_item.metadata, metadata) def testSerialization(self): container = self.env.account.container(Utils.create_name()) @@ -1478,9 +1481,9 @@ class TestFile(Base): if f['name'] != file_item['name']: continue - self.assertEquals(file_item['content_type'], - f['content_type']) - self.assertEquals(int(file_item['bytes']), f['bytes']) + self.assertEqual(file_item['content_type'], + f['content_type']) + self.assertEqual(int(file_item['bytes']), f['bytes']) d = datetime.strptime( file_item['last_modified'].split('.')[0], @@ -1488,7 +1491,7 @@ class TestFile(Base): lm = time.mktime(d.timetuple()) if 'last_modified' in f: - self.assertEquals(f['last_modified'], lm) + self.assertEqual(f['last_modified'], lm) else: f['last_modified'] = lm @@ -1500,11 +1503,11 @@ class TestFile(Base): headers = dict(self.env.conn.response.getheaders()) if format_type == 'json': - self.assertEquals(headers['content-type'], - 'application/json; charset=utf-8') + self.assertEqual(headers['content-type'], + 'application/json; charset=utf-8') elif format_type == 'xml': - self.assertEquals(headers['content-type'], - 'application/xml; charset=utf-8') + self.assertEqual(headers['content-type'], + 'application/xml; charset=utf-8') lm_diff = max([f['last_modified'] for f in files]) -\ min([f['last_modified'] for f in files]) @@ -1547,7 +1550,7 @@ class TestFile(Base): self.assert_('etag' in headers.keys()) header_etag = headers['etag'].strip('"') - self.assertEquals(etag, header_etag) + self.assertEqual(etag, header_etag) def testChunkedPut(self): if (web_front_end == 'apache2'): @@ -1565,7 +1568,7 @@ class TestFile(Base): self.assert_(data == file_item.read()) info = file_item.info() - self.assertEquals(etag, info['etag']) + self.assertEqual(etag, info['etag']) class TestFileUTF8(Base2, TestFile): @@ -1677,12 +1680,30 @@ class TestDlo(Base): file_contents, "aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffff") + def test_copy_manifest(self): + # Copying the manifest should result in another manifest + try: + man1_item = self.env.container.file('man1') + man1_item.copy(self.env.container.name, "copied-man1", + parms={'multipart-manifest': 'get'}) + + copied = self.env.container.file("copied-man1") + copied_contents = copied.read(parms={'multipart-manifest': 'get'}) + self.assertEqual(copied_contents, "man1-contents") + + copied_contents = copied.read() + self.assertEqual( + copied_contents, + "aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeee") + finally: + # try not to leave this around for other tests to stumble over + self.env.container.file("copied-man1").delete() class TestDloUTF8(Base2, TestDlo): set_up = False -class TestFileComparisonEnv: +class TestFileComparisonEnv(object): @classmethod def setUp(cls): cls.conn = Connection(config) @@ -1806,19 +1827,8 @@ class TestSloEnv(object): cls.conn.authenticate() if cls.slo_enabled is None: - status = cls.conn.make_request('GET', '/info', - cfg={'verbatim_path': True}) - if not (200 <= status <= 299): - # Can't tell if SLO is enabled or not since we're running - # against an old cluster, so let's skip the tests instead of - # possibly having spurious failures. - cls.slo_enabled = False - else: - # Don't bother looking for ValueError here. If something is - # responding to a GET /info request with invalid JSON, then - # the cluster is broken and a test failure will let us know. - cluster_info = json.loads(cls.conn.response.read()) - cls.slo_enabled = 'slo' in cluster_info + cluster_info = cls.conn.cluster_info() + cls.slo_enabled = 'slo' in cluster_info if not cls.slo_enabled: return @@ -2034,5 +2044,347 @@ class TestSloUTF8(Base2, TestSlo): set_up = False +class TestObjectVersioningEnv(object): + versioning_enabled = None # tri-state: None initially, then True/False + + @classmethod + def setUp(cls): + cls.conn = Connection(config) + cls.conn.authenticate() + + cls.account = Account(cls.conn, config.get('account', + config['username'])) + + # avoid getting a prefix that stops halfway through an encoded + # character + prefix = Utils.create_name().decode("utf-8")[:10].encode("utf-8") + + cls.versions_container = cls.account.container(prefix + "-versions") + if not cls.versions_container.create(): + raise ResponseError(cls.conn.response) + + cls.container = cls.account.container(prefix + "-objs") + if not cls.container.create( + hdrs={'X-Versions-Location': cls.versions_container.name}): + raise ResponseError(cls.conn.response) + + container_info = cls.container.info() + # if versioning is off, then X-Versions-Location won't persist + cls.versioning_enabled = 'versions' in container_info + + +class TestObjectVersioning(Base): + env = TestObjectVersioningEnv + set_up = False + + def setUp(self): + super(TestObjectVersioning, self).setUp() + if self.env.versioning_enabled is False: + raise SkipTest("Object versioning not enabled") + elif self.env.versioning_enabled is not True: + # just some sanity checking + raise Exception( + "Expected versioning_enabled to be True/False, got %r" % + (self.env.versioning_enabled,)) + + def test_overwriting(self): + container = self.env.container + versions_container = self.env.versions_container + obj_name = Utils.create_name() + + versioned_obj = container.file(obj_name) + versioned_obj.write("aaaaa") + + self.assertEqual(0, versions_container.info()['object_count']) + + versioned_obj.write("bbbbb") + + # the old version got saved off + self.assertEqual(1, versions_container.info()['object_count']) + versioned_obj_name = versions_container.files()[0] + self.assertEqual( + "aaaaa", versions_container.file(versioned_obj_name).read()) + + # if we overwrite it again, there are two versions + versioned_obj.write("ccccc") + self.assertEqual(2, versions_container.info()['object_count']) + + # as we delete things, the old contents return + self.assertEqual("ccccc", versioned_obj.read()) + versioned_obj.delete() + self.assertEqual("bbbbb", versioned_obj.read()) + versioned_obj.delete() + self.assertEqual("aaaaa", versioned_obj.read()) + versioned_obj.delete() + self.assertRaises(ResponseError, versioned_obj.read) + + +class TestObjectVersioningUTF8(Base2, TestObjectVersioning): + set_up = False + + +class TestTempurlEnv(object): + tempurl_enabled = None # tri-state: None initially, then True/False + + @classmethod + def setUp(cls): + cls.conn = Connection(config) + cls.conn.authenticate() + + if cls.tempurl_enabled is None: + cluster_info = cls.conn.cluster_info() + cls.tempurl_enabled = 'tempurl' in cluster_info + if not cls.tempurl_enabled: + return + cls.tempurl_methods = cluster_info['tempurl']['methods'] + + cls.tempurl_key = Utils.create_name() + cls.tempurl_key2 = Utils.create_name() + + cls.account = Account( + cls.conn, config.get('account', config['username'])) + cls.account.delete_containers() + cls.account.update_metadata({ + 'temp-url-key': cls.tempurl_key, + 'temp-url-key-2': cls.tempurl_key2 + }) + + cls.container = cls.account.container(Utils.create_name()) + if not cls.container.create(): + raise ResponseError(cls.conn.response) + + cls.obj = cls.container.file(Utils.create_name()) + cls.obj.write("obj contents") + cls.other_obj = cls.container.file(Utils.create_name()) + cls.other_obj.write("other obj contents") + + +class TestTempurl(Base): + env = TestTempurlEnv + set_up = False + + def setUp(self): + super(TestTempurl, self).setUp() + if self.env.tempurl_enabled is False: + raise SkipTest("TempURL not enabled") + elif self.env.tempurl_enabled is not True: + # just some sanity checking + raise Exception( + "Expected tempurl_enabled to be True/False, got %r" % + (self.env.tempurl_enabled,)) + + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'GET', expires, self.env.conn.make_path(self.env.obj.path), + self.env.tempurl_key) + self.obj_tempurl_parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + def tempurl_sig(self, method, expires, path, key): + return hmac.new( + key, + '%s\n%s\n%s' % (method, expires, urllib.unquote(path)), + hashlib.sha1).hexdigest() + + def test_GET(self): + contents = self.env.obj.read( + parms=self.obj_tempurl_parms, + cfg={'no_auth_token': True}) + self.assertEqual(contents, "obj contents") + + # GET tempurls also allow HEAD requests + self.assert_(self.env.obj.info(parms=self.obj_tempurl_parms, + cfg={'no_auth_token': True})) + + def test_GET_with_key_2(self): + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'GET', expires, self.env.conn.make_path(self.env.obj.path), + self.env.tempurl_key2) + parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + contents = self.env.obj.read(parms=parms, cfg={'no_auth_token': True}) + self.assertEqual(contents, "obj contents") + + def test_PUT(self): + new_obj = self.env.container.file(Utils.create_name()) + + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'PUT', expires, self.env.conn.make_path(new_obj.path), + self.env.tempurl_key) + put_parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + new_obj.write('new obj contents', + parms=put_parms, cfg={'no_auth_token': True}) + self.assertEqual(new_obj.read(), "new obj contents") + + # PUT tempurls also allow HEAD requests + self.assert_(new_obj.info(parms=put_parms, + cfg={'no_auth_token': True})) + + def test_HEAD(self): + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'HEAD', expires, self.env.conn.make_path(self.env.obj.path), + self.env.tempurl_key) + head_parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + self.assert_(self.env.obj.info(parms=head_parms, + cfg={'no_auth_token': True})) + # HEAD tempurls don't allow PUT or GET requests, despite the fact that + # PUT and GET tempurls both allow HEAD requests + self.assertRaises(ResponseError, self.env.other_obj.read, + cfg={'no_auth_token': True}, + parms=self.obj_tempurl_parms) + self.assert_status([401]) + + self.assertRaises(ResponseError, self.env.other_obj.write, + 'new contents', + cfg={'no_auth_token': True}, + parms=self.obj_tempurl_parms) + self.assert_status([401]) + + def test_different_object(self): + contents = self.env.obj.read( + parms=self.obj_tempurl_parms, + cfg={'no_auth_token': True}) + self.assertEqual(contents, "obj contents") + + self.assertRaises(ResponseError, self.env.other_obj.read, + cfg={'no_auth_token': True}, + parms=self.obj_tempurl_parms) + self.assert_status([401]) + + def test_changing_sig(self): + contents = self.env.obj.read( + parms=self.obj_tempurl_parms, + cfg={'no_auth_token': True}) + self.assertEqual(contents, "obj contents") + + parms = self.obj_tempurl_parms.copy() + if parms['temp_url_sig'][0] == 'a': + parms['temp_url_sig'] = 'b' + parms['temp_url_sig'][1:] + else: + parms['temp_url_sig'] = 'a' + parms['temp_url_sig'][1:] + + self.assertRaises(ResponseError, self.env.obj.read, + cfg={'no_auth_token': True}, + parms=parms) + self.assert_status([401]) + + def test_changing_expires(self): + contents = self.env.obj.read( + parms=self.obj_tempurl_parms, + cfg={'no_auth_token': True}) + self.assertEqual(contents, "obj contents") + + parms = self.obj_tempurl_parms.copy() + if parms['temp_url_expires'][-1] == '0': + parms['temp_url_expires'] = parms['temp_url_expires'][:-1] + '1' + else: + parms['temp_url_expires'] = parms['temp_url_expires'][:-1] + '0' + + self.assertRaises(ResponseError, self.env.obj.read, + cfg={'no_auth_token': True}, + parms=parms) + self.assert_status([401]) + + +class TestTempurlUTF8(Base2, TestTempurl): + set_up = False + + +class TestSloTempurlEnv(object): + enabled = None # tri-state: None initially, then True/False + + @classmethod + def setUp(cls): + cls.conn = Connection(config) + cls.conn.authenticate() + + if cls.enabled is None: + cluster_info = cls.conn.cluster_info() + cls.enabled = 'tempurl' in cluster_info and 'slo' in cluster_info + + cls.tempurl_key = Utils.create_name() + + cls.account = Account( + cls.conn, config.get('account', config['username'])) + cls.account.delete_containers() + cls.account.update_metadata({'temp-url-key': cls.tempurl_key}) + + cls.manifest_container = cls.account.container(Utils.create_name()) + cls.segments_container = cls.account.container(Utils.create_name()) + if not cls.manifest_container.create(): + raise ResponseError(cls.conn.response) + if not cls.segments_container.create(): + raise ResponseError(cls.conn.response) + + seg1 = cls.segments_container.file(Utils.create_name()) + seg1.write('1' * 1024 * 1024) + + seg2 = cls.segments_container.file(Utils.create_name()) + seg2.write('2' * 1024 * 1024) + + cls.manifest_data = [{'size_bytes': 1024 * 1024, + 'etag': seg1.md5, + 'path': '/%s/%s' % (cls.segments_container.name, + seg1.name)}, + {'size_bytes': 1024 * 1024, + 'etag': seg2.md5, + 'path': '/%s/%s' % (cls.segments_container.name, + seg2.name)}] + + cls.manifest = cls.manifest_container.file(Utils.create_name()) + cls.manifest.write( + json.dumps(cls.manifest_data), + parms={'multipart-manifest': 'put'}) + + +class TestSloTempurl(Base): + env = TestSloTempurlEnv + set_up = False + + def setUp(self): + super(TestSloTempurl, self).setUp() + if self.env.enabled is False: + raise SkipTest("TempURL and SLO not both enabled") + elif self.env.enabled is not True: + # just some sanity checking + raise Exception( + "Expected enabled to be True/False, got %r" % + (self.env.enabled,)) + + def tempurl_sig(self, method, expires, path, key): + return hmac.new( + key, + '%s\n%s\n%s' % (method, expires, urllib.unquote(path)), + hashlib.sha1).hexdigest() + + def test_GET(self): + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'GET', expires, self.env.conn.make_path(self.env.manifest.path), + self.env.tempurl_key) + parms = {'temp_url_sig': sig, 'temp_url_expires': str(expires)} + + contents = self.env.manifest.read( + parms=parms, + cfg={'no_auth_token': True}) + self.assertEqual(len(contents), 2 * 1024 * 1024) + + # GET tempurls also allow HEAD requests + self.assert_(self.env.manifest.info( + parms=parms, cfg={'no_auth_token': True})) + + +class TestSloTempurlUTF8(Base2, TestSloTempurl): + set_up = False + + if __name__ == '__main__': unittest.main() diff --git a/test/functional_auth/gswauth/conf/account-server.conf b/test/functional_auth/common_conf/account-server.conf index 4996367..549a094 100644 --- a/test/functional_auth/gswauth/conf/account-server.conf +++ b/test/functional_auth/common_conf/account-server.conf @@ -30,3 +30,7 @@ log_level = WARN # normal request logging for the account server to unclutter the log # files. Warnings and errors will still be logged. log_requests = off + +# The following parameter is used by object-expirer and needs to be same +# across all conf files! +auto_create_account_prefix = gs diff --git a/test/functional_auth/gswauth/conf/container-server.conf b/test/functional_auth/common_conf/container-server.conf index 122d97e..487ccb8 100644 --- a/test/functional_auth/gswauth/conf/container-server.conf +++ b/test/functional_auth/common_conf/container-server.conf @@ -33,3 +33,7 @@ log_requests = off #enable object versioning for functional test allow_versions = on + +# The following parameter is used by object-expirer and needs to be same +# across all conf files! +auto_create_account_prefix = gs diff --git a/test/functional_auth/gswauth/conf/fs.conf b/test/functional_auth/common_conf/fs.conf index b06a854..b06a854 100644 --- a/test/functional_auth/gswauth/conf/fs.conf +++ b/test/functional_auth/common_conf/fs.conf diff --git a/test/functional_auth/common_conf/object-expirer.conf b/test/functional_auth/common_conf/object-expirer.conf new file mode 100644 index 0000000..4449ee2 --- /dev/null +++ b/test/functional_auth/common_conf/object-expirer.conf @@ -0,0 +1,27 @@ +#TODO: Add documentation to explain various options +#For now, refer: https://github.com/openstack/swift/blob/master/etc/object-expirer.conf-sample + +[DEFAULT] + +[object-expirer] +user = root +log_facility = LOG_LOCAL2 +log_level = DEBUG +# The following parameters are used by object-expirer and needs to be same +# across all conf files! +auto_create_account_prefix = gs +expiring_objects_account_name = expiring + +interval = 30 + +[pipeline:main] +pipeline = catch_errors cache proxy-server + +[app:proxy-server] +use = egg:gluster_swift#proxy + +[filter:cache] +use = egg:swift#memcache + +[filter:catch_errors] +use = egg:swift#catch_errors diff --git a/test/functional_auth/tempauth/conf/object-server.conf b/test/functional_auth/common_conf/object-server.conf index 3cb9ead..2599c87 100644 --- a/test/functional_auth/tempauth/conf/object-server.conf +++ b/test/functional_auth/common_conf/object-server.conf @@ -46,3 +46,8 @@ disk_chunk_size = 65536 # Adjust this value match whatever is set for the disk_chunk_size initially. # This will provide a reasonable starting point for tuning this value. network_chunk_size = 65556 + +# The following parameter is used by object-expirer and needs to be same +# across all conf files! +auto_create_account_prefix = gs +expiring_objects_account_name = expiring diff --git a/test/functional_auth/gswauth/conf/swift.conf b/test/functional_auth/common_conf/swift.conf index f64ba5a..f64ba5a 100644 --- a/test/functional_auth/gswauth/conf/swift.conf +++ b/test/functional_auth/common_conf/swift.conf diff --git a/test/functional_auth/swiftkerbauth/conf/test.conf b/test/functional_auth/common_conf/test.conf index 643f2d1..15c9aea 100644 --- a/test/functional_auth/swiftkerbauth/conf/test.conf +++ b/test/functional_auth/common_conf/test.conf @@ -1,16 +1,25 @@ [func_test] -# Swiftkerbauth configuration +# sample config auth_host = 127.0.0.1 auth_port = 8080 +auth_ssl = no auth_prefix = /auth/ -auth_scheme = http:// -auth_mode = passive -auth_version = 1 -domain_name = RHELBOX.COM +## sample config for Swift with Keystone +#auth_version = 2 +#auth_host = localhost +#auth_port = 5000 +#auth_ssl = no +#auth_prefix = /v2.0/ + +# GSWauth internal admin user configuration information +admin_key = gswauthkey +admin_user = .super_admin + +# Gluster setup information +devices = /mnt/gluster-object +gsmetadata_volume = gsmetadata -#All the accounts, users & passwords to be prepared on kerberos server. # Primary functional test account (needs admin access to the account) -# Note: Account name to be prepared on kerberos server 'AUTH_accoun' account = test username = tester password = testing diff --git a/test/functional_auth/gswauth/conf/object-expirer.conf b/test/functional_auth/gswauth/conf/object-expirer.conf deleted file mode 100644 index b75963c..0000000 --- a/test/functional_auth/gswauth/conf/object-expirer.conf +++ /dev/null @@ -1,17 +0,0 @@ -[DEFAULT] - -[object-expirer] -# auto_create_account_prefix = . - -[pipeline:main] -pipeline = catch_errors cache proxy-server - -[app:proxy-server] -use = egg:swift#proxy - -[filter:cache] -use = egg:swift#memcache -memcache_servers = 127.0.0.1:11211 - -[filter:catch_errors] -use = egg:swift#catch_errors diff --git a/test/functional_auth/gswauth/conf/object-server.conf b/test/functional_auth/gswauth/conf/object-server.conf deleted file mode 100644 index 3cb9ead..0000000 --- a/test/functional_auth/gswauth/conf/object-server.conf +++ /dev/null @@ -1,48 +0,0 @@ -[DEFAULT] -devices = /mnt/gluster-object -# -# Once you are confident that your startup processes will always have your -# gluster volumes properly mounted *before* the object-server workers start, -# you can *consider* setting this value to "false" to reduce the per-request -# overhead it can incur. -# -# *** Keep false for Functional Tests *** -mount_check = false -bind_port = 6010 -# -# Maximum number of clients one worker can process simultaneously (it will -# actually accept N + 1). Setting this to one (1) will only handle one request -# at a time, without accepting another request concurrently. By increasing the -# number of workers to a much higher value, one can prevent slow file system -# operations for one request from starving other requests. -max_clients = 1024 -# -# If not doing the above, setting this value initially to match the number of -# CPUs is a good starting point for determining the right value. -workers = 1 -# Override swift's default behaviour for fallocate. -disable_fallocate = true - -[pipeline:main] -pipeline = object-server - -[app:object-server] -use = egg:gluster_swift#object -user = root -log_facility = LOG_LOCAL2 -log_level = WARN -# -# For performance, after ensuring things are running in a stable manner, you -# can turn off normal request logging for the object server to reduce the -# per-request overhead and unclutter the log files. Warnings and errors will -# still be logged. -log_requests = off -# -# Adjust this value to match the stripe width of the underlying storage array -# (not the stripe element size). This will provide a reasonable starting point -# for tuning this value. -disk_chunk_size = 65536 -# -# Adjust this value match whatever is set for the disk_chunk_size initially. -# This will provide a reasonable starting point for tuning this value. -network_chunk_size = 65556 diff --git a/test/functional_auth/gswauth/conf/proxy-server.conf b/test/functional_auth/gswauth/conf/proxy-server.conf index 165cb0c..0a3c1f6 100644 --- a/test/functional_auth/gswauth/conf/proxy-server.conf +++ b/test/functional_auth/gswauth/conf/proxy-server.conf @@ -5,7 +5,7 @@ user = root workers = 1 [pipeline:main] -pipeline = catch_errors healthcheck proxy-logging cache gswauth proxy-logging proxy-server +pipeline = catch_errors healthcheck proxy-logging cache tempurl gswauth proxy-logging proxy-server [app:proxy-server] use = egg:gluster_swift#proxy @@ -48,6 +48,10 @@ object_chunk_size = 65536 # amount of memory available on the system can accommodate increased values # for object_chunk_size. put_queue_depth = 10 +# The following parameter is used by object-expirer and needs to be same +# across all conf files! +auto_create_account_prefix = gs +expiring_objects_account_name = expiring [filter:catch_errors] use = egg:swift#catch_errors @@ -58,21 +62,17 @@ use = egg:swift#proxy_logging [filter:healthcheck] use = egg:swift#healthcheck -[filter:tempauth] -use = egg:swift#tempauth -user_admin_admin = admin .admin .reseller_admin -user_test_tester = testing .admin -user_test2_tester2 = testing2 .admin -user_test_tester3 = testing3 +[filter:cache] +use = egg:swift#memcache +# Update this line to contain a comma separated list of memcache servers +# shared by all nodes running the proxy-server service. +memcache_servers = localhost:11211 + +[filter:tempurl] +use = egg:swift#tempurl [filter:gswauth] use = egg:gluster_swift#gswauth set log_name = gswauth super_admin_key = gswauthkey metadata_volume = gsmetadata - -[filter:cache] -use = egg:swift#memcache -# Update this line to contain a comma separated list of memcache servers -# shared by all nodes running the proxy-server service. -memcache_servers = localhost:11211 diff --git a/test/functional_auth/keystone/conf/account-server.conf b/test/functional_auth/keystone/conf/account-server.conf deleted file mode 100644 index 4996367..0000000 --- a/test/functional_auth/keystone/conf/account-server.conf +++ /dev/null @@ -1,32 +0,0 @@ -[DEFAULT] -devices = /mnt/gluster-object -# -# Once you are confident that your startup processes will always have your -# gluster volumes properly mounted *before* the account-server workers start, -# you can *consider* setting this value to "false" to reduce the per-request -# overhead it can incur. -# -# *** Keep false for Functional Tests *** -mount_check = false -bind_port = 6012 -# -# Override swift's default behaviour for fallocate. -disable_fallocate = true -# -# One or two workers should be sufficient for almost any installation of -# Gluster. -workers = 1 - -[pipeline:main] -pipeline = account-server - -[app:account-server] -use = egg:gluster_swift#account -user = root -log_facility = LOG_LOCAL2 -log_level = WARN -# -# After ensuring things are running in a stable manner, you can turn off -# normal request logging for the account server to unclutter the log -# files. Warnings and errors will still be logged. -log_requests = off diff --git a/test/functional_auth/keystone/conf/container-server.conf b/test/functional_auth/keystone/conf/container-server.conf deleted file mode 100644 index 122d97e..0000000 --- a/test/functional_auth/keystone/conf/container-server.conf +++ /dev/null @@ -1,35 +0,0 @@ -[DEFAULT] -devices = /mnt/gluster-object -# -# Once you are confident that your startup processes will always have your -# gluster volumes properly mounted *before* the container-server workers -# start, you can *consider* setting this value to "false" to reduce the -# per-request overhead it can incur. -# -# *** Keep false for Functional Tests *** -mount_check = false -bind_port = 6011 -# -# Override swift's default behaviour for fallocate. -disable_fallocate = true -# -# One or two workers should be sufficient for almost any installation of -# Gluster. -workers = 1 - -[pipeline:main] -pipeline = container-server - -[app:container-server] -use = egg:gluster_swift#container -user = root -log_facility = LOG_LOCAL2 -log_level = WARN -# -# After ensuring things are running in a stable manner, you can turn off -# normal request logging for the container server to unclutter the log -# files. Warnings and errors will still be logged. -log_requests = off - -#enable object versioning for functional test -allow_versions = on diff --git a/test/functional_auth/keystone/conf/fs.conf b/test/functional_auth/keystone/conf/fs.conf deleted file mode 100644 index b06a854..0000000 --- a/test/functional_auth/keystone/conf/fs.conf +++ /dev/null @@ -1,19 +0,0 @@ -[DEFAULT] -# -# IP address of a node in the GlusterFS server cluster hosting the -# volumes to be served via Swift API. -mount_ip = localhost - -# Performance optimization parameter. When turned off, the filesystem will -# see a reduced number of stat calls, resulting in substantially faster -# response time for GET and HEAD container requests on containers with large -# numbers of objects, at the expense of an accurate count of combined bytes -# used by all objects in the container. For most installations "off" works -# fine. -# -# *** Keep on for Functional Tests *** -accurate_size_in_listing = on - -# *** Keep on for Functional Tests *** -container_update_object_count = on -account_update_container_count = on diff --git a/test/functional_auth/keystone/conf/object-server.conf b/test/functional_auth/keystone/conf/object-server.conf deleted file mode 100644 index 3cb9ead..0000000 --- a/test/functional_auth/keystone/conf/object-server.conf +++ /dev/null @@ -1,48 +0,0 @@ -[DEFAULT] -devices = /mnt/gluster-object -# -# Once you are confident that your startup processes will always have your -# gluster volumes properly mounted *before* the object-server workers start, -# you can *consider* setting this value to "false" to reduce the per-request -# overhead it can incur. -# -# *** Keep false for Functional Tests *** -mount_check = false -bind_port = 6010 -# -# Maximum number of clients one worker can process simultaneously (it will -# actually accept N + 1). Setting this to one (1) will only handle one request -# at a time, without accepting another request concurrently. By increasing the -# number of workers to a much higher value, one can prevent slow file system -# operations for one request from starving other requests. -max_clients = 1024 -# -# If not doing the above, setting this value initially to match the number of -# CPUs is a good starting point for determining the right value. -workers = 1 -# Override swift's default behaviour for fallocate. -disable_fallocate = true - -[pipeline:main] -pipeline = object-server - -[app:object-server] -use = egg:gluster_swift#object -user = root -log_facility = LOG_LOCAL2 -log_level = WARN -# -# For performance, after ensuring things are running in a stable manner, you -# can turn off normal request logging for the object server to reduce the -# per-request overhead and unclutter the log files. Warnings and errors will -# still be logged. -log_requests = off -# -# Adjust this value to match the stripe width of the underlying storage array -# (not the stripe element size). This will provide a reasonable starting point -# for tuning this value. -disk_chunk_size = 65536 -# -# Adjust this value match whatever is set for the disk_chunk_size initially. -# This will provide a reasonable starting point for tuning this value. -network_chunk_size = 65556 diff --git a/test/functional_auth/keystone/conf/proxy-server.conf b/test/functional_auth/keystone/conf/proxy-server.conf index 084e6a5..72a84da 100644 --- a/test/functional_auth/keystone/conf/proxy-server.conf +++ b/test/functional_auth/keystone/conf/proxy-server.conf @@ -6,7 +6,7 @@ workers = 1 [pipeline:main] #pipeline = catch_errors healthcheck proxy-logging cache tempauth proxy-logging proxy-server -pipeline = catch_errors healthcheck proxy-logging cache authtoken keystoneauth proxy-logging proxy-server +pipeline = catch_errors healthcheck proxy-logging cache tempurl authtoken keystoneauth proxy-logging proxy-server [app:proxy-server] use = egg:gluster_swift#proxy @@ -49,6 +49,10 @@ object_chunk_size = 65536 # amount of memory available on the system can accommodate increased values # for object_chunk_size. put_queue_depth = 10 +# The following parameter is used by object-expirer and needs to be same +# across all conf files! +auto_create_account_prefix = gs +expiring_objects_account_name = expiring [filter:catch_errors] use = egg:swift#catch_errors @@ -59,13 +63,14 @@ use = egg:swift#proxy_logging [filter:healthcheck] use = egg:swift#healthcheck -[filter:tempauth] -use = egg:swift#tempauth -user_admin_admin = admin .admin .reseller_admin -user_d4dde08c621a4f0fb4cde0ac6a62aa0c_tester = testing .admin -user_test_tester = testing .admin -user_test2_tester2 = testing2 .admin -user_test_tester3 = testing3 +[filter:cache] +use = egg:swift#memcache +# Update this line to contain a comma separated list of memcache servers +# shared by all nodes running the proxy-server service. +memcache_servers = localhost:11211 + +[filter:tempurl] +use = egg:swift#tempurl [filter:authtoken] paste.filter_factory = keystoneclient.middleware.auth_token:filter_factory @@ -87,8 +92,4 @@ operator_roles = admin is_admin = true cache = swift.cache -[filter:cache] -use = egg:swift#memcache -# Update this line to contain a comma separated list of memcache servers -# shared by all nodes running the proxy-server service. -memcache_servers = localhost:11211 + diff --git a/test/functional_auth/keystone/conf/swift.conf b/test/functional_auth/keystone/conf/swift.conf deleted file mode 100644 index ce9a4d0..0000000 --- a/test/functional_auth/keystone/conf/swift.conf +++ /dev/null @@ -1,85 +0,0 @@ -[DEFAULT] - - -[swift-hash] -# random unique string that can never change (DO NOT LOSE) -swift_hash_path_suffix = gluster - - -# The swift-constraints section sets the basic constraints on data -# saved in the swift cluster. - -[swift-constraints] - -# max_file_size is the largest "normal" object that can be saved in -# the cluster. This is also the limit on the size of each segment of -# a "large" object when using the large object manifest support. -# This value is set in bytes. Setting it to lower than 1MiB will cause -# some tests to fail. -# Default is 1 TiB = 2**30*1024 -max_file_size = 1099511627776 - - -# max_meta_name_length is the max number of bytes in the utf8 encoding -# of the name portion of a metadata header. - -#max_meta_name_length = 128 - - -# max_meta_value_length is the max number of bytes in the utf8 encoding -# of a metadata value - -#max_meta_value_length = 256 - - -# max_meta_count is the max number of metadata keys that can be stored -# on a single account, container, or object - -#max_meta_count = 90 - - -# max_meta_overall_size is the max number of bytes in the utf8 encoding -# of the metadata (keys + values) - -#max_meta_overall_size = 4096 - - -# max_object_name_length is the max number of bytes in the utf8 encoding of an -# object name: Gluster FS can handle much longer file names, but the length -# between the slashes of the URL is handled below. Remember that most web -# clients can't handle anything greater than 2048, and those that do are -# rather clumsy. - -max_object_name_length = 2048 - -# max_object_name_component_length (GlusterFS) is the max number of bytes in -# the utf8 encoding of an object name component (the part between the -# slashes); this is a limit imposed by the underlying file system (for XFS it -# is 255 bytes). - -max_object_name_component_length = 255 - -# container_listing_limit is the default (and max) number of items -# returned for a container listing request - -#container_listing_limit = 10000 - - -# account_listing_limit is the default (and max) number of items returned -# for an account listing request - -#account_listing_limit = 10000 - - -# max_account_name_length is the max number of bytes in the utf8 encoding of -# an account name: Gluster FS Filename limit (XFS limit?), must be the same -# size as max_object_name_component_length above. - -max_account_name_length = 255 - - -# max_container_name_length is the max number of bytes in the utf8 encoding -# of a container name: Gluster FS Filename limit (XFS limit?), must be the same -# size as max_object_name_component_length above. - -max_container_name_length = 255 diff --git a/test/functional_auth/swiftkerbauth/conf/account-server.conf b/test/functional_auth/swiftkerbauth/conf/account-server.conf deleted file mode 100644 index 9ad458a..0000000 --- a/test/functional_auth/swiftkerbauth/conf/account-server.conf +++ /dev/null @@ -1,36 +0,0 @@ -[DEFAULT] -# -# Default gluster mount point to be used for object store,can be changed by -# setting the following value in {account,container,object}-server.conf files. -# It is recommended to keep this value same for all the three services but can -# be kept different if environment demands. -devices = /mnt/gluster-object -# -# Once you are confident that your startup processes will always have your -# gluster volumes properly mounted *before* the account-server workers start, -# you can *consider* setting this value to "false" to reduce the per-request -# overhead it can incur. -mount_check = true -bind_port = 6012 -# -# Override swift's default behaviour for fallocate. -disable_fallocate = true -# -# One or two workers should be sufficient for almost any installation of -# Gluster. -workers = 1 - -[pipeline:main] -pipeline = account-server - -[app:account-server] -use = egg:gluster_swift#account -user = root -log_facility = LOG_LOCAL2 -log_level = WARN -# -# After ensuring things are running in a stable manner, you can turn off -# normal request logging for the account server to unclutter the log -# files. Warnings and errors will still be logged. -log_requests = off - diff --git a/test/functional_auth/swiftkerbauth/conf/container-server.conf b/test/functional_auth/swiftkerbauth/conf/container-server.conf deleted file mode 100644 index a406b4d..0000000 --- a/test/functional_auth/swiftkerbauth/conf/container-server.conf +++ /dev/null @@ -1,36 +0,0 @@ -[DEFAULT] -# -# Default gluster mount point to be used for object store,can be changed by -# setting the following value in {account,container,object}-server.conf files. -# It is recommended to keep this value same for all the three services but can -# be kept different if environment demands. -devices = /mnt/gluster-object -# -# Once you are confident that your startup processes will always have your -# gluster volumes properly mounted *before* the container-server workers -# start, you can *consider* setting this value to "false" to reduce the -# per-request overhead it can incur. -mount_check = true -bind_port = 6011 -# -# Override swift's default behaviour for fallocate. -disable_fallocate = true -# -# One or two workers should be sufficient for almost any installation of -# Gluster. -workers = 1 - -[pipeline:main] -pipeline = container-server - -[app:container-server] -use = egg:gluster_swift#container -user = root -log_facility = LOG_LOCAL2 -log_level = WARN -# -# After ensuring things are running in a stable manner, you can turn off -# normal request logging for the container server to unclutter the log -# files. Warnings and errors will still be logged. -log_requests = off - diff --git a/test/functional_auth/swiftkerbauth/conf/fs.conf b/test/functional_auth/swiftkerbauth/conf/fs.conf deleted file mode 100644 index 6d2a791..0000000 --- a/test/functional_auth/swiftkerbauth/conf/fs.conf +++ /dev/null @@ -1,13 +0,0 @@ -[DEFAULT] -# -# IP address of a node in the GlusterFS server cluster hosting the -# volumes to be served via Swift API. -mount_ip = localhost - -# Performance optimization parameter. When turned off, the filesystem will -# see a reduced number of stat calls, resulting in substantially faster -# response time for GET and HEAD container requests on containers with large -# numbers of objects, at the expense of an accurate count of combined bytes -# used by all objects in the container. For most installations "off" works -# fine. -accurate_size_in_listing = off
\ No newline at end of file diff --git a/test/functional_auth/swiftkerbauth/conf/object-server.conf b/test/functional_auth/swiftkerbauth/conf/object-server.conf deleted file mode 100644 index d10d282..0000000 --- a/test/functional_auth/swiftkerbauth/conf/object-server.conf +++ /dev/null @@ -1,51 +0,0 @@ -[DEFAULT] -# -# Default gluster mount point to be used for object store,can be changed by -# setting the following value in {account,container,object}-server.conf files. -# It is recommended to keep this value same for all the three services but can -# be kept different if environment demands. -devices = /mnt/gluster-object -# -# Once you are confident that your startup processes will always have your -# gluster volumes properly mounted *before* the object-server workers start, -# you can *consider* setting this value to "false" to reduce the per-request -# overhead it can incur. -mount_check = true -bind_port = 6010 -# -# Maximum number of clients one worker can process simultaneously (it will -# actually accept N + 1). Setting this to one (1) will only handle one request -# at a time, without accepting another request concurrently. By increasing the -# number of workers to a much higher value, one can prevent slow file system -# operations for one request from starving other requests. -max_clients = 1024 -# -# If not doing the above, setting this value initially to match the number of -# CPUs is a good starting point for determining the right value. -workers = 1 -# Override swift's default behaviour for fallocate. -disable_fallocate = true - -[pipeline:main] -pipeline = object-server - -[app:object-server] -use = egg:gluster_swift#object -user = root -log_facility = LOG_LOCAL2 -log_level = WARN -# -# For performance, after ensuring things are running in a stable manner, you -# can turn off normal request logging for the object server to reduce the -# per-request overhead and unclutter the log files. Warnings and errors will -# still be logged. -log_requests = off -# -# Adjust this value to match the stripe width of the underlying storage array -# (not the stripe element size). This will provide a reasonable starting point -# for tuning this value. -disk_chunk_size = 65536 -# -# Adjust this value match whatever is set for the disk_chunk_size initially. -# This will provide a reasonable starting point for tuning this value. -network_chunk_size = 65536 diff --git a/test/functional_auth/swiftkerbauth/conf/proxy-server.conf b/test/functional_auth/swiftkerbauth/conf/proxy-server.conf index e6f8aa1..248f87c 100644 --- a/test/functional_auth/swiftkerbauth/conf/proxy-server.conf +++ b/test/functional_auth/swiftkerbauth/conf/proxy-server.conf @@ -5,7 +5,7 @@ user = root workers = 1 [pipeline:main] -pipeline = catch_errors healthcheck proxy-logging cache proxy-logging kerbauth proxy-server +pipeline = catch_errors healthcheck proxy-logging cache tempurl proxy-logging kerbauth proxy-server [app:proxy-server] use = egg:gluster_swift#proxy @@ -48,6 +48,10 @@ object_chunk_size = 65536 # amount of memory available on the system can accommodate increased values # for object_chunk_size. put_queue_depth = 10 +# The following parameter is used by object-expirer and needs to be same +# across all conf files! +auto_create_account_prefix = gs +expiring_objects_account_name = expiring [filter:catch_errors] use = egg:swift#catch_errors @@ -59,12 +63,15 @@ access_log_level = WARN [filter:healthcheck] use = egg:swift#healthcheck -[filter:kerbauth] -use = egg:gluster_swift#kerbauth -ext_authentication_url = http://client.rhelbox.com/cgi-bin/swift-auth - [filter:cache] use = egg:swift#memcache # Update this line to contain a comma separated list of memcache servers # shared by all nodes running the proxy-server service. memcache_servers = localhost:11211 + +[filter:tempurl] +use = egg:swift#tempurl + +[filter:kerbauth] +use = egg:gluster_swift#kerbauth +ext_authentication_url = http://client.rhelbox.com/cgi-bin/swift-auth diff --git a/test/functional_auth/swiftkerbauth/conf/swift.conf b/test/functional_auth/swiftkerbauth/conf/swift.conf deleted file mode 100644 index 0d25209..0000000 --- a/test/functional_auth/swiftkerbauth/conf/swift.conf +++ /dev/null @@ -1,84 +0,0 @@ -[DEFAULT] - - -[swift-hash] -# random unique string that can never change (DO NOT LOSE) -swift_hash_path_suffix = gluster - - -# The swift-constraints section sets the basic constraints on data -# saved in the swift cluster. - -[swift-constraints] -# max_file_size is the largest "normal" object that can be saved in -# the cluster. This is also the limit on the size of each segment of -# a "large" object when using the large object manifest support. -# This value is set in bytes. Setting it to lower than 1MiB will cause -# some tests to fail. -# Default is 1 TiB = 2**30*1024 - -max_file_size = 1099511627776 - -# max_meta_name_length is the max number of bytes in the utf8 encoding -# of the name portion of a metadata header. - -#max_meta_name_length = 128 - - -# max_meta_value_length is the max number of bytes in the utf8 encoding -# of a metadata value - -#max_meta_value_length = 256 - - -# max_meta_count is the max number of metadata keys that can be stored -# on a single account, container, or object - -#max_meta_count = 90 - - -# max_meta_overall_size is the max number of bytes in the utf8 encoding -# of the metadata (keys + values) - -#max_meta_overall_size = 4096 - - -# max_object_name_length is the max number of bytes in the utf8 encoding of an -# object name: Gluster FS can handle much longer file names, but the length -# between the slashes of the URL is handled below. Remember that most web -# clients can't handle anything greater than 2048, and those that do are -# rather clumsy. - -max_object_name_length = 2048 - -# max_object_name_component_length (GlusterFS) is the max number of bytes in -# the utf8 encoding of an object name component (the part between the -# slashes); this is a limit imposed by the underlying file system (for XFS it -# is 255 bytes). - -max_object_name_component_length = 255 - -# container_listing_limit is the default (and max) number of items -# returned for a container listing request - -#container_listing_limit = 10000 - - -# account_listing_limit is the default (and max) number of items returned -# for an account listing request - -#account_listing_limit = 10000 - - -# max_account_name_length is the max number of bytes in the utf8 encoding of -# an account name: Gluster FS Filename limit (XFS limit?), must be the same -# size as max_object_name_component_length above. - -max_account_name_length = 255 - - -# max_container_name_length is the max number of bytes in the utf8 encoding -# of a container name: Gluster FS Filename limit (XFS limit?), must be the same -# size as max_object_name_component_length above. - -max_container_name_length = 255 diff --git a/test/functional_auth/swiftkerbauth/test_swkrbath_active.py b/test/functional_auth/swiftkerbauth/test_swkrbath_active.py deleted file mode 100644 index 86c79ef..0000000 --- a/test/functional_auth/swiftkerbauth/test_swkrbath_active.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/python - -# Copyright (c) 2010-2014 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re -import unittest -from nose import SkipTest -import commands -import os -from test import get_config -from swift.common.bufferedhttp import http_connect_raw as http_connect - -config = get_config('func_test') - -class Utils: - @classmethod - def SwiftKerbAuthPrep(self, - user=config['username'],domain=config['domain_name'],\ - passwd=config['password']): - username = '%s@%s' % (user, domain) - return commands.getstatusoutput('kinit %s <<< %s' % (username, passwd)) - - @classmethod - def SwiftKerbAuthCleanAll(self): - return commands.getstatusoutput('kdestroy') - - -class TestSwKrbAthActive(unittest.TestCase): - def setUp(self): - #Perform kinit in active mode. - (status, output) = Utils.SwiftKerbAuthPrep() - self.assertEqual(status, 0, \ - 'swkrbauth prep failed with valid credentials'+output) - self.auth_host = config['auth_host'] - self.auth_port = int(config['auth_port']) - self.auth_prefix = config.get('auth_prefix', '/auth/') - self.auth_version = str(config.get('auth_version', '1')) - self.account_name = config['account'] - self.username = config['username'] - self.password = config['password'] - self.auth_scheme = config['auth_scheme'] - - #Prepare auth_url. e.g. http://client.rhelbox.com:8080/auth/v1.0 - if self.auth_version == "1": - self.auth_path = '%sv1.0' % (self.auth_prefix) - else: - self.auth_path = self.auth_prefix - self.auth_netloc = "%s:%d" % (self.auth_host, self.auth_port) - auth_url = self.auth_scheme + self.auth_netloc + self.auth_path - - #Obtain the X-Auth-Token from kerberos server to use it in furhter - #testing - self.auth_token = None - (status, output) = commands.getstatusoutput('curl -v -u : --negotiate\ - --location-trusted %s' % (auth_url)) - self.assertEqual(status, 0, 'Token negotiation failed:' +output) - match = re.search('X-Auth-Token: AUTH.*', output) - if match: - self.auth_token = match.group(0).split(':')[1].strip() - else: - self.fail('No X-Auth-Token found, failed') - - def tearDown(self): - Utils.SwiftKerbAuthCleanAll() - - - def _get_auth_token(self): - return {'X-Auth-Token' : self.auth_token} - - def testGetAccounts(self): - #TODO: The test case is to perform GET on the account mentioned via - #configuration file. This is a sample test case. The whole test - #suite can be enhanced further to have further complicated test cases. - path = '/v1/AUTH_%s' % (config['account']) - - headers = self._get_auth_token() - conn = http_connect(config['auth_host'], config['auth_port'], 'GET', - path, headers) - resp = conn.getresponse() - self.assertTrue(resp.status == 204) diff --git a/test/functional_auth/tempauth/conf/account-server.conf b/test/functional_auth/tempauth/conf/account-server.conf deleted file mode 100644 index 4996367..0000000 --- a/test/functional_auth/tempauth/conf/account-server.conf +++ /dev/null @@ -1,32 +0,0 @@ -[DEFAULT] -devices = /mnt/gluster-object -# -# Once you are confident that your startup processes will always have your -# gluster volumes properly mounted *before* the account-server workers start, -# you can *consider* setting this value to "false" to reduce the per-request -# overhead it can incur. -# -# *** Keep false for Functional Tests *** -mount_check = false -bind_port = 6012 -# -# Override swift's default behaviour for fallocate. -disable_fallocate = true -# -# One or two workers should be sufficient for almost any installation of -# Gluster. -workers = 1 - -[pipeline:main] -pipeline = account-server - -[app:account-server] -use = egg:gluster_swift#account -user = root -log_facility = LOG_LOCAL2 -log_level = WARN -# -# After ensuring things are running in a stable manner, you can turn off -# normal request logging for the account server to unclutter the log -# files. Warnings and errors will still be logged. -log_requests = off diff --git a/test/functional_auth/tempauth/conf/container-server.conf b/test/functional_auth/tempauth/conf/container-server.conf deleted file mode 100644 index 122d97e..0000000 --- a/test/functional_auth/tempauth/conf/container-server.conf +++ /dev/null @@ -1,35 +0,0 @@ -[DEFAULT] -devices = /mnt/gluster-object -# -# Once you are confident that your startup processes will always have your -# gluster volumes properly mounted *before* the container-server workers -# start, you can *consider* setting this value to "false" to reduce the -# per-request overhead it can incur. -# -# *** Keep false for Functional Tests *** -mount_check = false -bind_port = 6011 -# -# Override swift's default behaviour for fallocate. -disable_fallocate = true -# -# One or two workers should be sufficient for almost any installation of -# Gluster. -workers = 1 - -[pipeline:main] -pipeline = container-server - -[app:container-server] -use = egg:gluster_swift#container -user = root -log_facility = LOG_LOCAL2 -log_level = WARN -# -# After ensuring things are running in a stable manner, you can turn off -# normal request logging for the container server to unclutter the log -# files. Warnings and errors will still be logged. -log_requests = off - -#enable object versioning for functional test -allow_versions = on diff --git a/test/functional_auth/tempauth/conf/fs.conf b/test/functional_auth/tempauth/conf/fs.conf deleted file mode 100644 index b06a854..0000000 --- a/test/functional_auth/tempauth/conf/fs.conf +++ /dev/null @@ -1,19 +0,0 @@ -[DEFAULT] -# -# IP address of a node in the GlusterFS server cluster hosting the -# volumes to be served via Swift API. -mount_ip = localhost - -# Performance optimization parameter. When turned off, the filesystem will -# see a reduced number of stat calls, resulting in substantially faster -# response time for GET and HEAD container requests on containers with large -# numbers of objects, at the expense of an accurate count of combined bytes -# used by all objects in the container. For most installations "off" works -# fine. -# -# *** Keep on for Functional Tests *** -accurate_size_in_listing = on - -# *** Keep on for Functional Tests *** -container_update_object_count = on -account_update_container_count = on diff --git a/test/functional_auth/tempauth/conf/object-expirer.conf b/test/functional_auth/tempauth/conf/object-expirer.conf deleted file mode 100644 index b75963c..0000000 --- a/test/functional_auth/tempauth/conf/object-expirer.conf +++ /dev/null @@ -1,17 +0,0 @@ -[DEFAULT] - -[object-expirer] -# auto_create_account_prefix = . - -[pipeline:main] -pipeline = catch_errors cache proxy-server - -[app:proxy-server] -use = egg:swift#proxy - -[filter:cache] -use = egg:swift#memcache -memcache_servers = 127.0.0.1:11211 - -[filter:catch_errors] -use = egg:swift#catch_errors diff --git a/test/functional_auth/tempauth/conf/proxy-server.conf b/test/functional_auth/tempauth/conf/proxy-server.conf index 830fadf..554ce61 100644 --- a/test/functional_auth/tempauth/conf/proxy-server.conf +++ b/test/functional_auth/tempauth/conf/proxy-server.conf @@ -5,7 +5,7 @@ user = root workers = 1 [pipeline:main] -pipeline = catch_errors healthcheck proxy-logging cache tempauth proxy-logging proxy-server +pipeline = catch_errors healthcheck proxy-logging cache tempurl tempauth proxy-logging proxy-server [app:proxy-server] use = egg:gluster_swift#proxy @@ -48,6 +48,10 @@ object_chunk_size = 65536 # amount of memory available on the system can accommodate increased values # for object_chunk_size. put_queue_depth = 10 +# The following parameter is used by object-expirer and needs to be same +# across all conf files! +auto_create_account_prefix = gs +expiring_objects_account_name = expiring [filter:catch_errors] use = egg:swift#catch_errors @@ -58,15 +62,18 @@ use = egg:swift#proxy_logging [filter:healthcheck] use = egg:swift#healthcheck +[filter:cache] +use = egg:swift#memcache +# Update this line to contain a comma separated list of memcache servers +# shared by all nodes running the proxy-server service. +memcache_servers = localhost:11211 + +[filter:tempurl] +use = egg:swift#tempurl + [filter:tempauth] use = egg:swift#tempauth user_admin_admin = admin .admin .reseller_admin user_test_tester = testing .admin user_test2_tester2 = testing2 .admin user_test_tester3 = testing3 - -[filter:cache] -use = egg:swift#memcache -# Update this line to contain a comma separated list of memcache servers -# shared by all nodes running the proxy-server service. -memcache_servers = localhost:11211 diff --git a/test/functional_auth/tempauth/conf/swift.conf b/test/functional_auth/tempauth/conf/swift.conf deleted file mode 100644 index ce9a4d0..0000000 --- a/test/functional_auth/tempauth/conf/swift.conf +++ /dev/null @@ -1,85 +0,0 @@ -[DEFAULT] - - -[swift-hash] -# random unique string that can never change (DO NOT LOSE) -swift_hash_path_suffix = gluster - - -# The swift-constraints section sets the basic constraints on data -# saved in the swift cluster. - -[swift-constraints] - -# max_file_size is the largest "normal" object that can be saved in -# the cluster. This is also the limit on the size of each segment of -# a "large" object when using the large object manifest support. -# This value is set in bytes. Setting it to lower than 1MiB will cause -# some tests to fail. -# Default is 1 TiB = 2**30*1024 -max_file_size = 1099511627776 - - -# max_meta_name_length is the max number of bytes in the utf8 encoding -# of the name portion of a metadata header. - -#max_meta_name_length = 128 - - -# max_meta_value_length is the max number of bytes in the utf8 encoding -# of a metadata value - -#max_meta_value_length = 256 - - -# max_meta_count is the max number of metadata keys that can be stored -# on a single account, container, or object - -#max_meta_count = 90 - - -# max_meta_overall_size is the max number of bytes in the utf8 encoding -# of the metadata (keys + values) - -#max_meta_overall_size = 4096 - - -# max_object_name_length is the max number of bytes in the utf8 encoding of an -# object name: Gluster FS can handle much longer file names, but the length -# between the slashes of the URL is handled below. Remember that most web -# clients can't handle anything greater than 2048, and those that do are -# rather clumsy. - -max_object_name_length = 2048 - -# max_object_name_component_length (GlusterFS) is the max number of bytes in -# the utf8 encoding of an object name component (the part between the -# slashes); this is a limit imposed by the underlying file system (for XFS it -# is 255 bytes). - -max_object_name_component_length = 255 - -# container_listing_limit is the default (and max) number of items -# returned for a container listing request - -#container_listing_limit = 10000 - - -# account_listing_limit is the default (and max) number of items returned -# for an account listing request - -#account_listing_limit = 10000 - - -# max_account_name_length is the max number of bytes in the utf8 encoding of -# an account name: Gluster FS Filename limit (XFS limit?), must be the same -# size as max_object_name_component_length above. - -max_account_name_length = 255 - - -# max_container_name_length is the max number of bytes in the utf8 encoding -# of a container name: Gluster FS Filename limit (XFS limit?), must be the same -# size as max_object_name_component_length above. - -max_container_name_length = 255 diff --git a/test/functional_auth/swiftkerbauth/__init__.py b/test/object_expirer_functional/__init__.py index e69de29..e69de29 100644 --- a/test/functional_auth/swiftkerbauth/__init__.py +++ b/test/object_expirer_functional/__init__.py diff --git a/test/object_expirer_functional/test_object_expirer.py b/test/object_expirer_functional/test_object_expirer.py new file mode 100644 index 0000000..aaec75e --- /dev/null +++ b/test/object_expirer_functional/test_object_expirer.py @@ -0,0 +1,332 @@ +# Copyright (c) 2014 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time + +from swift.common.manager import Manager +from swift.common.internal_client import InternalClient + +from test.functional.tests import Base, config, Utils +from test.functional.swift_test_client import Account, Connection, \ + ResponseError + + +class TestObjectExpirerEnv: + @classmethod + def setUp(cls): + cls.conn = Connection(config) + cls.conn.authenticate() + cls.account = Account(cls.conn, + config.get('account', + config['username'])) + cls.account.delete_containers() + cls.container = cls.account.container(Utils.create_name()) + if not cls.container.create(): + raise ResponseError(cls.conn.response) + cls.file_size = 8 + cls.root_dir = os.path.join('/mnt/gluster-object', + cls.account.conn.storage_url.split('/')[2].split('_')[1]) + cls.client = InternalClient('/etc/swift/object-expirer.conf', + 'Test Object Expirer', 1) + cls.expirer = Manager(['object-expirer']) + + +class TestObjectExpirer(Base): + env = TestObjectExpirerEnv + set_up = False + + def test_object_expiry_X_Delete_At_PUT(self): + obj = self.env.container.file(Utils.create_name()) + x_delete_at = str(int(time.time()) + 2) + obj.write_random(self.env.file_size, + hdrs={'X-Delete-At': x_delete_at}) + + # Object is not expired. Should still be accessible. + obj.read() + self.assert_status(200) + + # Ensure X-Delete-At is saved as object metadata. + self.assertEqual(x_delete_at, str(obj.info()['x_delete_at'])) + + # Wait for object to be expired. + time.sleep(3) + + # Object has expired. Should no longer be accessible. + self.assertRaises(ResponseError, obj.read) + self.assert_status(404) + + # Object should still be present on filesystem. + self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # But, GET on container should list the expired object. + result = self.env.container.files() + self.assertTrue(obj.name in self.env.container.files()) + + # Check existence of corresponding tracker object in gsexpiring + # account. + enteredLoop = False + for c in self.env.client.iter_containers("gsexpiring"): + for o in self.env.client.iter_objects("gsexpiring", c['name']): + enteredLoop = True + l = o['name'].split('/') + self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name)) + self.assertEqual(l[1], self.env.container.name) + self.assertEqual(l[2], obj.name) + if not enteredLoop: + self.fail("Tracker object not found.") + + # Run expirer daemon once. + self.env.expirer.once() + + # Ensure object is physically deleted from filesystem. + self.assertFalse(os.path.exists(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # Ensure tracker object is consumed. + try: + self.env.client.iter_containers("gsexpiring").next() + except StopIteration: + pass + else: + self.fail("Tracker object persists!") + + # GET on container should no longer list the object. + self.assertFalse(obj.name in self.env.container.files()) + + def test_object_expiry_X_Delete_After_PUT(self): + obj = self.env.container.file(Utils.create_name()) + obj.write_random(self.env.file_size, + hdrs={'X-Delete-After': 2}) + + # Object is not expired. Should still be accessible. + obj.read() + self.assert_status(200) + + # Ensure X-Delete-At is saved as object metadata. + self.assertTrue(str(obj.info()['x_delete_at'])) + + # Wait for object to be expired. + time.sleep(3) + + # Object has expired. Should no longer be accessible. + self.assertRaises(ResponseError, obj.read) + self.assert_status(404) + + # Object should still be present on filesystem. + self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # But, GET on container should list the expired object. + result = self.env.container.files() + self.assertTrue(obj.name in self.env.container.files()) + + # Check existence of corresponding tracker object in gsexpiring + # account. + enteredLoop = False + for c in self.env.client.iter_containers("gsexpiring"): + for o in self.env.client.iter_objects("gsexpiring", c['name']): + enteredLoop = True + l = o['name'].split('/') + self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name)) + self.assertEqual(l[1], self.env.container.name) + self.assertEqual(l[2], obj.name) + if not enteredLoop: + self.fail("Tracker object not found.") + + # Run expirer daemon once. + self.env.expirer.once() + + # Ensure object is physically deleted from filesystem. + self.assertFalse(os.path.exists(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # Ensure tracker object is consumed. + try: + self.env.client.iter_containers("gsexpiring").next() + except StopIteration: + pass + else: + self.fail("Tracker object persists!") + + # GET on container should no longer list the object. + self.assertFalse(obj.name in self.env.container.files()) + + + def test_object_expiry_X_Delete_At_POST(self): + + # Create normal object + obj = self.env.container.file(Utils.create_name()) + obj.write_random(self.env.file_size) + obj.read() + self.assert_status(200) + + # Send POST on that object and set it to be expired. + x_delete_at = str(int(time.time()) + 2) + obj.sync_metadata(metadata={'X-Delete-At': x_delete_at}, + cfg={'x_delete_at': x_delete_at}) + + # Ensure X-Delete-At is saved as object metadata. + self.assertEqual(x_delete_at, str(obj.info()['x_delete_at'])) + + # Object is not expired. Should still be accessible. + obj.read() + self.assert_status(200) + + # Wait for object to be expired. + time.sleep(3) + + # Object has expired. Should no longer be accessible. + self.assertRaises(ResponseError, obj.read) + self.assert_status(404) + + # Object should still be present on filesystem. + self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # But, GET on container should list the expired object. + result = self.env.container.files() + self.assertTrue(obj.name in self.env.container.files()) + + # Check existence of corresponding tracker object in gsexpiring + # account. + + enteredLoop = False + for c in self.env.client.iter_containers("gsexpiring"): + for o in self.env.client.iter_objects("gsexpiring", c['name']): + enteredLoop = True + l = o['name'].split('/') + self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name)) + self.assertEqual(l[1], self.env.container.name) + self.assertEqual(l[2], obj.name) + if not enteredLoop: + self.fail("Tracker object not found.") + + # Run expirer daemon once. + self.env.expirer.once() + time.sleep(3) + + # Ensure object is physically deleted from filesystem. + self.assertFalse(os.path.exists(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # Ensure tracker object is consumed. + try: + self.env.client.iter_containers("gsexpiring").next() + except StopIteration: + pass + else: + self.fail("Tracker object persists!") + + # GET on container should no longer list the object. + self.assertFalse(obj.name in self.env.container.files()) + + + def test_object_expiry_X_Delete_After_POST(self): + + # Create normal object + obj = self.env.container.file(Utils.create_name()) + obj.write_random(self.env.file_size) + obj.read() + self.assert_status(200) + + # Send POST on that object and set it to be expired. + obj.sync_metadata(metadata={'X-Delete-After': 2}, + cfg={'x_delete_after': 2}) + + # Ensure X-Delete-At is saved as object metadata. + self.assertTrue(str(obj.info()['x_delete_at'])) + + # Object is not expired. Should still be accessible. + obj.read() + self.assert_status(200) + + # Wait for object to be expired. + time.sleep(3) + + # Object has expired. Should no longer be accessible. + self.assertRaises(ResponseError, obj.read) + self.assert_status(404) + + # Object should still be present on filesystem. + self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # But, GET on container should list the expired object. + result = self.env.container.files() + self.assertTrue(obj.name in self.env.container.files()) + + # Check existence of corresponding tracker object in gsexpiring + # account. + + enteredLoop = False + for c in self.env.client.iter_containers("gsexpiring"): + for o in self.env.client.iter_objects("gsexpiring", c['name']): + enteredLoop = True + l = o['name'].split('/') + self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name)) + self.assertEqual(l[1], self.env.container.name) + self.assertEqual(l[2], obj.name) + if not enteredLoop: + self.fail("Tracker object not found.") + + # Run expirer daemon once. + self.env.expirer.once() + time.sleep(3) + + # Ensure object is physically deleted from filesystem. + self.assertFalse(os.path.exists(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # Ensure tracker object is consumed. + try: + self.env.client.iter_containers("gsexpiring").next() + except StopIteration: + pass + else: + self.fail("Tracker object persists!") + + # GET on container should no longer list the object. + self.assertFalse(obj.name in self.env.container.files()) + + + def test_object_expiry_err(self): + obj = self.env.container.file(Utils.create_name()) + + # X-Delete-At is invalid or is in the past + for i in (-2, 'abc', str(int(time.time()) - 2), 5.8): + self.assertRaises(ResponseError, + obj.write_random, + self.env.file_size, + hdrs={'X-Delete-At': i}) + self.assert_status(400) + + # X-Delete-After is invalid. + for i in (-2, 'abc', 3.7): + self.assertRaises(ResponseError, + obj.write_random, + self.env.file_size, + hdrs={'X-Delete-After': i}) + self.assert_status(400) + diff --git a/test/unit/__init__.py b/test/unit/__init__.py index 847479a..a1bfef8 100644 --- a/test/unit/__init__.py +++ b/test/unit/__init__.py @@ -33,6 +33,7 @@ from hashlib import md5 from eventlet import sleep, Timeout import logging.handlers from httplib import HTTPException +from numbers import Number class FakeRing(object): @@ -248,6 +249,7 @@ class FakeLogger(logging.Logger): if 'facility' in kwargs: self.facility = kwargs['facility'] self.statsd_client = None + self.thread_locals = None def _clear(self): self.log_dict = defaultdict(list) @@ -465,8 +467,11 @@ def fake_http_connect(*code_iter, **kwargs): self.body = body self.headers = headers or {} self.timestamp = timestamp - if kwargs.get('slow') and isinstance(kwargs['slow'], list): - kwargs['slow'][0] -= 1 + if 'slow' in kwargs and isinstance(kwargs['slow'], list): + try: + self._next_sleep = kwargs['slow'].pop(0) + except IndexError: + self._next_sleep = None def getresponse(self): if kwargs.get('raise_exc'): @@ -482,6 +487,8 @@ def fake_http_connect(*code_iter, **kwargs): return FakeConn(507) if self.expect_status == -4: return FakeConn(201) + if self.expect_status == 412: + return FakeConn(412) return FakeConn(100) def getheaders(self): @@ -510,31 +517,39 @@ def fake_http_connect(*code_iter, **kwargs): headers['x-container-timestamp'] = '1' except StopIteration: pass - if self.am_slow(): + am_slow, value = self.get_slow() + if am_slow: headers['content-length'] = '4' headers.update(self.headers) return headers.items() - def am_slow(self): - if kwargs.get('slow') and isinstance(kwargs['slow'], list): - return kwargs['slow'][0] >= 0 - return bool(kwargs.get('slow')) + def get_slow(self): + if 'slow' in kwargs and isinstance(kwargs['slow'], list): + if self._next_sleep is not None: + return True, self._next_sleep + else: + return False, 0.01 + if kwargs.get('slow') and isinstance(kwargs['slow'], Number): + return True, kwargs['slow'] + return bool(kwargs.get('slow')), 0.1 def read(self, amt=None): - if self.am_slow(): + am_slow, value = self.get_slow() + if am_slow: if self.sent < 4: self.sent += 1 - sleep(0.1) + sleep(value) return ' ' rv = self.body[:amt] self.body = self.body[amt:] return rv def send(self, amt=None): - if self.am_slow(): + am_slow, value = self.get_slow() + if am_slow: if self.received < 4: self.received += 1 - sleep(0.1) + sleep(value) def getheader(self, name, default=None): return dict(self.getheaders()).get(name.lower(), default) @@ -584,4 +599,6 @@ def fake_http_connect(*code_iter, **kwargs): return FakeConn(status, etag, body=body, timestamp=timestamp, expect_status=expect_status, headers=headers) + connect.code_iter = code_iter + return connect diff --git a/test/unit/common/middleware/swiftkerbauth/__init__.py b/test/unit/common/middleware/swiftkerbauth/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/test/unit/common/middleware/swiftkerbauth/__init__.py +++ /dev/null diff --git a/test/unit/common/middleware/swiftkerbauth/test_kerbauth.py b/test/unit/common/middleware/swiftkerbauth/test_kerbauth.py deleted file mode 100644 index 537b8d3..0000000 --- a/test/unit/common/middleware/swiftkerbauth/test_kerbauth.py +++ /dev/null @@ -1,478 +0,0 @@ -# Copyright (c) 2013 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import errno -import unittest -from time import time -from mock import patch, Mock -from test.unit import FakeMemcache -from swift.common.swob import Request, Response -from gluster.swift.common.middleware.swiftkerbauth import kerbauth as auth - -EXT_AUTHENTICATION_URL = "127.0.0.1" -REDIRECT_STATUS = 303 # HTTPSeeOther - - -def my_filter_factory(global_conf, **local_conf): - if 'ext_authentication_url' not in global_conf: - global_conf['ext_authentication_url'] = EXT_AUTHENTICATION_URL - conf = global_conf.copy() - conf.update(local_conf) - - def auth_filter(app): - return auth.KerbAuth(app, conf) - return auth_filter - -# Monkey patching filter_factory to always pass ext_authentication_url -# as a parameter. Absence of ext_authentication_url raises a RuntimeError - - -def patch_filter_factory(): - auth.filter_factory = my_filter_factory - - -def unpatch_filter_factory(): - reload(auth) - - -class FakeApp(object): - - def __init__(self, status_headers_body_iter=None, acl=None, sync_key=None): - self.calls = 0 - self.status_headers_body_iter = status_headers_body_iter - if not self.status_headers_body_iter: - self.status_headers_body_iter = iter([('404 Not Found', {}, '')]) - self.acl = acl - self.sync_key = sync_key - - def __call__(self, env, start_response): - self.calls += 1 - self.request = Request.blank('', environ=env) - if self.acl: - self.request.acl = self.acl - if self.sync_key: - self.request.environ['swift_sync_key'] = self.sync_key - if 'swift.authorize' in env: - resp = env['swift.authorize'](self.request) - if resp: - return resp(env, start_response) - status, headers, body = self.status_headers_body_iter.next() - return Response(status=status, headers=headers, - body=body)(env, start_response) - - -class TestKerbAuth(unittest.TestCase): - - # Patch auth.filter_factory() - patch_filter_factory() - - def setUp(self): - self.test_auth = \ - auth.filter_factory({'auth_method': 'active'})(FakeApp()) - self.test_auth_passive = \ - auth.filter_factory({'auth_method': 'passive'})(FakeApp()) - - def _make_request(self, path, **kwargs): - req = Request.blank(path, **kwargs) - req.environ['swift.cache'] = FakeMemcache() - return req - - def test_no_ext_authentication_url(self): - app = FakeApp() - try: - # Use original auth.filter_factory and NOT monkey patched version - unpatch_filter_factory() - auth.filter_factory({})(app) - except RuntimeError as e: - # Restore monkey patched version - patch_filter_factory() - self.assertTrue(e.args[0].startswith("Missing filter parameter " - "ext_authentication_url")) - - def test_reseller_prefix_init(self): - app = FakeApp() - ath = auth.filter_factory({})(app) - self.assertEquals(ath.reseller_prefix, 'AUTH_') - ath = auth.filter_factory({'reseller_prefix': 'TEST'})(app) - self.assertEquals(ath.reseller_prefix, 'TEST_') - ath = auth.filter_factory({'reseller_prefix': 'TEST_'})(app) - self.assertEquals(ath.reseller_prefix, 'TEST_') - - def test_auth_prefix_init(self): - app = FakeApp() - ath = auth.filter_factory({})(app) - self.assertEquals(ath.auth_prefix, '/auth/') - ath = auth.filter_factory({'auth_prefix': ''})(app) - self.assertEquals(ath.auth_prefix, '/auth/') - ath = auth.filter_factory({'auth_prefix': '/'})(app) - self.assertEquals(ath.auth_prefix, '/auth/') - ath = auth.filter_factory({'auth_prefix': '/test/'})(app) - self.assertEquals(ath.auth_prefix, '/test/') - ath = auth.filter_factory({'auth_prefix': '/test'})(app) - self.assertEquals(ath.auth_prefix, '/test/') - ath = auth.filter_factory({'auth_prefix': 'test/'})(app) - self.assertEquals(ath.auth_prefix, '/test/') - ath = auth.filter_factory({'auth_prefix': 'test'})(app) - self.assertEquals(ath.auth_prefix, '/test/') - - def test_top_level_redirect(self): - req = self._make_request('/') - resp = req.get_response(self.test_auth) - self.assertEquals(resp.status_int, REDIRECT_STATUS) - self.assertEquals(req.environ['swift.authorize'], - self.test_auth.denied_response) - - def test_passive_top_level_deny(self): - req = self._make_request('/') - resp = req.get_response(self.test_auth_passive) - self.assertEquals(resp.status_int, 401) - self.assertEquals(req.environ['swift.authorize'], - self.test_auth_passive.denied_response) - - def test_passive_deny_invalid_token(self): - req = self._make_request('/v1/AUTH_account', - headers={'X-Auth-Token': 'AUTH_t'}) - resp = req.get_response(self.test_auth_passive) - self.assertEquals(resp.status_int, 401) - - def test_override_asked_for_and_allowed(self): - self.test_auth = \ - auth.filter_factory({'allow_overrides': 'true'})(FakeApp()) - req = self._make_request('/v1/AUTH_account', - environ={'swift.authorize_override': True}) - resp = req.get_response(self.test_auth) - self.assertEquals(resp.status_int, 404) - self.assertTrue('swift.authorize' not in req.environ) - - def test_override_default_allowed(self): - req = self._make_request('/v1/AUTH_account', - environ={'swift.authorize_override': True}) - resp = req.get_response(self.test_auth) - self.assertEquals(resp.status_int, 404) - self.assertTrue('swift.authorize' not in req.environ) - - def test_options_call(self): - req = self._make_request('/v1/AUTH_cfa/c/o', - environ={'REQUEST_METHOD': 'OPTIONS'}) - resp = self.test_auth.authorize(req) - self.assertEquals(resp, None) - - def test_auth_deny_non_reseller_prefix_no_override(self): - fake_authorize = lambda x: Response(status='500 Fake') - req = self._make_request('/v1/BLAH_account', - headers={'X-Auth-Token': 'BLAH_t'}, - environ={'swift.authorize': fake_authorize} - ) - resp = req.get_response(self.test_auth) - self.assertEquals(resp.status_int, 500) - self.assertEquals(req.environ['swift.authorize'], fake_authorize) - - def test_authorize_acl_group_access(self): - req = self._make_request('/v1/AUTH_cfa') - req.remote_user = 'act:usr,act' - resp = self.test_auth.authorize(req) - self.assertEquals(resp.status_int, 403) - req = self._make_request('/v1/AUTH_cfa') - req.remote_user = 'act:usr,act' - req.acl = 'act' - self.assertEquals(self.test_auth.authorize(req), None) - req = self._make_request('/v1/AUTH_cfa') - req.remote_user = 'act:usr,act' - req.acl = 'act:usr' - self.assertEquals(self.test_auth.authorize(req), None) - req = self._make_request('/v1/AUTH_cfa') - req.remote_user = 'act:usr,act' - - def test_deny_cross_reseller(self): - # Tests that cross-reseller is denied, even if ACLs/group names match - req = self._make_request('/v1/OTHER_cfa') - req.remote_user = 'act:usr,act,AUTH_cfa' - req.acl = 'act' - resp = self.test_auth.authorize(req) - self.assertEquals(resp.status_int, 403) - - def test_authorize_acl_referer_after_user_groups(self): - req = self._make_request('/v1/AUTH_cfa/c') - req.remote_user = 'act:usr' - req.acl = '.r:*,act:usr' - self.assertEquals(self.test_auth.authorize(req), None) - - def test_detect_reseller_request(self): - req = self._make_request('/v1/AUTH_admin', - headers={'X-Auth-Token': 'AUTH_t'}) - cache_key = 'AUTH_/token/AUTH_t' - cache_entry = (time() + 3600, '.reseller_admin') - req.environ['swift.cache'].set(cache_key, cache_entry) - req.get_response(self.test_auth) - self.assertTrue(req.environ.get('reseller_request', False)) - - def test_regular_is_not_owner(self): - orig_authorize = self.test_auth.authorize - owner_values = [] - - def mitm_authorize(req): - rv = orig_authorize(req) - owner_values.append(req.environ.get('swift_owner', False)) - return rv - - self.test_auth.authorize = mitm_authorize - - req = self._make_request( - '/v1/AUTH_cfa/c', - headers={'X-Auth-Token': 'AUTH_t'}) - req.remote_user = 'act:usr' - self.test_auth.authorize(req) - self.assertEquals(owner_values, [False]) - - def test_no_memcache(self): - env = {'swift.cache': None} - try: - self.test_auth.get_groups(env, None) - except Exception as e: - self.assertTrue(e.args[0].startswith("Memcache required")) - - def test_handle_request(self): - req = self._make_request('/auth/v1.0') - resp = self.test_auth.handle_request(req) - self.assertEquals(resp.status_int, REDIRECT_STATUS) - - def test_handle_request_bad_request(self): - req = self._make_request('////') - resp = self.test_auth.handle_request(req) - self.assertEquals(resp.status_int, 404) - - def test_handle_request_no_handler(self): - req = self._make_request('/blah/blah/blah/blah') - resp = self.test_auth.handle_request(req) - self.assertEquals(resp.status_int, 400) - - def test_handle_get_token_bad_request(self): - req = self._make_request('/blah/blah') - resp = self.test_auth.handle_get_token(req) - self.assertEquals(resp.status_int, 400) - req = self._make_request('/////') - resp = self.test_auth.handle_get_token(req) - self.assertEquals(resp.status_int, 404) - - def test_passive_handle_get_token_no_user_or_key(self): - #No user and key - req = self._make_request('/auth/v1.0') - resp = self.test_auth_passive.handle_get_token(req) - self.assertEquals(resp.status_int, REDIRECT_STATUS) - #User given but no key - req = self._make_request('/auth/v1.0', - headers={'X-Auth-User': 'test:user'}) - resp = self.test_auth_passive.handle_get_token(req) - self.assertEquals(resp.status_int, 401) - - def test_passive_handle_get_token_account_in_req_path(self): - req = self._make_request('/v1/test/auth', - headers={'X-Auth-User': 'test:user', - 'X-Auth-Key': 'password'}) - _mock_run_kinit = Mock(return_value=0) - _mock_get_groups = Mock(return_value="user,auth_test") - with patch('gluster.swift.common.middleware.swiftkerbauth.kerbauth.run_kinit', _mock_run_kinit): - with patch('gluster.swift.common.middleware.swiftkerbauth.kerbauth.get_groups_from_username', - _mock_get_groups): - resp = self.test_auth_passive.handle_get_token(req) - _mock_run_kinit.assert_called_once_with('user', 'password') - self.assertEquals(_mock_get_groups.call_count, 2) - self.assertEquals(resp.status_int, 200) - self.assertTrue(resp.headers['X-Auth-Token'] is not None) - self.assertTrue(resp.headers['X-Storage-Token'] is not None) - self.assertTrue(resp.headers['X-Storage-Url'] is not None) - - def test_passive_handle_get_token_user_invalid_or_no__account(self): - #X-Auth-User not in acc:user format - req = self._make_request('/auth/v1.0', - headers={'X-Auth-User': 'user'}) - resp = self.test_auth_passive.handle_get_token(req) - self.assertEquals(resp.status_int, 401) - req = self._make_request('/v1/test/auth', - headers={'X-Auth-User': 'user'}) - resp = self.test_auth_passive.handle_get_token(req) - self.assertEquals(resp.status_int, 401) - # Account name mismatch - req = self._make_request('/v1/test/auth', - headers={'X-Auth-User': 'wrongacc:user'}) - resp = self.test_auth_passive.handle_get_token(req) - self.assertEquals(resp.status_int, 401) - - def test_passive_handle_get_token_no_kinit(self): - req = self._make_request('/auth/v1.0', - headers={'X-Auth-User': 'test:user', - 'X-Auth-Key': 'password'}) - _mock_run_kinit = Mock(side_effect=OSError(errno.ENOENT, - os.strerror(errno.ENOENT))) - with patch('gluster.swift.common.middleware.swiftkerbauth.kerbauth.run_kinit', _mock_run_kinit): - resp = self.test_auth_passive.handle_get_token(req) - self.assertEquals(resp.status_int, 500) - self.assertTrue("kinit command not found" in resp.body) - _mock_run_kinit.assert_called_once_with('user', 'password') - - def test_passive_handle_get_token_kinit_fail(self): - req = self._make_request('/auth/v1.0', - headers={'X-Auth-User': 'test:user', - 'X-Auth-Key': 'password'}) - _mock_run_kinit = Mock(return_value=1) - with patch('gluster.swift.common.middleware.swiftkerbauth.kerbauth.run_kinit', _mock_run_kinit): - resp = self.test_auth_passive.handle_get_token(req) - self.assertEquals(resp.status_int, 401) - _mock_run_kinit.assert_called_once_with('user', 'password') - - def test_passive_handle_get_token_kinit_success_token_not_present(self): - req = self._make_request('/auth/v1.0', - headers={'X-Auth-User': 'test:user', - 'X-Auth-Key': 'password'}) - _mock_run_kinit = Mock(return_value=0) - _mock_get_groups = Mock(return_value="user,auth_test") - with patch('gluster.swift.common.middleware.swiftkerbauth.kerbauth.run_kinit', _mock_run_kinit): - with patch('gluster.swift.common.middleware.swiftkerbauth.kerbauth.get_groups_from_username', - _mock_get_groups): - resp = self.test_auth_passive.handle_get_token(req) - _mock_run_kinit.assert_called_once_with('user', 'password') - self.assertEquals(_mock_get_groups.call_count, 2) - self.assertEquals(resp.status_int, 200) - self.assertTrue(resp.headers['X-Auth-Token'] is not None) - self.assertTrue(resp.headers['X-Storage-Token'] is not None) - self.assertTrue(resp.headers['X-Storage-Url'] is not None) - - def test_passive_handle_get_token_kinit_realm_and_memcache(self): - req = self._make_request('/auth/v1.0', - headers={'X-Auth-User': 'test:user', - 'X-Auth-Key': 'password'}) - req.environ['swift.cache'] = None - _auth_passive = \ - auth.filter_factory({'auth_method': 'passive', - 'realm_name': 'EXAMPLE.COM'})(FakeApp()) - _mock_run_kinit = Mock(return_value=0) - _mock_get_groups = Mock(return_value="user,auth_test") - with patch('gluster.swift.common.middleware.swiftkerbauth.kerbauth.run_kinit', _mock_run_kinit): - with patch('gluster.swift.common.middleware.swiftkerbauth.kerbauth.get_groups_from_username', - _mock_get_groups): - try: - _auth_passive.handle_get_token(req) - except Exception as e: - self.assertTrue(e.args[0].startswith("Memcache " - "required")) - else: - self.fail("Expected Exception - Memcache required") - _mock_run_kinit.assert_called_once_with('user@EXAMPLE.COM', 'password') - _mock_get_groups.assert_called_once_with('user') - - def test_passive_handle_get_token_user_in_any__account(self): - req = self._make_request('/auth/v1.0', - headers={'X-Auth-User': 'test:user', - 'X-Auth-Key': 'password'}) - _mock_run_kinit = Mock(return_value=0) - _mock_get_groups = Mock(return_value="user,auth_blah") - with patch('gluster.swift.common.middleware.swiftkerbauth.kerbauth.run_kinit', _mock_run_kinit): - with patch('gluster.swift.common.middleware.swiftkerbauth.kerbauth.get_groups_from_username', - _mock_get_groups): - resp = self.test_auth_passive.handle_get_token(req) - self.assertEquals(resp.status_int, 401) - _mock_run_kinit.assert_called_once_with('user', 'password') - _mock_get_groups.assert_called_once_with('user') - - def test_handle(self): - req = self._make_request('/auth/v1.0') - resp = req.get_response(self.test_auth) - self.assertEquals(resp.status_int, REDIRECT_STATUS) - - def test_authorize_invalid_req(self): - req = self._make_request('/') - resp = self.test_auth.authorize(req) - self.assertEquals(resp.status_int, 404) - - def test_authorize_set_swift_owner(self): - req = self._make_request('/v1/AUTH_test/c1/o1') - req.remote_user = 'test,auth_reseller_admin' - resp = self.test_auth.authorize(req) - self.assertEquals(req.environ['swift_owner'], True) - self.assertTrue(resp is None) - req = self._make_request('/v1/AUTH_test/c1/o1') - req.remote_user = 'test,auth_test' - resp = self.test_auth.authorize(req) - self.assertEquals(req.environ['swift_owner'], True) - self.assertTrue(resp is None) - - def test_authorize_swift_sync_key(self): - req = self._make_request( - '/v1/AUTH_cfa/c/o', - environ={'swift_sync_key': 'secret'}, - headers={'x-container-sync-key': 'secret', - 'x-timestamp': '123.456'}) - resp = self.test_auth.authorize(req) - self.assertTrue(resp is None) - - def test_authorize_acl_referrer_access(self): - req = self._make_request('/v1/AUTH_cfa/c') - req.remote_user = 'act:usr,act' - resp = self.test_auth.authorize(req) - self.assertEquals(resp.status_int, 403) - req = self._make_request('/v1/AUTH_cfa/c') - req.remote_user = 'act:usr,act' - req.acl = '.r:*,.rlistings' - self.assertEquals(self.test_auth.authorize(req), None) - req = self._make_request('/v1/AUTH_cfa/c') - req.remote_user = 'act:usr,act' - req.acl = '.r:*' # No listings allowed - resp = self.test_auth.authorize(req) - self.assertEquals(resp.status_int, 403) - req = self._make_request('/v1/AUTH_cfa/c') - req.remote_user = 'act:usr,act' - req.acl = '.r:.example.com,.rlistings' - resp = self.test_auth.authorize(req) - self.assertEquals(resp.status_int, 403) - req = self._make_request('/v1/AUTH_cfa/c') - req.remote_user = 'act:usr,act' - req.referer = 'http://www.example.com/index.html' - req.acl = '.r:.example.com,.rlistings' - self.assertEquals(self.test_auth.authorize(req), None) - req = self._make_request('/v1/AUTH_cfa/c') - resp = self.test_auth.authorize(req) - self.assertEquals(resp.status_int, REDIRECT_STATUS) - req = self._make_request('/v1/AUTH_cfa/c') - req.acl = '.r:*,.rlistings' - self.assertEquals(self.test_auth.authorize(req), None) - req = self._make_request('/v1/AUTH_cfa/c') - req.acl = '.r:*' # No listings allowed - resp = self.test_auth.authorize(req) - self.assertEquals(resp.status_int, REDIRECT_STATUS) - req = self._make_request('/v1/AUTH_cfa/c') - req.acl = '.r:.example.com,.rlistings' - resp = self.test_auth.authorize(req) - self.assertEquals(resp.status_int, REDIRECT_STATUS) - req = self._make_request('/v1/AUTH_cfa/c') - req.referer = 'http://www.example.com/index.html' - req.acl = '.r:.example.com,.rlistings' - self.assertEquals(self.test_auth.authorize(req), None) - - def test_handle_x_storage_token(self): - req = self._make_request( - '/auth/v1.0', - headers={'x-storage-token': 'blahblah', }) - resp = req.get_response(self.test_auth) - self.assertEquals(resp.status_int, REDIRECT_STATUS) - - def test_invalid_token(self): - req = self._make_request('/k1/test') - req.environ['HTTP_X_AUTH_TOKEN'] = 'AUTH_blahblahblah' - resp = req.get_response(self.test_auth) - self.assertEquals(resp.status_int, REDIRECT_STATUS) - -if __name__ == '__main__': - unittest.main() diff --git a/test/unit/common/middleware/swiftkerbauth/test_kerbauth_utils.py b/test/unit/common/middleware/swiftkerbauth/test_kerbauth_utils.py deleted file mode 100644 index 2a4e90b..0000000 --- a/test/unit/common/middleware/swiftkerbauth/test_kerbauth_utils.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright (c) 2013 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest -import re -from time import time -from test.unit import FakeMemcache -from gluster.swift.common.middleware.swiftkerbauth import kerbauth_utils as ku - - -class TestKerbUtils(unittest.TestCase): - - def test_get_remote_user(self): - env = {'REMOTE_USER': "auth_admin@EXAMPLE.COM"} - result = ku.get_remote_user(env) - self.assertEqual(result, "auth_admin") - - def test_get_remote_user_err(self): - env = {'REMOTE_USER': "auth_admin"} - try: - ku.get_remote_user(env) - except RuntimeError as err: - self.assertTrue(err.args[0].startswith("Malformed REMOTE_USER")) - else: - self.fail("Expected RuntimeError") - - def test_get_auth_data(self): - mc = FakeMemcache() - expiry = time() + 100 - ku.set_auth_data(mc, "root", "AUTH_tk", expiry, "root,admin") - (token, expires, groups) = ku.get_auth_data(mc, "root") - self.assertEqual(("AUTH_tk", expiry, "root,admin"), - (token, expires, groups)) - - def test_get_auth_data_err(self): - mc = FakeMemcache() - (token, expires, groups) = ku.get_auth_data(mc, "root") - self.assertEqual((token, expires, groups), (None, None, None)) - - expiry = time() - 1 - ku.set_auth_data(mc, "root", "AUTH_tk", expiry, "root,admin") - (token, expires, groups) = ku.get_auth_data(mc, "root") - self.assertEqual((token, expires, groups), (None, None, None)) - - def test_set_auth_data(self): - mc = FakeMemcache() - expiry = time() + 100 - ku.set_auth_data(mc, "root", "AUTH_tk", expiry, "root,admin") - - def test_generate_token(self): - token = ku.generate_token() - matches = re.match('AUTH_tk[a-f0-9]{32}', token) - self.assertTrue(matches is not None) - - def test_get_groups_from_username(self): - groups = ku.get_groups_from_username("root") - self.assertTrue("root" in groups) - - def test_get_groups_from_username_err(self): - try: - ku.get_groups_from_username("Zroot") - except RuntimeError as err: - self.assertTrue(err.args[0].startswith("Failure running id -G")) - else: - self.fail("Expected RuntimeError") diff --git a/test/unit/common/test_constraints.py b/test/unit/common/test_constraints.py index 6c78d75..e8ddd69 100644 --- a/test/unit/common/test_constraints.py +++ b/test/unit/common/test_constraints.py @@ -57,10 +57,10 @@ class TestConstraints(unittest.TestCase): cnt.set_object_name_component_length() self.assertEqual(len, cnt.get_object_name_component_length()) - with patch('swift.common.constraints.constraints_conf_int', - mock_constraints_conf_int): - cnt.set_object_name_component_length() - self.assertEqual(cnt.get_object_name_component_length(), 1000) + with patch('swift.common.constraints.constraints_conf_int', + mock_constraints_conf_int): + cnt.set_object_name_component_length() + self.assertEqual(cnt.get_object_name_component_length(), 1000) def test_validate_obj_name_component(self): max_obj_len = cnt.get_object_name_component_length() @@ -75,65 +75,6 @@ class TestConstraints(unittest.TestCase): self.assertTrue(cnt.validate_obj_name_component('..')) self.assertTrue(cnt.validate_obj_name_component('')) - def test_validate_headers(self): - req = Mock() - req.headers = [] - self.assertEqual(cnt.validate_headers(req), '') - req.headers = ['x-some-header'] - self.assertEqual(cnt.validate_headers(req), '') - #TODO: Although we now support x-delete-at and x-delete-after, - #retained this test case as we may add some other header to - #unsupported list in future - raise SkipTest - req.headers = ['x-delete-at', 'x-some-header'] - self.assertNotEqual(cnt.validate_headers(req), '') - req.headers = ['x-delete-after', 'x-some-header'] - self.assertNotEqual(cnt.validate_headers(req), '') - req.headers = ['x-delete-at', 'x-delete-after', 'x-some-header'] - self.assertNotEqual(cnt.validate_headers(req), '') - - def test_validate_headers_ignoring_config_set(self): - with patch('gluster.swift.common.constraints.' - 'Glusterfs._ignore_unsupported_headers', True): - req = Mock() - req.headers = [] - self.assertEqual(cnt.validate_headers(req), '') - req.headers = ['x-some-header'] - self.assertEqual(cnt.validate_headers(req), '') - #TODO: Although we now support x-delete-at and x-delete-after, - #retained this test case as we may add some other header to - #unsupported list in future - raise SkipTest - req.headers = ['x-delete-at', 'x-some-header'] - self.assertEqual(cnt.validate_headers(req), '') - req.headers = ['x-delete-after', 'x-some-header'] - self.assertEqual(cnt.validate_headers(req), '') - req.headers = ['x-delete-at', 'x-delete-after', 'x-some-header'] - self.assertEqual(cnt.validate_headers(req), '') - - def test_gluster_check_metadata(self): - mock_check_metadata = Mock() - with patch('gluster.swift.common.constraints.__check_metadata', - mock_check_metadata): - req = Mock() - req.headers = [] - cnt.gluster_check_metadata(req, 'object') - self.assertTrue(1, mock_check_metadata.call_count) - cnt.gluster_check_metadata(req, 'object', POST=False) - self.assertTrue(1, mock_check_metadata.call_count) - req.headers = ['x-some-header'] - self.assertEqual(cnt.gluster_check_metadata(req, 'object', POST=False), None) - #TODO: Although we now support x-delete-at and x-delete-after, - #retained this test case as we may add some other header to - #unsupported list in future - raise SkipTest - req.headers = ['x-delete-at', 'x-some-header'] - self.assertNotEqual(cnt.gluster_check_metadata(req, 'object', POST=False), None) - req.headers = ['x-delete-after', 'x-some-header'] - self.assertNotEqual(cnt.gluster_check_metadata(req, 'object', POST=False), None) - req.headers = ['x-delete-at', 'x-delete-after', 'x-some-header'] - self.assertNotEqual(cnt.gluster_check_metadata(req, 'object', POST=False), None) - def test_gluster_check_object_creation(self): with patch('gluster.swift.common.constraints.__check_object_creation', mock_check_object_creation): @@ -147,9 +88,3 @@ class TestConstraints(unittest.TestCase): req = Mock() req.headers = [] self.assertTrue(cnt.gluster_check_object_creation(req, 'dir/.')) - #TODO: Although we now support x-delete-at and x-delete-after, - #retained this test case as we may add some other header to - #unsupported list in future - raise SkipTest - req.headers = ['x-delete-at'] - self.assertTrue(cnt.gluster_check_object_creation(req, 'dir/z')) diff --git a/test/unit/common/test_diskdir.py b/test/unit/common/test_diskdir.py index f32c3ad..ebc554a 100644 --- a/test/unit/common/test_diskdir.py +++ b/test/unit/common/test_diskdir.py @@ -408,7 +408,7 @@ class TestContainerBroker(unittest.TestCase): self.assertEqual(os.path.basename(broker.db_file), 'db_file.db') broker.initialize(self.initial_ts) self.assertTrue(os.path.isdir(self.container)) - self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP]) + self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP][0]) self.assertFalse(broker.is_deleted()) def test_creation_existing(self): @@ -419,7 +419,7 @@ class TestContainerBroker(unittest.TestCase): self.assertEqual(os.path.basename(broker.db_file), 'db_file.db') broker.initialize(self.initial_ts) self.assertTrue(os.path.isdir(self.container)) - self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP]) + self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP][0]) self.assertFalse(broker.is_deleted()) def test_creation_existing_bad_metadata(self): @@ -432,7 +432,7 @@ class TestContainerBroker(unittest.TestCase): self.assertEqual(os.path.basename(broker.db_file), 'db_file.db') broker.initialize(self.initial_ts) self.assertTrue(os.path.isdir(self.container)) - self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP]) + self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP][0]) self.assertFalse(broker.is_deleted()) def test_empty(self): @@ -954,7 +954,7 @@ class TestContainerBroker(unittest.TestCase): self.assertEqual(os.path.basename(broker.db_file), 'db_file.db') broker.initialize(self.initial_ts) self.assertTrue(os.path.isdir(self.container)) - self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP]) + self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP][0]) self.assertFalse(broker.is_deleted()) broker.delete_db(normalize_timestamp(time())) self.assertTrue(broker.is_deleted()) @@ -1005,7 +1005,7 @@ class TestAccountBroker(unittest.TestCase): self.assertEqual(os.path.basename(broker.db_file), 'db_file.db') broker.initialize(self.initial_ts) self.assertTrue(os.path.isdir(self.drive_fullpath)) - self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP]) + self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP][0]) self.assertFalse(broker.is_deleted()) def test_creation_bad_metadata(self): @@ -1016,7 +1016,7 @@ class TestAccountBroker(unittest.TestCase): self.assertEqual(os.path.basename(broker.db_file), 'db_file.db') broker.initialize(self.initial_ts) self.assertTrue(os.path.isdir(self.drive_fullpath)) - self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP]) + self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP][0]) self.assertFalse(broker.is_deleted()) def test_empty(self): @@ -1219,7 +1219,7 @@ class TestAccountBroker(unittest.TestCase): self.assertEqual(os.path.basename(broker.db_file), 'db_file.db') broker.initialize(self.initial_ts) self.assertTrue(os.path.isdir(self.drive_fullpath)) - self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP]) + self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP][0]) self.assertFalse(broker.is_deleted()) broker.delete_db(normalize_timestamp(time())) # Deleting the "db" should be a NOOP diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index dd03bd8..55bd0cf 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -25,9 +25,10 @@ import hashlib import tarfile import shutil from collections import defaultdict -from mock import patch +from mock import patch, Mock from gluster.swift.common import utils, Glusterfs -from gluster.swift.common.exceptions import GlusterFileSystemOSError +from gluster.swift.common.exceptions import GlusterFileSystemOSError,\ + GlusterFileSystemIOError from swift.common.exceptions import DiskFileNoSpace # @@ -712,6 +713,18 @@ class TestUtils(unittest.TestCase): ret = utils.validate_object(md) assert ret + def test_validate_object_with_stat(self): + md = {utils.X_TIMESTAMP: 'na', + utils.X_CONTENT_TYPE: 'na', + utils.X_ETAG: 'bad', + utils.X_CONTENT_LENGTH: '12345', + utils.X_TYPE: utils.OBJECT, + utils.X_OBJECT_TYPE: 'na'} + fake_stat = Mock(st_size=12346) + self.assertFalse(utils.validate_object(md, fake_stat)) + fake_stat = Mock(st_size=12345) + self.assertTrue(utils.validate_object(md, fake_stat)) + class TestUtilsDirObjects(unittest.TestCase): @@ -794,7 +807,8 @@ class TestUtilsDirObjects(unittest.TestCase): def _mock_rm(path): print "_mock_rm-metadata_enoent(%s)" % path shutil.rmtree(path) - raise OSError(errno.ENOENT, os.strerror(errno.ENOENT)) + raise GlusterFileSystemIOError(errno.ENOENT, + os.strerror(errno.ENOENT)) # Remove the files for f in self.files: @@ -805,8 +819,8 @@ class TestUtilsDirObjects(unittest.TestCase): try: try: self.assertTrue(utils.rmobjdir(self.rootdir)) - except OSError: - self.fail("Unexpected OSError") + except IOError: + self.fail("Unexpected IOError") else: pass finally: diff --git a/test/unit/obj/test_diskfile.py b/test/unit/obj/test_diskfile.py index f8c26db..1fe0904 100644 --- a/test/unit/obj/test_diskfile.py +++ b/test/unit/obj/test_diskfile.py @@ -223,7 +223,7 @@ class TestDiskFile(unittest.TestCase): ini_md = { 'X-Type': 'Object', 'X-Object-Type': 'file', - 'Content-Length': 5, + 'Content-Length': 4, 'ETag': 'etag', 'X-Timestamp': 'ts', 'Content-Type': 'application/loctet-stream'} @@ -283,9 +283,8 @@ class TestDiskFile(unittest.TestCase): with gdf.open(): assert gdf._is_dir assert gdf._data_file == the_dir - assert gdf._metadata == exp_md - def _create_and_get_diskfile(self, dev, par, acc, con, obj): + def _create_and_get_diskfile(self, dev, par, acc, con, obj, fsize=256): # FIXME: assumes account === volume the_path = os.path.join(self.td, dev, con) the_file = os.path.join(the_path, obj) @@ -293,7 +292,7 @@ class TestDiskFile(unittest.TestCase): base_dir = os.path.dirname(the_file) os.makedirs(base_dir) with open(the_file, "wb") as fd: - fd.write("y" * 256) + fd.write("y" * fsize) gdf = self._get_diskfile(dev, par, acc, con, obj) assert gdf._obj == base_obj assert not gdf._is_dir @@ -353,6 +352,26 @@ class TestDiskFile(unittest.TestCase): assert len(chunks) == 1, repr(chunks) assert called[0] == 1, called + def test_reader_larger_file(self): + closed = [False] + fd = [-1] + + def mock_close(*args, **kwargs): + closed[0] = True + os.close(fd[0]) + + with mock.patch("gluster.swift.obj.diskfile.do_close", mock_close): + gdf = self._create_and_get_diskfile("vol0", "p57", "ufo47", "bar", "z", fsize=1024*1024*2) + with gdf.open(): + assert gdf._fd is not None + assert gdf._data_file == os.path.join(self.td, "vol0", "bar", "z") + reader = gdf.reader() + assert reader._fd is not None + fd[0] = reader._fd + chunks = [ck for ck in reader] + assert reader._fd is None + assert closed[0] + def test_reader_dir_object(self): called = [False] @@ -926,3 +945,31 @@ class TestDiskFile(unittest.TestCase): dw.write("123") os.unlink(saved_tmppath) assert not os.path.exists(saved_tmppath) + + def test_unlink_not_called_after_rename(self): + the_obj_path = os.path.join("b", "a") + the_file = os.path.join(the_obj_path, "z") + gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", the_file) + + body = '1234\n' + etag = md5(body).hexdigest() + metadata = { + 'X-Timestamp': '1234', + 'Content-Type': 'file', + 'ETag': etag, + 'Content-Length': '5', + } + + _mock_do_unlink = Mock() # Shouldn't be called + with patch("gluster.swift.obj.diskfile.do_unlink", _mock_do_unlink): + with gdf.create() as dw: + assert dw._tmppath is not None + tmppath = dw._tmppath + dw.write(body) + dw.put(metadata) + # do_unlink is not called if dw._tmppath is set to None + assert dw._tmppath is None + self.assertFalse(_mock_do_unlink.called) + + assert os.path.exists(gdf._data_file) # Real file exists + assert not os.path.exists(tmppath) # Temp file does not exist diff --git a/test/unit/obj/test_expirer.py b/test/unit/obj/test_expirer.py index 4329eef..236775e 100644 --- a/test/unit/obj/test_expirer.py +++ b/test/unit/obj/test_expirer.py @@ -46,7 +46,7 @@ class TestObjectExpirer(TestCase): self.old_loadapp = internal_client.loadapp self.old_sleep = internal_client.sleep - internal_client.loadapp = lambda x: None + internal_client.loadapp = lambda *a, **kw: None internal_client.sleep = not_sleep def teardown(self): @@ -618,7 +618,7 @@ class TestObjectExpirer(TestCase): start_response('204 No Content', [('Content-Length', '0')]) return [] - internal_client.loadapp = lambda x: fake_app + internal_client.loadapp = lambda *a, **kw: fake_app x = expirer.ObjectExpirer({}) ts = '1234' @@ -635,7 +635,7 @@ class TestObjectExpirer(TestCase): start_response('204 No Content', [('Content-Length', '0')]) return [] - internal_client.loadapp = lambda x: fake_app + internal_client.loadapp = lambda *a, **kw: fake_app x = expirer.ObjectExpirer({}) ts = '1234' @@ -649,7 +649,7 @@ class TestObjectExpirer(TestCase): start_response('404 Not Found', [('Content-Length', '0')]) return [] - internal_client.loadapp = lambda x: fake_app + internal_client.loadapp = lambda *a, **kw: fake_app x = expirer.ObjectExpirer({}) x.delete_actual_object('/path/to/object', '1234') @@ -661,7 +661,7 @@ class TestObjectExpirer(TestCase): [('Content-Length', '0')]) return [] - internal_client.loadapp = lambda x: fake_app + internal_client.loadapp = lambda *a, **kw: fake_app x = expirer.ObjectExpirer({}) x.delete_actual_object('/path/to/object', '1234') @@ -674,7 +674,7 @@ class TestObjectExpirer(TestCase): [('Content-Length', '0')]) return [] - internal_client.loadapp = lambda x: fake_app + internal_client.loadapp = lambda *a, **kw: fake_app x = expirer.ObjectExpirer({}) exc = None @@ -692,7 +692,7 @@ class TestObjectExpirer(TestCase): x = expirer.ObjectExpirer({}) x.swift.make_request = mock.MagicMock() x.delete_actual_object(name, timestamp) - x.swift.make_request.assert_called_once() + self.assertTrue(x.swift.make_request.called) self.assertEqual(x.swift.make_request.call_args[0][1], '/v1/' + urllib.quote(name)) diff --git a/test/unit/proxy/controllers/test_obj.py b/test/unit/proxy/controllers/test_obj.py index aada616..4942691 100755 --- a/test/unit/proxy/controllers/test_obj.py +++ b/test/unit/proxy/controllers/test_obj.py @@ -22,7 +22,7 @@ import mock import swift from swift.proxy import server as proxy_server from swift.common.swob import HTTPException -from test.unit import FakeRing, FakeMemcache, fake_http_connect +from test.unit import FakeRing, FakeMemcache, fake_http_connect, debug_logger @contextmanager @@ -90,26 +90,96 @@ class TestObjControllerWriteAffinity(unittest.TestCase): class TestObjController(unittest.TestCase): + def setUp(self): + logger = debug_logger('proxy-server') + logger.thread_locals = ('txn1', '127.0.0.2') + self.app = proxy_server.Application( + None, FakeMemcache(), account_ring=FakeRing(), + container_ring=FakeRing(), object_ring=FakeRing(), + logger=logger) + self.controller = proxy_server.ObjectController(self.app, + 'a', 'c', 'o') + self.controller.container_info = mock.MagicMock(return_value={ + 'partition': 1, + 'nodes': [ + {'ip': '127.0.0.1', 'port': '1', 'device': 'sda'}, + {'ip': '127.0.0.1', 'port': '2', 'device': 'sda'}, + {'ip': '127.0.0.1', 'port': '3', 'device': 'sda'}, + ], + 'write_acl': None, + 'read_acl': None, + 'sync_key': None, + 'versions': None}) + + def test_PUT_simple(self): + req = swift.common.swob.Request.blank('/v1/a/c/o') + req.headers['content-length'] = '0' + with set_http_connect(201, 201, 201): + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 201) + + def test_PUT_if_none_match(self): + req = swift.common.swob.Request.blank('/v1/a/c/o') + req.headers['if-none-match'] = '*' + req.headers['content-length'] = '0' + with set_http_connect(201, 201, 201): + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 201) + + def test_PUT_if_none_match_denied(self): + req = swift.common.swob.Request.blank('/v1/a/c/o') + req.headers['if-none-match'] = '*' + req.headers['content-length'] = '0' + with set_http_connect(201, (412, 412), 201): + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 412) + + def test_PUT_if_none_match_not_star(self): + req = swift.common.swob.Request.blank('/v1/a/c/o') + req.headers['if-none-match'] = 'somethingelse' + req.headers['content-length'] = '0' + with set_http_connect(201, 201, 201): + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 400) + + def test_GET_simple(self): + req = swift.common.swob.Request.blank('/v1/a/c/o') + with set_http_connect(200): + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 200) + + def test_DELETE_simple(self): + req = swift.common.swob.Request.blank('/v1/a/c/o') + with set_http_connect(204, 204, 204): + resp = self.controller.DELETE(req) + self.assertEquals(resp.status_int, 204) + + def test_POST_simple(self): + req = swift.common.swob.Request.blank('/v1/a/c/o') + with set_http_connect(200, 200, 200, 201, 201, 201): + resp = self.controller.POST(req) + self.assertEquals(resp.status_int, 202) + + def test_COPY_simple(self): + req = swift.common.swob.Request.blank('/v1/a/c/o') + with set_http_connect(200, 200, 200, 201, 201, 201): + resp = self.controller.POST(req) + self.assertEquals(resp.status_int, 202) + + def test_HEAD_simple(self): + req = swift.common.swob.Request.blank('/v1/a/c/o') + with set_http_connect(200, 200, 200, 201, 201, 201): + resp = self.controller.POST(req) + self.assertEquals(resp.status_int, 202) def test_PUT_log_info(self): # mock out enough to get to the area of the code we want to test with mock.patch('swift.proxy.controllers.obj.check_object_creation', mock.MagicMock(return_value=None)): - app = mock.MagicMock() - app.container_ring.get_nodes.return_value = (1, [2]) - app.object_ring.get_nodes.return_value = (1, [2]) - controller = proxy_server.ObjectController(app, 'a', 'c', 'o') - controller.container_info = mock.MagicMock(return_value={ - 'partition': 1, - 'nodes': [{}], - 'write_acl': None, - 'sync_key': None, - 'versions': None}) - # and now test that we add the header to log_info req = swift.common.swob.Request.blank('/v1/a/c/o') req.headers['x-copy-from'] = 'somewhere' try: - controller.PUT(req) + self.controller.PUT(req) except HTTPException: pass self.assertEquals( @@ -119,7 +189,7 @@ class TestObjController(unittest.TestCase): req.method = 'POST' req.headers['x-copy-from'] = 'elsewhere' try: - controller.PUT(req) + self.controller.PUT(req) except HTTPException: pass self.assertEquals(req.environ.get('swift.log_info'), None) diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 0cb2278..1a59016 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -30,6 +30,7 @@ from urllib import quote from hashlib import md5 from tempfile import mkdtemp import weakref +import re import mock from eventlet import sleep, spawn, wsgi, listen @@ -60,7 +61,8 @@ from swift.proxy.controllers.base import get_container_memcache_key, \ get_account_memcache_key, cors_validation import swift.proxy.controllers from swift.common.request_helpers import get_sys_meta_prefix -from swift.common.swob import Request, Response, HTTPUnauthorized +from swift.common.swob import Request, Response, HTTPUnauthorized, \ + HTTPException # mocks logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) @@ -245,6 +247,7 @@ def set_http_connect(*args, **kwargs): swift.proxy.controllers.obj.http_connect = new_connect swift.proxy.controllers.account.http_connect = new_connect swift.proxy.controllers.container.http_connect = new_connect + return new_connect # tests @@ -692,10 +695,10 @@ class TestObjectController(unittest.TestCase): def setUp(self): self.app = proxy_server.Application(None, FakeMemcache(), + logger=debug_logger('proxy-ut'), account_ring=FakeRing(), container_ring=FakeRing(), object_ring=FakeRing()) - monkey_patch_mimetools() def tearDown(self): self.app.account_ring.set_replicas(3) @@ -802,6 +805,7 @@ class TestObjectController(unittest.TestCase): self.app.update_request(req) self.app.memcache.store = {} res = controller.PUT(req) + self.assertEqual(test_errors, []) self.assertTrue(res.status.startswith('201 ')) def test_PUT_respects_write_affinity(self): @@ -1609,7 +1613,7 @@ class TestObjectController(unittest.TestCase): dev['ip'] = '127.0.0.1' dev['port'] = 1 - class SlowBody(): + class SlowBody(object): def __init__(self): self.sent = 0 @@ -1658,7 +1662,7 @@ class TestObjectController(unittest.TestCase): dev['ip'] = '127.0.0.1' dev['port'] = 1 - class SlowBody(): + class SlowBody(object): def __init__(self): self.sent = 0 @@ -1693,7 +1697,7 @@ class TestObjectController(unittest.TestCase): dev['port'] = 1 req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) self.app.update_request(req) - set_http_connect(200, 200, 200, slow=True) + set_http_connect(200, 200, 200, slow=0.1) req.sent_size = 0 resp = req.get_response(self.app) got_exc = False @@ -1703,7 +1707,7 @@ class TestObjectController(unittest.TestCase): got_exc = True self.assert_(not got_exc) self.app.recoverable_node_timeout = 0.1 - set_http_connect(200, 200, 200, slow=True) + set_http_connect(200, 200, 200, slow=1.0) resp = req.get_response(self.app) got_exc = False try: @@ -1718,16 +1722,17 @@ class TestObjectController(unittest.TestCase): self.app.update_request(req) self.app.recoverable_node_timeout = 0.1 - set_http_connect(200, 200, 200, slow=[3]) + set_http_connect(200, 200, 200, slow=[1.0, 1.0, 1.0]) resp = req.get_response(self.app) got_exc = False try: - resp.body + self.assertEquals('', resp.body) except ChunkReadTimeout: got_exc = True self.assert_(got_exc) - set_http_connect(200, 200, 200, body='lalala', slow=[2]) + set_http_connect(200, 200, 200, body='lalala', + slow=[1.0, 1.0]) resp = req.get_response(self.app) got_exc = False try: @@ -1736,8 +1741,8 @@ class TestObjectController(unittest.TestCase): got_exc = True self.assert_(not got_exc) - set_http_connect(200, 200, 200, body='lalala', slow=[2], - etags=['a', 'a', 'a']) + set_http_connect(200, 200, 200, body='lalala', + slow=[1.0, 1.0], etags=['a', 'a', 'a']) resp = req.get_response(self.app) got_exc = False try: @@ -1746,8 +1751,8 @@ class TestObjectController(unittest.TestCase): got_exc = True self.assert_(not got_exc) - set_http_connect(200, 200, 200, body='lalala', slow=[2], - etags=['a', 'b', 'a']) + set_http_connect(200, 200, 200, body='lalala', + slow=[1.0, 1.0], etags=['a', 'b', 'a']) resp = req.get_response(self.app) got_exc = False try: @@ -1757,8 +1762,8 @@ class TestObjectController(unittest.TestCase): self.assert_(not got_exc) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) - set_http_connect(200, 200, 200, body='lalala', slow=[2], - etags=['a', 'b', 'b']) + set_http_connect(200, 200, 200, body='lalala', + slow=[1.0, 1.0], etags=['a', 'b', 'b']) resp = req.get_response(self.app) got_exc = False try: @@ -1787,17 +1792,17 @@ class TestObjectController(unittest.TestCase): 'Content-Type': 'text/plain'}, body=' ') self.app.update_request(req) - set_http_connect(200, 200, 201, 201, 201, slow=True) + set_http_connect(200, 200, 201, 201, 201, slow=0.1) resp = req.get_response(self.app) self.assertEquals(resp.status_int, 201) self.app.node_timeout = 0.1 - set_http_connect(201, 201, 201, slow=True) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '4', 'Content-Type': 'text/plain'}, body=' ') self.app.update_request(req) + set_http_connect(201, 201, 201, slow=1.0) resp = req.get_response(self.app) self.assertEquals(resp.status_int, 503) @@ -2227,307 +2232,322 @@ class TestObjectController(unittest.TestCase): resp = controller.PUT(req) self.assertEquals(resp.status_int, 400) - def test_copy_from(self): + @contextmanager + def controller_context(self, req, *args, **kwargs): + _v, account, container, obj = utils.split_path(req.path, 4, 4, True) + controller = proxy_server.ObjectController(self.app, account, + container, obj) + self.app.update_request(req) + self.app.memcache.store = {} with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') - # initial source object PUT - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0'}) - self.app.update_request(req) - set_http_connect(200, 200, 201, 201, 201) - # acct cont obj obj obj + new_connect = set_http_connect(*args, **kwargs) + yield controller + unused_status_list = [] + while True: + try: + unused_status_list.append(new_connect.code_iter.next()) + except StopIteration: + break + if unused_status_list: + raise self.fail('UN-USED STATUS CODES: %r' % + unused_status_list) + + def test_basic_put_with_x_copy_from(self): + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': 'c/o'}) + status_list = (200, 200, 200, 200, 200, 201, 201, 201) + # acct cont objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: resp = controller.PUT(req) - self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers['x-copied-from'], 'c/o') - # basic copy - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': 'c/o'}) - self.app.update_request(req) - set_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acct cont objc objc objc obj obj obj - self.app.memcache.store = {} + def test_basic_put_with_x_copy_from_across_container(self): + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': 'c2/o'}) + status_list = (200, 200, 200, 200, 200, 200, 201, 201, 201) + # acct cont conc objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: resp = controller.PUT(req) - self.assertEquals(resp.status_int, 201) - self.assertEquals(resp.headers['x-copied-from'], 'c/o') + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers['x-copied-from'], 'c2/o') - # non-zero content length - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '5', - 'X-Copy-From': 'c/o'}) - self.app.update_request(req) - set_http_connect(200, 200, 200, 200, 200, 200, 200) - # acct cont acct cont objc objc objc - self.app.memcache.store = {} + def test_copy_non_zero_content_length(self): + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '5', + 'X-Copy-From': 'c/o'}) + status_list = (200, 200) + # acct cont + with self.controller_context(req, *status_list) as controller: resp = controller.PUT(req) - self.assertEquals(resp.status_int, 400) + self.assertEquals(resp.status_int, 400) - # extra source path parsing - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': 'c/o/o2'}) - req.account = 'a' - set_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acct cont objc objc objc obj obj obj - self.app.memcache.store = {} + def test_copy_with_slashes_in_x_copy_from(self): + # extra source path parsing + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': 'c/o/o2'}) + status_list = (200, 200, 200, 200, 200, 201, 201, 201) + # acct cont objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: resp = controller.PUT(req) - self.assertEquals(resp.status_int, 201) - self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2') + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2') - # space in soure path - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': 'c/o%20o2'}) - req.account = 'a' - set_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acct cont objc objc objc obj obj obj - self.app.memcache.store = {} + def test_copy_with_spaces_in_x_copy_from(self): + # space in soure path + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': 'c/o%20o2'}) + status_list = (200, 200, 200, 200, 200, 201, 201, 201) + # acct cont objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: resp = controller.PUT(req) - self.assertEquals(resp.status_int, 201) - self.assertEquals(resp.headers['x-copied-from'], 'c/o%20o2') + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers['x-copied-from'], 'c/o%20o2') - # repeat tests with leading / - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o'}) - self.app.update_request(req) - set_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acct cont objc objc objc obj obj obj - self.app.memcache.store = {} + def test_copy_with_leading_slash_in_x_copy_from(self): + # repeat tests with leading / + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o'}) + status_list = (200, 200, 200, 200, 200, 201, 201, 201) + # acct cont objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: resp = controller.PUT(req) - self.assertEquals(resp.status_int, 201) - self.assertEquals(resp.headers['x-copied-from'], 'c/o') + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers['x-copied-from'], 'c/o') - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o/o2'}) - req.account = 'a' - set_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acct cont objc objc objc obj obj obj - self.app.memcache.store = {} + def test_copy_with_leading_slash_and_slashes_in_x_copy_from(self): + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o/o2'}) + status_list = (200, 200, 200, 200, 200, 201, 201, 201) + # acct cont objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: resp = controller.PUT(req) - self.assertEquals(resp.status_int, 201) - self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2') + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2') - # negative tests + def test_copy_with_no_object_in_x_copy_from(self): + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c'}) + status_list = (200, 200) + # acct cont + with self.controller_context(req, *status_list) as controller: + try: + controller.PUT(req) + except HTTPException as resp: + self.assertEquals(resp.status_int // 100, 4) # client error + else: + raise self.fail('Invalid X-Copy-From did not raise ' + 'client error') - # invalid x-copy-from path - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c'}) - self.app.update_request(req) - self.app.memcache.store = {} + def test_copy_server_error_reading_source(self): + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o'}) + status_list = (200, 200, 503, 503, 503) + # acct cont objc objc objc + with self.controller_context(req, *status_list) as controller: resp = controller.PUT(req) - self.assertEquals(resp.status_int // 100, 4) # client error + self.assertEquals(resp.status_int, 503) - # server error - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o'}) - self.app.update_request(req) - set_http_connect(200, 200, 503, 503, 503) - # acct cont objc objc objc - self.app.memcache.store = {} + def test_copy_not_found_reading_source(self): + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o'}) + # not found + status_list = (200, 200, 404, 404, 404) + # acct cont objc objc objc + with self.controller_context(req, *status_list) as controller: resp = controller.PUT(req) - self.assertEquals(resp.status_int, 503) + self.assertEquals(resp.status_int, 404) - # not found - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o'}) - self.app.update_request(req) - set_http_connect(200, 200, 404, 404, 404) - # acct cont objc objc objc - self.app.memcache.store = {} + def test_copy_with_some_missing_sources(self): + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o'}) + status_list = (200, 200, 404, 404, 200, 201, 201, 201) + # acct cont objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: resp = controller.PUT(req) - self.assertEquals(resp.status_int, 404) + self.assertEquals(resp.status_int, 201) - # some missing containers - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o'}) - self.app.update_request(req) - set_http_connect(200, 200, 404, 404, 200, 201, 201, 201) - # acct cont objc objc objc obj obj obj - self.app.memcache.store = {} + def test_copy_with_object_metadata(self): + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o', + 'X-Object-Meta-Ours': 'okay'}) + # test object metadata + status_list = (200, 200, 200, 200, 200, 201, 201, 201) + # acct cont objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: resp = controller.PUT(req) - self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers.get('x-object-meta-test'), 'testing') + self.assertEquals(resp.headers.get('x-object-meta-ours'), 'okay') + self.assertEquals(resp.headers.get('x-delete-at'), '9876543210') - # test object meta data - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o', - 'X-Object-Meta-Ours': 'okay'}) - self.app.update_request(req) - set_http_connect(200, 200, 200, 200, 200, 201, 201, 201) - # acct cont objc objc objc obj obj obj - self.app.memcache.store = {} - resp = controller.PUT(req) - self.assertEquals(resp.status_int, 201) - self.assertEquals(resp.headers.get('x-object-meta-test'), - 'testing') - self.assertEquals(resp.headers.get('x-object-meta-ours'), 'okay') - self.assertEquals(resp.headers.get('x-delete-at'), '9876543210') + def test_copy_source_larger_than_max_file_size(self): + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '0', + 'X-Copy-From': '/c/o'}) - # copy-from object is too large to fit in target object - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0', - 'X-Copy-From': '/c/o'}) - self.app.update_request(req) + # copy-from object is too large to fit in target object + class LargeResponseBody(object): - class LargeResponseBody(object): + def __len__(self): + return MAX_FILE_SIZE + 1 - def __len__(self): - return MAX_FILE_SIZE + 1 + def __getitem__(self, key): + return '' - def __getitem__(self, key): - return '' + copy_from_obj_body = LargeResponseBody() + status_list = (200, 200, 200, 200, 200) + # acct cont objc objc objc + kwargs = dict(body=copy_from_obj_body) + with self.controller_context(req, *status_list, + **kwargs) as controller: + self.app.update_request(req) - copy_from_obj_body = LargeResponseBody() - set_http_connect(200, 200, 200, 200, 200, 201, 201, 201, - body=copy_from_obj_body) self.app.memcache.store = {} resp = controller.PUT(req) self.assertEquals(resp.status_int, 413) - def test_COPY(self): - with save_globals(): - controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') - req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'Content-Length': '0'}) - req.account = 'a' - set_http_connect(200, 200, 201, 201, 201) - # acct cont obj obj obj - resp = controller.PUT(req) - self.assertEquals(resp.status_int, 201) - - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': 'c/o'}) - req.account = 'a' - set_http_connect(200, 200, 200, 200, 200, 201, 201, 201, 200, 200) - # acct cont acct cont objc objc objc obj obj obj - self.app.memcache.store = {} + def test_basic_COPY(self): + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': 'c/o2'}) + status_list = (200, 200, 200, 200, 200, 201, 201, 201) + # acct cont objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: resp = controller.COPY(req) - self.assertEquals(resp.status_int, 201) - self.assertEquals(resp.headers['x-copied-from'], 'c/o') + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers['x-copied-from'], 'c/o') - req = Request.blank('/v1/a/c/o/o2', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': 'c/o'}) - req.account = 'a' - controller.object_name = 'o/o2' - set_http_connect(200, 200, 200, 200, 200, 201, 201, 201, 200, 200) - # acct cont acct cont objc objc objc obj obj obj - self.app.memcache.store = {} + def test_COPY_across_containers(self): + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': 'c2/o'}) + status_list = (200, 200, 200, 200, 200, 200, 201, 201, 201) + # acct cont c2 objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: resp = controller.COPY(req) - self.assertEquals(resp.status_int, 201) - self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2') - - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c/o'}) - req.account = 'a' - controller.object_name = 'o' - set_http_connect(200, 200, 200, 200, 200, 201, 201, 201, 200, 200) - # acct cont acct cont objc objc objc obj obj obj - self.app.memcache.store = {} + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers['x-copied-from'], 'c/o') + + def test_COPY_source_with_slashes_in_name(self): + req = Request.blank('/v1/a/c/o/o2', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': 'c/o'}) + status_list = (200, 200, 200, 200, 200, 201, 201, 201) + # acct cont objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: resp = controller.COPY(req) - self.assertEquals(resp.status_int, 201) - self.assertEquals(resp.headers['x-copied-from'], 'c/o') + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2') - req = Request.blank('/v1/a/c/o/o2', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c/o'}) - req.account = 'a' - controller.object_name = 'o/o2' - set_http_connect(200, 200, 200, 200, 200, 201, 201, 201, 200, 200) - # acct cont acct cont objc objc objc obj obj obj - self.app.memcache.store = {} + def test_COPY_destination_leading_slash(self): + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c/o'}) + status_list = (200, 200, 200, 200, 200, 201, 201, 201) + # acct cont objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: resp = controller.COPY(req) - self.assertEquals(resp.status_int, 201) - self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2') + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers['x-copied-from'], 'c/o') + + def test_COPY_source_with_slashes_destination_leading_slash(self): + req = Request.blank('/v1/a/c/o/o2', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c/o'}) + status_list = (200, 200, 200, 200, 200, 201, 201, 201) + # acct cont objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: + resp = controller.COPY(req) + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2') - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': 'c_o'}) - req.account = 'a' - controller.object_name = 'o' - set_http_connect(200, 200) - # acct cont - self.app.memcache.store = {} + def test_COPY_no_object_in_destination(self): + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': 'c_o'}) + status_list = [] # no requests needed + with self.controller_context(req, *status_list) as controller: resp = controller.COPY(req) - self.assertEquals(resp.status_int, 412) + self.assertEquals(resp.status_int, 412) - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c/o'}) - req.account = 'a' - controller.object_name = 'o' - set_http_connect(200, 200, 503, 503, 503) - # acct cont objc objc objc - self.app.memcache.store = {} + def test_COPY_server_error_reading_source(self): + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c/o'}) + status_list = (200, 200, 503, 503, 503) + # acct cont objc objc objc + with self.controller_context(req, *status_list) as controller: resp = controller.COPY(req) - self.assertEquals(resp.status_int, 503) + self.assertEquals(resp.status_int, 503) - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c/o'}) - req.account = 'a' - controller.object_name = 'o' - set_http_connect(200, 200, 404, 404, 404) - # acct cont objc objc objc - self.app.memcache.store = {} + def test_COPY_not_found_reading_source(self): + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c/o'}) + status_list = (200, 200, 404, 404, 404) + # acct cont objc objc objc + with self.controller_context(req, *status_list) as controller: resp = controller.COPY(req) - self.assertEquals(resp.status_int, 404) + self.assertEquals(resp.status_int, 404) - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c/o'}) - req.account = 'a' - controller.object_name = 'o' - set_http_connect(200, 200, 404, 404, 200, 201, 201, 201) - # acct cont objc objc objc obj obj obj - self.app.memcache.store = {} + def test_COPY_with_some_missing_sources(self): + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c/o'}) + status_list = (200, 200, 404, 404, 200, 201, 201, 201) + # acct cont objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: resp = controller.COPY(req) - self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.status_int, 201) - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c/o', - 'X-Object-Meta-Ours': 'okay'}) - req.account = 'a' - controller.object_name = 'o' - set_http_connect(200, 200, 200, 200, 200, 201, 201, 201) - # acct cont objc objc objc obj obj obj - self.app.memcache.store = {} + def test_COPY_with_metadata(self): + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c/o', + 'X-Object-Meta-Ours': 'okay'}) + status_list = (200, 200, 200, 200, 200, 201, 201, 201) + # acct cont objc objc objc obj obj obj + with self.controller_context(req, *status_list) as controller: resp = controller.COPY(req) - self.assertEquals(resp.status_int, 201) - self.assertEquals(resp.headers.get('x-object-meta-test'), - 'testing') - self.assertEquals(resp.headers.get('x-object-meta-ours'), 'okay') - self.assertEquals(resp.headers.get('x-delete-at'), '9876543210') + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers.get('x-object-meta-test'), + 'testing') + self.assertEquals(resp.headers.get('x-object-meta-ours'), 'okay') + self.assertEquals(resp.headers.get('x-delete-at'), '9876543210') - req = Request.blank('/v1/a/c/o', - environ={'REQUEST_METHOD': 'COPY'}, - headers={'Destination': '/c/o'}) - self.app.update_request(req) + def test_COPY_source_larger_than_max_file_size(self): + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c/o'}) - class LargeResponseBody(object): + class LargeResponseBody(object): - def __len__(self): - return MAX_FILE_SIZE + 1 + def __len__(self): + return MAX_FILE_SIZE + 1 - def __getitem__(self, key): - return '' + def __getitem__(self, key): + return '' - copy_from_obj_body = LargeResponseBody() - set_http_connect(200, 200, 200, 200, 200, 201, 201, 201, - body=copy_from_obj_body) - self.app.memcache.store = {} + copy_from_obj_body = LargeResponseBody() + status_list = (200, 200, 200, 200, 200) + # acct cont objc objc objc + kwargs = dict(body=copy_from_obj_body) + with self.controller_context(req, *status_list, + **kwargs) as controller: resp = controller.COPY(req) - self.assertEquals(resp.status_int, 413) + self.assertEquals(resp.status_int, 413) def test_COPY_newest(self): with save_globals(): @@ -2574,7 +2594,7 @@ class TestObjectController(unittest.TestCase): def test_chunked_put(self): - class ChunkedFile(): + class ChunkedFile(object): def __init__(self, bytes): self.bytes = bytes @@ -3824,9 +3844,7 @@ class TestObjectController(unittest.TestCase): req.content_length = 0 resp = controller.OPTIONS(req) self.assertEquals(200, resp.status_int) - self.assertEquals( - 'https://bar.baz', - resp.headers['access-control-allow-origin']) + self.assertEquals('*', resp.headers['access-control-allow-origin']) for verb in 'OPTIONS COPY GET POST PUT DELETE HEAD'.split(): self.assertTrue( verb in resp.headers['access-control-allow-methods']) @@ -3842,10 +3860,11 @@ class TestObjectController(unittest.TestCase): def stubContainerInfo(*args): return { 'cors': { - 'allow_origin': 'http://foo.bar' + 'allow_origin': 'http://not.foo.bar' } } controller.container_info = stubContainerInfo + controller.app.strict_cors_mode = False def objectGET(controller, req): return Response(headers={ @@ -3876,6 +3895,50 @@ class TestObjectController(unittest.TestCase): 'x-trans-id', 'x-object-meta-color']) self.assertEquals(expected_exposed, exposed) + controller.app.strict_cors_mode = True + req = Request.blank( + '/v1/a/c/o.jpg', + {'REQUEST_METHOD': 'GET'}, + headers={'Origin': 'http://foo.bar'}) + + resp = cors_validation(objectGET)(controller, req) + + self.assertEquals(200, resp.status_int) + self.assertTrue('access-control-allow-origin' not in resp.headers) + + def test_CORS_valid_with_obj_headers(self): + with save_globals(): + controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') + + def stubContainerInfo(*args): + return { + 'cors': { + 'allow_origin': 'http://foo.bar' + } + } + controller.container_info = stubContainerInfo + + def objectGET(controller, req): + return Response(headers={ + 'X-Object-Meta-Color': 'red', + 'X-Super-Secret': 'hush', + 'Access-Control-Allow-Origin': 'http://obj.origin', + 'Access-Control-Expose-Headers': 'x-trans-id' + }) + + req = Request.blank( + '/v1/a/c/o.jpg', + {'REQUEST_METHOD': 'GET'}, + headers={'Origin': 'http://foo.bar'}) + + resp = cors_validation(objectGET)(controller, req) + + self.assertEquals(200, resp.status_int) + self.assertEquals('http://obj.origin', + resp.headers['access-control-allow-origin']) + self.assertEquals('x-trans-id', + resp.headers['access-control-expose-headers']) + def _gather_x_container_headers(self, controller_call, req, *connect_args, **kwargs): header_list = kwargs.pop('header_list', ['X-Container-Device', @@ -4092,7 +4155,8 @@ class TestContainerController(unittest.TestCase): self.app = proxy_server.Application(None, FakeMemcache(), account_ring=FakeRing(), container_ring=FakeRing(), - object_ring=FakeRing()) + object_ring=FakeRing(), + logger=FakeLogger()) def test_transfer_headers(self): src_headers = {'x-remove-versions-location': 'x', @@ -4843,9 +4907,7 @@ class TestContainerController(unittest.TestCase): req.content_length = 0 resp = controller.OPTIONS(req) self.assertEquals(200, resp.status_int) - self.assertEquals( - 'https://bar.baz', - resp.headers['access-control-allow-origin']) + self.assertEquals('*', resp.headers['access-control-allow-origin']) for verb in 'OPTIONS GET POST PUT DELETE HEAD'.split(): self.assertTrue( verb in resp.headers['access-control-allow-methods']) @@ -5016,11 +5078,60 @@ class TestContainerController(unittest.TestCase): 'X-Account-Device': 'sdc'} ]) + def test_PUT_backed_x_timestamp_header(self): + timestamps = [] + + def capture_timestamps(*args, **kwargs): + headers = kwargs['headers'] + timestamps.append(headers.get('X-Timestamp')) + + req = Request.blank('/v1/a/c', method='PUT', headers={'': ''}) + with save_globals(): + new_connect = set_http_connect(200, # account existance check + 201, 201, 201, + give_connect=capture_timestamps) + resp = self.app.handle_request(req) + + # sanity + self.assertRaises(StopIteration, new_connect.code_iter.next) + self.assertEqual(2, resp.status_int // 100) + + timestamps.pop(0) # account existance check + self.assertEqual(3, len(timestamps)) + for timestamp in timestamps: + self.assertEqual(timestamp, timestamps[0]) + self.assert_(re.match('[0-9]{10}\.[0-9]{5}', timestamp)) + + def test_DELETE_backed_x_timestamp_header(self): + timestamps = [] + + def capture_timestamps(*args, **kwargs): + headers = kwargs['headers'] + timestamps.append(headers.get('X-Timestamp')) + + req = Request.blank('/v1/a/c', method='DELETE', headers={'': ''}) + self.app.update_request(req) + with save_globals(): + new_connect = set_http_connect(200, # account existance check + 201, 201, 201, + give_connect=capture_timestamps) + resp = self.app.handle_request(req) + + # sanity + self.assertRaises(StopIteration, new_connect.code_iter.next) + self.assertEqual(2, resp.status_int // 100) + + timestamps.pop(0) # account existance check + self.assertEqual(3, len(timestamps)) + for timestamp in timestamps: + self.assertEqual(timestamp, timestamps[0]) + self.assert_(re.match('[0-9]{10}\.[0-9]{5}', timestamp)) + def test_node_read_timeout_retry_to_container(self): with save_globals(): req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'GET'}) self.app.node_timeout = 0.1 - set_http_connect(200, 200, 200, body='abcdef', slow=[2]) + set_http_connect(200, 200, 200, body='abcdef', slow=[1.0, 1.0]) resp = req.get_response(self.app) got_exc = False try: @@ -5547,6 +5658,143 @@ class TestAccountControllerFakeGetResponse(unittest.TestCase): resp = req.get_response(self.app) self.assertEqual(400, resp.status_int) + def test_account_acl_header_access(self): + acl = { + 'admin': ['AUTH_alice'], + 'read-write': ['AUTH_bob'], + 'read-only': ['AUTH_carol'], + } + prefix = get_sys_meta_prefix('account') + privileged_headers = {(prefix + 'core-access-control'): format_acl( + version=2, acl_dict=acl)} + + app = proxy_server.Application( + None, FakeMemcache(), account_ring=FakeRing(), + container_ring=FakeRing(), object_ring=FakeRing()) + + with save_globals(): + # Mock account server will provide privileged information (ACLs) + set_http_connect(200, 200, 200, headers=privileged_headers) + req = Request.blank('/v1/a', environ={'REQUEST_METHOD': 'GET'}) + resp = app.handle_request(req) + + # Not a swift_owner -- ACLs should NOT be in response + header = 'X-Account-Access-Control' + self.assert_(header not in resp.headers, '%r was in %r' % ( + header, resp.headers)) + + # Same setup -- mock acct server will provide ACLs + set_http_connect(200, 200, 200, headers=privileged_headers) + req = Request.blank('/v1/a', environ={'REQUEST_METHOD': 'GET', + 'swift_owner': True}) + resp = app.handle_request(req) + + # For a swift_owner, the ACLs *should* be in response + self.assert_(header in resp.headers, '%r not in %r' % ( + header, resp.headers)) + + def test_account_acls_through_delegation(self): + + # Define a way to grab the requests sent out from the AccountController + # to the Account Server, and a way to inject responses we'd like the + # Account Server to return. + resps_to_send = [] + + @contextmanager + def patch_account_controller_method(verb): + old_method = getattr(proxy_server.AccountController, verb) + new_method = lambda self, req, *_, **__: resps_to_send.pop(0) + try: + setattr(proxy_server.AccountController, verb, new_method) + yield + finally: + setattr(proxy_server.AccountController, verb, old_method) + + def make_test_request(http_method, swift_owner=True): + env = { + 'REQUEST_METHOD': http_method, + 'swift_owner': swift_owner, + } + acl = { + 'admin': ['foo'], + 'read-write': ['bar'], + 'read-only': ['bas'], + } + headers = {} if http_method in ('GET', 'HEAD') else { + 'x-account-access-control': format_acl(version=2, acl_dict=acl) + } + + return Request.blank('/v1/a', environ=env, headers=headers) + + # Our AccountController will invoke methods to communicate with the + # Account Server, and they will return responses like these: + def make_canned_response(http_method): + acl = { + 'admin': ['foo'], + 'read-write': ['bar'], + 'read-only': ['bas'], + } + headers = {'x-account-sysmeta-core-access-control': format_acl( + version=2, acl_dict=acl)} + canned_resp = Response(headers=headers) + canned_resp.environ = { + 'PATH_INFO': '/acct', + 'REQUEST_METHOD': http_method, + } + resps_to_send.append(canned_resp) + + app = proxy_server.Application( + None, FakeMemcache(), account_ring=FakeRing(), + container_ring=FakeRing(), object_ring=FakeRing()) + app.allow_account_management = True + + ext_header = 'x-account-access-control' + with patch_account_controller_method('GETorHEAD_base'): + # GET/HEAD requests should remap sysmeta headers from acct server + for verb in ('GET', 'HEAD'): + make_canned_response(verb) + req = make_test_request(verb) + resp = app.handle_request(req) + h = parse_acl(version=2, data=resp.headers.get(ext_header)) + self.assertEqual(h['admin'], ['foo']) + self.assertEqual(h['read-write'], ['bar']) + self.assertEqual(h['read-only'], ['bas']) + + # swift_owner = False: GET/HEAD shouldn't return sensitive info + make_canned_response(verb) + req = make_test_request(verb, swift_owner=False) + resp = app.handle_request(req) + h = resp.headers + self.assertEqual(None, h.get(ext_header)) + + # swift_owner unset: GET/HEAD shouldn't return sensitive info + make_canned_response(verb) + req = make_test_request(verb, swift_owner=False) + del req.environ['swift_owner'] + resp = app.handle_request(req) + h = resp.headers + self.assertEqual(None, h.get(ext_header)) + + # Verify that PUT/POST requests remap sysmeta headers from acct server + with patch_account_controller_method('make_requests'): + make_canned_response('PUT') + req = make_test_request('PUT') + resp = app.handle_request(req) + + h = parse_acl(version=2, data=resp.headers.get(ext_header)) + self.assertEqual(h['admin'], ['foo']) + self.assertEqual(h['read-write'], ['bar']) + self.assertEqual(h['read-only'], ['bas']) + + make_canned_response('POST') + req = make_test_request('POST') + resp = app.handle_request(req) + + h = parse_acl(version=2, data=resp.headers.get(ext_header)) + self.assertEqual(h['admin'], ['foo']) + self.assertEqual(h['read-write'], ['bar']) + self.assertEqual(h['read-only'], ['bas']) + class FakeObjectController(object): |