From 286a1308db72c5cfdd6ce16aff3f291ebce257c2 Mon Sep 17 00:00:00 2001 From: Peter Portante Date: Thu, 24 Oct 2013 16:15:25 -0400 Subject: Rebase to OpenStack Swift Havana (1.10.0) Change-Id: I90821230a1a7100c74d97cccc9c445251d0f65e7 Signed-off-by: Peter Portante Reviewed-on: http://review.gluster.org/6157 Reviewed-by: Luis Pabon Tested-by: Luis Pabon --- test/unit/__init__.py | 103 ++- test/unit/common/test_fs_utils.py | 73 ++- test/unit/obj/test_diskfile.py | 431 ++++++------ test/unit/proxy/controllers/test_account.py | 24 +- test/unit/proxy/controllers/test_base.py | 160 ++++- test/unit/proxy/controllers/test_container.py | 24 +- test/unit/proxy/controllers/test_obj.py | 64 +- test/unit/proxy/test_server.py | 904 ++++++++++++++++---------- 8 files changed, 1160 insertions(+), 623 deletions(-) (limited to 'test/unit') diff --git a/test/unit/__init__.py b/test/unit/__init__.py index 04895b4..76e09c1 100644 --- a/test/unit/__init__.py +++ b/test/unit/__init__.py @@ -1,3 +1,18 @@ +# Copyright (c) 2010-2012 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """ Swift tests """ import os @@ -8,6 +23,7 @@ from sys import exc_info from contextlib import contextmanager from collections import defaultdict from tempfile import NamedTemporaryFile +import time from eventlet.green import socket from tempfile import mkdtemp from shutil import rmtree @@ -142,13 +158,24 @@ def tmpfile(content): xattr_data = {} -def _get_inode(fd): - if not isinstance(fd, int): - try: - fd = fd.fileno() - except AttributeError: - return os.stat(fd).st_ino - return os.fstat(fd).st_ino +def _get_inode(fd_or_name): + try: + if isinstance(fd_or_name, int): + fd = fd_or_name + else: + try: + fd = fd_or_name.fileno() + except AttributeError: + fd = None + if fd is None: + ino = os.stat(fd_or_name).st_ino + else: + ino = os.fstat(fd).st_ino + except OSError as err: + ioerr = IOError() + ioerr.errno = err.errno + raise ioerr + return ino def _setxattr(fd, k, v): @@ -199,27 +226,57 @@ class NullLoggingHandler(logging.Handler): pass -class FakeLogger(object): +class UnmockTimeModule(object): + """ + Even if a test mocks time.time - you can restore unmolested behavior in a + another module who imports time directly by monkey patching it's imported + reference to the module with an instance of this class + """ + + _orig_time = time.time + + def __getattribute__(self, name): + if name == 'time': + return UnmockTimeModule._orig_time + return getattr(time, name) + + +# logging.LogRecord.__init__ calls time.time +logging.time = UnmockTimeModule() + + +class FakeLogger(logging.Logger): # a thread safe logger def __init__(self, *args, **kwargs): self._clear() + self.name = 'swift.unit.fake_logger' self.level = logging.NOTSET if 'facility' in kwargs: self.facility = kwargs['facility'] def _clear(self): self.log_dict = defaultdict(list) + self.lines_dict = defaultdict(list) def _store_in(store_name): def stub_fn(self, *args, **kwargs): self.log_dict[store_name].append((args, kwargs)) return stub_fn - error = _store_in('error') - info = _store_in('info') - warning = _store_in('warning') - debug = _store_in('debug') + def _store_and_log_in(store_name): + def stub_fn(self, *args, **kwargs): + self.log_dict[store_name].append((args, kwargs)) + self._log(store_name, args[0], args[1:], **kwargs) + return stub_fn + + def get_lines_for_level(self, level): + return self.lines_dict[level] + + error = _store_and_log_in('error') + info = _store_and_log_in('info') + warning = _store_and_log_in('warning') + debug = _store_and_log_in('debug') def exception(self, *args, **kwargs): self.log_dict['exception'].append((args, kwargs, str(exc_info()[1]))) @@ -267,7 +324,13 @@ class FakeLogger(object): pass def handle(self, record): - pass + try: + line = record.getMessage() + except TypeError: + print 'WARNING: unable to format log message %r %% %r' % ( + record.msg, record.args) + raise + self.lines_dict[record.levelno].append(line) def flush(self): pass @@ -354,11 +417,13 @@ def mock(update): else: deletes.append((module, attr)) setattr(module, attr, value) - yield True - for module, attr, value in returns: - setattr(module, attr, value) - for module, attr in deletes: - delattr(module, attr) + try: + yield True + finally: + for module, attr, value in returns: + setattr(module, attr, value) + for module, attr in deletes: + delattr(module, attr) def fake_http_connect(*code_iter, **kwargs): @@ -466,6 +531,8 @@ def fake_http_connect(*code_iter, **kwargs): body_iter = iter(body_iter) def connect(*args, **ckwargs): + if kwargs.get('slow_connect', False): + sleep(0.1) if 'give_content_type' in kwargs: if len(args) >= 7 and 'Content-Type' in args[6]: kwargs['give_content_type'](args[6]['Content-Type']) diff --git a/test/unit/common/test_fs_utils.py b/test/unit/common/test_fs_utils.py index 02c6ecb..19fc2df 100644 --- a/test/unit/common/test_fs_utils.py +++ b/test/unit/common/test_fs_utils.py @@ -30,8 +30,22 @@ from gluster.swift.common.exceptions import NotDirectoryError, \ def mock_os_fsync(fd): return True -def mock_tpool_execute(func, *args, **kwargs): - func(*args, **kwargs) +def mock_os_fdatasync(fd): + return True + + +class TestFakefile(unittest.TestCase): + """ Tests for common.fs_utils.Fake_file """ + + def test_Fake_file(self): + path = "/tmp/bar" + ff = fs.Fake_file(path) + self.assertEqual(path, ff.path) + self.assertEqual(0, ff.tell()) + self.assertEqual(None, ff.read(50)) + self.assertEqual(-1, ff.fileno()) + self.assertEqual(None, ff.close()) + class TestFsUtils(unittest.TestCase): """ Tests for common.fs_utils """ @@ -688,9 +702,8 @@ class TestFsUtils(unittest.TestCase): fd, tmpfile = mkstemp(dir=tmpdir) try: os.write(fd, 'test') - with patch('eventlet.tpool.execute', mock_tpool_execute): - with patch('os.fsync', mock_os_fsync): - assert fs.do_fsync(fd) is None + with patch('os.fsync', mock_os_fsync): + assert fs.do_fsync(fd) is None except GlusterFileSystemOSError as ose: self.fail('Opening a temporary file failed with %s' %ose.strerror) else: @@ -704,15 +717,47 @@ class TestFsUtils(unittest.TestCase): try: fd, tmpfile = mkstemp(dir=tmpdir) os.write(fd, 'test') - with patch('eventlet.tpool.execute', mock_tpool_execute): - with patch('os.fsync', mock_os_fsync): - assert fs.do_fsync(fd) is None + with patch('os.fsync', mock_os_fsync): + assert fs.do_fsync(fd) is None + os.close(fd) + try: + fs.do_fsync(fd) + except GlusterFileSystemOSError: + pass + else: + self.fail("Expected GlusterFileSystemOSError") + finally: + shutil.rmtree(tmpdir) + + def test_do_fdatasync(self): + tmpdir = mkdtemp() + try: + fd, tmpfile = mkstemp(dir=tmpdir) + try: + os.write(fd, 'test') + with patch('os.fdatasync', mock_os_fdatasync): + assert fs.do_fdatasync(fd) is None + except GlusterFileSystemOSError as ose: + self.fail('Opening a temporary file failed with %s' %ose.strerror) + else: os.close(fd) - try: - fs.do_fsync(fd) - except GlusterFileSystemOSError: - pass - else: - self.fail("Expected GlusterFileSystemOSError") + finally: + shutil.rmtree(tmpdir) + + + def test_do_fdatasync_err(self): + tmpdir = mkdtemp() + try: + fd, tmpfile = mkstemp(dir=tmpdir) + os.write(fd, 'test') + with patch('os.fdatasync', mock_os_fdatasync): + assert fs.do_fdatasync(fd) is None + os.close(fd) + try: + fs.do_fdatasync(fd) + except GlusterFileSystemOSError: + pass + else: + self.fail("Expected GlusterFileSystemOSError") finally: shutil.rmtree(tmpdir) diff --git a/test/unit/obj/test_diskfile.py b/test/unit/obj/test_diskfile.py index 7475105..27ff51e 100644 --- a/test/unit/obj/test_diskfile.py +++ b/test/unit/obj/test_diskfile.py @@ -43,18 +43,35 @@ from test.unit import FakeLogger _metadata = {} -def _mock_read_metadata(filename): +def _mapit(filename_or_fd): + if isinstance(filename_or_fd, int): + statmeth = os.fstat + else: + statmeth = os.lstat + try: + stats = statmeth(filename_or_fd) + except OSError as err: + if err.errno == errno.ENOENT: + raise GlusterFileSystemOSError( + err.errno, '%s, os.fstat(%s)' % (err.strerror, filename_or_fd)) + raise + return stats.st_ino + + +def _mock_read_metadata(filename_or_fd): global _metadata - if filename in _metadata: - md = _metadata[filename] + ino = _mapit(filename_or_fd) + if ino in _metadata: + md = _metadata[ino].copy() else: md = {} return md -def _mock_write_metadata(filename, metadata): +def _mock_write_metadata(filename_or_fd, metadata): global _metadata - _metadata[filename] = metadata + ino = _mapit(filename_or_fd) + _metadata[ino] = metadata.copy() def _mock_clear_metadata(): @@ -127,8 +144,7 @@ class TestDiskFile(unittest.TestCase): assert gdf.logger == self.lg assert gdf.uid == DEFAULT_UID assert gdf.gid == DEFAULT_GID - assert gdf.metadata == {} - assert gdf.meta_file is None + assert gdf._metadata == None assert gdf.data_file is None assert gdf.fp is None assert gdf.iter_etag is None @@ -146,7 +162,7 @@ class TestDiskFile(unittest.TestCase): assert gdf.datadir == os.path.join(self.td, "vol0", "bar", "b", "a") assert gdf.device_path == os.path.join(self.td, "vol0") - def test_constructor_no_metadata(self): + def test_open_no_metadata(self): the_path = os.path.join(self.td, "vol0", "bar") the_file = os.path.join(the_path, "z") os.makedirs(the_path) @@ -164,12 +180,13 @@ class TestDiskFile(unittest.TestCase): 'Content-Type': 'application/octet-stream'} gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z") assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - assert gdf.fp is None - assert gdf.metadata == exp_md + with gdf.open(): + assert gdf.data_file == the_file + assert not gdf._is_dir + assert gdf.fp is not None + assert gdf._metadata == exp_md - def test_constructor_existing_metadata(self): + def test_open_existing_metadata(self): the_path = os.path.join(self.td, "vol0", "bar") the_file = os.path.join(the_path, "z") os.makedirs(the_path) @@ -182,37 +199,39 @@ class TestDiskFile(unittest.TestCase): 'ETag': 'etag', 'X-Timestamp': 'ts', 'Content-Type': 'application/loctet-stream'} - _metadata[the_file] = ini_md + _metadata[_mapit(the_file)] = ini_md exp_md = ini_md.copy() del exp_md['X-Type'] del exp_md['X-Object-Type'] gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z") assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - assert gdf.fp is None - assert gdf.metadata == exp_md + with gdf.open(): + assert gdf.data_file == the_file + assert not gdf._is_dir + assert gdf.fp is not None + assert gdf._metadata == exp_md - def test_constructor_invalid_existing_metadata(self): + def test_open_invalid_existing_metadata(self): the_path = os.path.join(self.td, "vol0", "bar") the_file = os.path.join(the_path, "z") + os.makedirs(the_path) + with open(the_file, "wb") as fd: + fd.write("1234") inv_md = { 'Content-Length': 5, 'ETag': 'etag', 'X-Timestamp': 'ts', 'Content-Type': 'application/loctet-stream'} - _metadata[the_file] = inv_md - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") + _metadata[_mapit(the_file)] = inv_md gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z") assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - assert gdf.fp is None - assert gdf.metadata != inv_md + with gdf.open(): + assert gdf.data_file == the_file + assert not gdf._is_dir + assert gdf.fp is not None + assert gdf._metadata != inv_md - def test_constructor_isdir(self): + def test_open_isdir(self): the_path = os.path.join(self.td, "vol0", "bar") the_dir = os.path.join(the_path, "d") os.makedirs(the_dir) @@ -223,29 +242,16 @@ class TestDiskFile(unittest.TestCase): 'ETag': 'etag', 'X-Timestamp': 'ts', 'Content-Type': 'application/loctet-stream'} - _metadata[the_dir] = ini_md + _metadata[_mapit(the_dir)] = ini_md exp_md = ini_md.copy() del exp_md['X-Type'] del exp_md['X-Object-Type'] - gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "d", - keep_data_fp=True) + gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "d") assert gdf._obj == "d" - assert gdf.data_file == the_dir - assert gdf._is_dir - assert gdf.metadata == exp_md - - def test_constructor_keep_data_fp(self): - the_path = os.path.join(self.td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z", - keep_data_fp=True) - assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - assert gdf.fp is not None + with gdf.open(): + assert gdf.data_file == the_dir + assert gdf._is_dir + assert gdf._metadata == exp_md def test_constructor_chunk_size(self): gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z", @@ -258,8 +264,7 @@ class TestDiskFile(unittest.TestCase): assert gdf.iter_hook == 'hook' def test_close_no_open_fp(self): - gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z", - keep_data_fp=True) + gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z") gdf._is_dir = False self.called = False @@ -276,28 +281,32 @@ class TestDiskFile(unittest.TestCase): the_dir = "dir" self.called = False os.makedirs(os.path.join(the_cont, the_dir)) - gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir", - keep_data_fp=True) - - ret = isinstance(gdf.fp, Fake_file) - self.assertTrue(ret) + gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir") + with gdf.open(): + ret = isinstance(gdf.fp, Fake_file) + self.assertTrue(ret) - # Get a File descriptor - fd = gdf.fp + # Get a "Fake_file" pointer + ffp = gdf.fp - # This expected to call Fake_file interfaces - ret = fd.tell() - self.assertEqual(ret , 0) + # This expected to call Fake_file interfaces + ret = ffp.tell() + self.assertEqual(ret, 0) - ret = fd.read(1) - self.assertEqual(ret , 0) + ret = ffp.read(1) + self.assertEqual(ret, None) - ret = fd.fileno() - self.assertEqual(ret, -1) + ret = ffp.fileno() + self.assertEqual(ret, -1) - ret = fd.close() - self.assertFalse(self.called) + def our_do_close(ffp): + self.called = True + with mock.patch("gluster.swift.obj.diskfile.do_close", + our_do_close): + ret = ffp.close() + self.assertEqual(ret, None) + self.assertFalse(self.called) def test_close_file_object(self): the_cont = os.path.join(self.td, "vol0", "bar") @@ -306,22 +315,23 @@ class TestDiskFile(unittest.TestCase): os.makedirs(the_cont) with open(the_file, "wb") as fd: fd.write("1234") - gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z", - keep_data_fp=True) + gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z") def our_do_close(fp): self.called = True with mock.patch("gluster.swift.obj.diskfile.do_close", our_do_close): - gdf.close() + with gdf.open(): + assert not self.called assert self.called def test_is_deleted(self): gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z") - assert gdf.is_deleted() - gdf.data_file = os.path.join(self.td, "bar") - assert not gdf.is_deleted() + with gdf.open(): + assert gdf.is_deleted() + gdf.data_file = os.path.join(self.td, "bar") + assert not gdf.is_deleted() def test_create_dir_object_no_md(self): the_cont = os.path.join(self.td, "vol0", "bar") @@ -334,7 +344,7 @@ class TestDiskFile(unittest.TestCase): gdf._create_dir_object(the_dir) full_dir_path = os.path.join(the_cont, the_dir) assert os.path.isdir(full_dir_path) - assert full_dir_path not in _metadata + assert _mapit(full_dir_path) not in _metadata def test_create_dir_object_with_md(self): the_cont = os.path.join(self.td, "vol0", "bar") @@ -349,7 +359,7 @@ class TestDiskFile(unittest.TestCase): gdf._create_dir_object(the_dir, dir_md) full_dir_path = os.path.join(the_cont, the_dir) assert os.path.isdir(full_dir_path) - assert full_dir_path in _metadata + assert _mapit(full_dir_path) in _metadata def test_create_dir_object_exists(self): the_path = os.path.join(self.td, "vol0", "bar") @@ -371,7 +381,7 @@ class TestDiskFile(unittest.TestCase): DiskFileError, gdf._create_dir_object, the_dir) gluster.swift.obj.diskfile.do_chown = dc self.assertFalse(os.path.isdir(the_dir)) - self.assertFalse(the_dir in _metadata) + self.assertFalse(_mapit(the_dir) in _metadata) def test_create_dir_object_do_stat_failure(self): the_path = os.path.join(self.td, "vol0", "bar") @@ -393,25 +403,26 @@ class TestDiskFile(unittest.TestCase): DiskFileError, gdf._create_dir_object, the_dir) gluster.swift.obj.diskfile.do_chown = dc self.assertFalse(os.path.isdir(the_dir)) - self.assertFalse(the_dir in _metadata) + self.assertFalse(_mapit(the_dir) in _metadata) def test_put_metadata(self): - the_path = os.path.join(self.td, "vol0", "bar") - the_dir = os.path.join(the_path, "z") + the_dir = os.path.join(self.td, "vol0", "bar", "z") os.makedirs(the_dir) gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z") md = {'Content-Type': 'application/octet-stream', 'a': 'b'} gdf.put_metadata(md.copy()) - assert gdf.metadata == md, "gdf.metadata = %r, md = %r" % ( - gdf.metadata, md) - assert _metadata[the_dir] == md + assert gdf._metadata is None + fmd = _metadata[_mapit(the_dir)] + md.update({'X-Object-Type': 'file', 'X-Type': 'Object'}) + assert fmd == md, "on-disk md = %r, md = %r" % (fmd, md) def test_put_w_tombstone(self): gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z") - assert gdf.metadata == {} + assert gdf._metadata == None gdf.put_metadata({'x': '1'}, tombstone=True) - assert gdf.metadata == {} + assert gdf._metadata is None + assert _metadata == {} def test_put_w_meta_file(self): the_path = os.path.join(self.td, "vol0", "bar") @@ -420,11 +431,13 @@ class TestDiskFile(unittest.TestCase): with open(the_file, "wb") as fd: fd.write("1234") gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z") - newmd = gdf.metadata.copy() - newmd['X-Object-Meta-test'] = '1234' + with gdf.open(): + newmd = gdf.get_metadata().copy() + newmd['X-Object-Meta-test'] = '1234' gdf.put_metadata(newmd) - assert gdf.metadata == newmd - assert _metadata[the_file] == newmd + assert gdf._metadata is None + fmd = _metadata[_mapit(the_file)] + assert fmd == newmd, "on-disk md = %r, newmd = %r" % (fmd, newmd) def test_put_w_meta_file_no_content_type(self): the_path = os.path.join(self.td, "vol0", "bar") @@ -433,67 +446,72 @@ class TestDiskFile(unittest.TestCase): with open(the_file, "wb") as fd: fd.write("1234") gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z") - newmd = gdf.metadata.copy() - newmd['Content-Type'] = '' - newmd['X-Object-Meta-test'] = '1234' + with gdf.open(): + newmd = gdf.get_metadata().copy() + newmd['Content-Type'] = '' + newmd['X-Object-Meta-test'] = '1234' gdf.put_metadata(newmd) - assert gdf.metadata == newmd - assert _metadata[the_file] == newmd + assert gdf._metadata is None + fmd = _metadata[_mapit(the_file)] + assert fmd == newmd, "on-disk md = %r, newmd = %r" % (fmd, newmd) def test_put_w_meta_dir(self): the_path = os.path.join(self.td, "vol0", "bar") the_dir = os.path.join(the_path, "dir") os.makedirs(the_dir) gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir") - newmd = gdf.metadata.copy() - newmd['X-Object-Meta-test'] = '1234' + with gdf.open(): + newmd = gdf.get_metadata().copy() + newmd['X-Object-Meta-test'] = '1234' gdf.put_metadata(newmd) - assert gdf.metadata == newmd - assert _metadata[the_dir] == newmd + assert gdf._metadata is None + fmd = _metadata[_mapit(the_dir)] + assert fmd == newmd, "on-disk md = %r, newmd = %r" % (fmd, newmd) def test_put_w_marker_dir(self): the_path = os.path.join(self.td, "vol0", "bar") the_dir = os.path.join(the_path, "dir") os.makedirs(the_dir) gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir") - newmd = gdf.metadata.copy() - newmd['X-Object-Meta-test'] = '1234' + with gdf.open(): + newmd = gdf.get_metadata().copy() + newmd['X-Object-Meta-test'] = '1234' gdf.put_metadata(newmd) - assert gdf.metadata == newmd - assert _metadata[the_dir] == newmd + assert gdf._metadata is None + fmd = _metadata[_mapit(the_dir)] + assert fmd == newmd, "on-disk md = %r, newmd = %r" % (fmd, newmd) def test_put_w_marker_dir_create(self): the_cont = os.path.join(self.td, "vol0", "bar") the_dir = os.path.join(the_cont, "dir") os.makedirs(the_cont) gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir") - assert gdf.metadata == {} + assert gdf._metadata == None newmd = { 'ETag': 'etag', 'X-Timestamp': 'ts', 'Content-Type': 'application/directory'} - with gdf.writer() as dw: - dw.put(newmd, extension='.dir') - assert gdf.data_file == the_dir - for key, val in newmd.items(): - assert gdf.metadata[key] == val - assert _metadata[the_dir][key] == val - assert gdf.metadata[X_OBJECT_TYPE] == DIR_OBJECT - assert _metadata[the_dir][X_OBJECT_TYPE] == DIR_OBJECT + with gdf.create() as dw: + dw.put(newmd.copy(), extension='.dir') + with gdf.open(): + assert gdf.data_file == the_dir + for key, val in newmd.items(): + assert gdf._metadata[key] == val + assert _metadata[_mapit(the_dir)][key] == val + assert X_OBJECT_TYPE not in gdf._metadata, "md = %r" % gdf._metadata + assert _metadata[_mapit(the_dir)][X_OBJECT_TYPE] == DIR_OBJECT def test_put_is_dir(self): the_path = os.path.join(self.td, "vol0", "bar") the_dir = os.path.join(the_path, "dir") os.makedirs(the_dir) gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir") - origmd = gdf.metadata.copy() - origfmd = _metadata[the_dir] - newmd = gdf.metadata.copy() # FIXME: This is a hack to get to the code-path; it is not clear # how this can happen normally. - newmd['Content-Type'] = '' - newmd['X-Object-Meta-test'] = '1234' - with gdf.writer() as dw: + newmd = { + 'Content-Type': '', + 'X-Object-Meta-test': '1234'} + with gdf.create() as dw: try: dw.put(newmd, extension='.data') except DiskFileError: @@ -501,8 +519,6 @@ class TestDiskFile(unittest.TestCase): else: self.fail("Expected to encounter" " 'already-exists-as-dir' exception") - assert gdf.metadata == origmd - assert _metadata[the_dir] == origfmd def test_put(self): the_cont = os.path.join(self.td, "vol0", "bar") @@ -525,7 +541,7 @@ class TestDiskFile(unittest.TestCase): 'Content-Length': '5', } - with gdf.writer() as dw: + with gdf.create() as dw: assert dw.tmppath is not None tmppath = dw.tmppath dw.write(body) @@ -561,7 +577,7 @@ class TestDiskFile(unittest.TestCase): with mock.patch("os.open", mock_open): try: - with gdf.writer() as dw: + with gdf.create() as dw: assert dw.tmppath is not None dw.write(body) dw.put(metadata) @@ -601,7 +617,7 @@ class TestDiskFile(unittest.TestCase): with mock.patch("gluster.swift.obj.diskfile.sleep", mock_sleep): with mock.patch("os.rename", mock_rename): try: - with gdf.writer() as dw: + with gdf.create() as dw: assert dw.tmppath is not None dw.write(body) dw.put(metadata) @@ -631,7 +647,7 @@ class TestDiskFile(unittest.TestCase): 'Content-Length': '5', } - with gdf.writer() as dw: + with gdf.create() as dw: assert dw.tmppath is not None tmppath = dw.tmppath dw.write(body) @@ -642,32 +658,32 @@ class TestDiskFile(unittest.TestCase): assert os.path.exists(gdf.data_file) assert not os.path.exists(tmppath) - def test_unlinkold_no_metadata(self): + def test_delete_no_metadata(self): gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z") - assert gdf.metadata == {} + assert gdf._metadata == None _saved_rmobjdir = gluster.swift.obj.diskfile.rmobjdir gluster.swift.obj.diskfile.rmobjdir = _mock_rmobjdir try: - gdf.unlinkold(None) + gdf.delete(1.0) except MockException as exp: self.fail(str(exp)) finally: gluster.swift.obj.diskfile.rmobjdir = _saved_rmobjdir - def test_unlinkold_same_timestamp(self): + def test_delete_same_timestamp(self): gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z") - assert gdf.metadata == {} - gdf.metadata['X-Timestamp'] = 1 + assert gdf._metadata == None + gdf._metadata = {'X-Timestamp': 1} _saved_rmobjdir = gluster.swift.obj.diskfile.rmobjdir gluster.swift.obj.diskfile.rmobjdir = _mock_rmobjdir try: - gdf.unlinkold(1) + gdf.delete(1) except MockException as exp: self.fail(str(exp)) finally: gluster.swift.obj.diskfile.rmobjdir = _saved_rmobjdir - def test_unlinkold_file(self): + def test_delete_file(self): the_path = os.path.join(self.td, "vol0", "bar") the_file = os.path.join(the_path, "z") os.makedirs(the_path) @@ -675,15 +691,14 @@ class TestDiskFile(unittest.TestCase): fd.write("1234") gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z") assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - - later = float(gdf.metadata['X-Timestamp']) + 1 - gdf.unlinkold(normalize_timestamp(later)) + with gdf.open(): + later = float(gdf.get_metadata()['X-Timestamp']) + 1 + assert gdf.data_file == the_file + gdf.delete(normalize_timestamp(later)) assert os.path.isdir(gdf.datadir) assert not os.path.exists(os.path.join(gdf.datadir, gdf._obj)) - def test_unlinkold_file_not_found(self): + def test_delete_file_not_found(self): the_path = os.path.join(self.td, "vol0", "bar") the_file = os.path.join(the_path, "z") os.makedirs(the_path) @@ -691,18 +706,19 @@ class TestDiskFile(unittest.TestCase): fd.write("1234") gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z") assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir + with gdf.open(): + later = float(gdf._metadata['X-Timestamp']) + 1 + assert gdf.data_file == the_file + assert not gdf._is_dir # Handle the case the file is not in the directory listing. os.unlink(the_file) - later = float(gdf.metadata['X-Timestamp']) + 1 - gdf.unlinkold(normalize_timestamp(later)) + gdf.delete(normalize_timestamp(later)) assert os.path.isdir(gdf.datadir) assert not os.path.exists(os.path.join(gdf.datadir, gdf._obj)) - def test_unlinkold_file_unlink_error(self): + def test_delete_file_unlink_error(self): the_path = os.path.join(self.td, "vol0", "bar") the_file = os.path.join(the_path, "z") os.makedirs(the_path) @@ -710,10 +726,10 @@ class TestDiskFile(unittest.TestCase): fd.write("1234") gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z") assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - - later = float(gdf.metadata['X-Timestamp']) + 1 + with gdf.open(): + assert gdf.data_file == the_file + assert not gdf._is_dir + later = float(gdf._metadata['X-Timestamp']) + 1 def _mock_os_unlink_eacces_err(f): raise OSError(errno.EACCES, os.strerror(errno.EACCES)) @@ -725,7 +741,7 @@ class TestDiskFile(unittest.TestCase): # Handle the case os_unlink() raises an OSError with patch("os.unlink", _mock_os_unlink_eacces_err): try: - gdf.unlinkold(normalize_timestamp(later)) + gdf.delete(normalize_timestamp(later)) except OSError as e: assert e.errno == errno.EACCES else: @@ -736,17 +752,17 @@ class TestDiskFile(unittest.TestCase): assert os.path.isdir(gdf.datadir) assert os.path.exists(os.path.join(gdf.datadir, gdf._obj)) - def test_unlinkold_is_dir(self): + def test_delete_is_dir(self): the_path = os.path.join(self.td, "vol0", "bar") the_dir = os.path.join(the_path, "d") os.makedirs(the_dir) - gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "d", - keep_data_fp=True) - assert gdf.data_file == the_dir - assert gdf._is_dir - - later = float(gdf.metadata['X-Timestamp']) + 1 - gdf.unlinkold(normalize_timestamp(later)) + gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "d") + assert gdf._obj == "d" + with gdf.open(): + assert gdf.data_file == the_dir + assert gdf._is_dir + later = float(gdf._metadata['X-Timestamp']) + 1 + gdf.delete(normalize_timestamp(later)) assert os.path.isdir(gdf.datadir) assert not os.path.exists(os.path.join(gdf.datadir, gdf._obj)) @@ -758,9 +774,10 @@ class TestDiskFile(unittest.TestCase): fd.write("1234") gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z") assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - assert 4 == gdf.get_data_file_size() + with gdf.open(): + assert gdf.data_file == the_file + assert not gdf._is_dir + assert 4 == gdf.get_data_file_size() def test_get_data_file_size_md_restored(self): the_path = os.path.join(self.td, "vol0", "bar") @@ -770,12 +787,13 @@ class TestDiskFile(unittest.TestCase): fd.write("1234") gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z") assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - assert 4 == gdf.metadata['Content-Length'] - gdf.metadata['Content-Length'] = 3 - assert 4 == gdf.get_data_file_size() - assert 4 == gdf.metadata['Content-Length'] + with gdf.open(): + assert gdf.data_file == the_file + assert not gdf._is_dir + assert 4 == gdf._metadata['Content-Length'] + gdf._metadata['Content-Length'] = 3 + assert 4 == gdf.get_data_file_size() + assert 4 == gdf._metadata['Content-Length'] def test_get_data_file_size_dne(self): gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", @@ -795,15 +813,16 @@ class TestDiskFile(unittest.TestCase): fd.write("1234") gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z") assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - gdf.data_file = gdf.data_file + ".dne" - try: - gdf.get_data_file_size() - except DiskFileNotExist: - pass - else: - self.fail("Expected DiskFileNotExist exception") + with gdf.open(): + assert gdf.data_file == the_file + assert not gdf._is_dir + gdf.data_file = gdf.data_file + ".dne" + try: + gdf.get_data_file_size() + except DiskFileNotExist: + pass + else: + self.fail("Expected DiskFileNotExist exception") def test_get_data_file_size_os_err(self): the_path = os.path.join(self.td, "vol0", "bar") @@ -813,55 +832,57 @@ class TestDiskFile(unittest.TestCase): fd.write("1234") gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z") assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - stats = os.stat(the_path) - try: - os.chmod(the_path, 0) + with gdf.open(): + assert gdf.data_file == the_file + assert not gdf._is_dir + stats = os.stat(the_path) + try: + os.chmod(the_path, 0) - def _mock_getsize_eaccess_err(f): - raise OSError(errno.EACCES, os.strerror(errno.EACCES)) + def _mock_getsize_eaccess_err(f): + raise OSError(errno.EACCES, os.strerror(errno.EACCES)) - with patch("os.path.getsize", _mock_getsize_eaccess_err): - try: - gdf.get_data_file_size() - except OSError as err: - assert err.errno == errno.EACCES - else: - self.fail("Expected OSError exception") - finally: - os.chmod(the_path, stats.st_mode) + with patch("os.path.getsize", _mock_getsize_eaccess_err): + try: + gdf.get_data_file_size() + except OSError as err: + assert err.errno == errno.EACCES + else: + self.fail("Expected OSError exception") + finally: + os.chmod(the_path, stats.st_mode) def test_get_data_file_size_dir(self): the_path = os.path.join(self.td, "vol0", "bar") the_dir = os.path.join(the_path, "d") os.makedirs(the_dir) - gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "d", - keep_data_fp=True) + gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "d") assert gdf._obj == "d" - assert gdf.data_file == the_dir - assert gdf._is_dir - assert 0 == gdf.get_data_file_size() + with gdf.open(): + assert gdf.data_file == the_dir + assert gdf._is_dir + assert 0 == gdf.get_data_file_size() def test_filter_metadata(self): gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z") - assert gdf.metadata == {} + assert gdf._metadata == None gdf._filter_metadata() - assert gdf.metadata == {} + assert gdf._metadata == None - gdf.metadata[X_TYPE] = 'a' - gdf.metadata[X_OBJECT_TYPE] = 'b' - gdf.metadata['foobar'] = 'c' + gdf._metadata = {} + gdf._metadata[X_TYPE] = 'a' + gdf._metadata[X_OBJECT_TYPE] = 'b' + gdf._metadata['foobar'] = 'c' gdf._filter_metadata() - assert X_TYPE not in gdf.metadata - assert X_OBJECT_TYPE not in gdf.metadata - assert 'foobar' in gdf.metadata + assert X_TYPE not in gdf._metadata + assert X_OBJECT_TYPE not in gdf._metadata + assert 'foobar' in gdf._metadata - def test_writer(self): + def test_create(self): gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir/z") saved_tmppath = '' saved_fd = None - with gdf.writer() as dw: + with gdf.create() as dw: assert gdf.datadir == os.path.join(self.td, "vol0", "bar", "dir") assert os.path.isdir(gdf.datadir) saved_tmppath = dw.tmppath @@ -881,10 +902,10 @@ class TestDiskFile(unittest.TestCase): self.fail("Exception expected") assert not os.path.exists(saved_tmppath) - def test_writer_err_on_close(self): + def test_create_err_on_close(self): gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir/z") saved_tmppath = '' - with gdf.writer() as dw: + with gdf.create() as dw: assert gdf.datadir == os.path.join(self.td, "vol0", "bar", "dir") assert os.path.isdir(gdf.datadir) saved_tmppath = dw.tmppath @@ -896,10 +917,10 @@ class TestDiskFile(unittest.TestCase): os.close(dw.fd) assert not os.path.exists(saved_tmppath) - def test_writer_err_on_unlink(self): + def test_create_err_on_unlink(self): gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir/z") saved_tmppath = '' - with gdf.writer() as dw: + with gdf.create() as dw: assert gdf.datadir == os.path.join(self.td, "vol0", "bar", "dir") assert os.path.isdir(gdf.datadir) saved_tmppath = dw.tmppath diff --git a/test/unit/proxy/controllers/test_account.py b/test/unit/proxy/controllers/test_account.py index 4d67d65..394ada7 100644 --- a/test/unit/proxy/controllers/test_account.py +++ b/test/unit/proxy/controllers/test_account.py @@ -1,4 +1,4 @@ -# Copyright (c) 2010-2012 OpenStack, LLC. +# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -40,6 +40,28 @@ class TestAccountController(unittest.TestCase): self.assertEqual(headers_to_account_info(resp.headers), resp.environ['swift.account/AUTH_bob']) + def test_swift_owner(self): + owner_headers = { + 'x-account-meta-temp-url-key': 'value', + 'x-account-meta-temp-url-key-2': 'value'} + controller = proxy_server.AccountController(self.app, 'a') + + req = Request.blank('/a') + with mock.patch('swift.proxy.controllers.base.http_connect', + fake_http_connect(200, 200, headers=owner_headers)): + resp = controller.HEAD(req) + self.assertEquals(2, resp.status_int // 100) + for key in owner_headers: + self.assertTrue(key not in resp.headers) + + req = Request.blank('/a', environ={'swift_owner': True}) + with mock.patch('swift.proxy.controllers.base.http_connect', + fake_http_connect(200, 200, headers=owner_headers)): + resp = controller.HEAD(req) + self.assertEquals(2, resp.status_int // 100) + for key in owner_headers: + self.assertTrue(key in resp.headers) + if __name__ == '__main__': unittest.main() diff --git a/test/unit/proxy/controllers/test_base.py b/test/unit/proxy/controllers/test_base.py index 02692eb..8214b98 100644 --- a/test/unit/proxy/controllers/test_base.py +++ b/test/unit/proxy/controllers/test_base.py @@ -1,4 +1,4 @@ -# Copyright (c) 2010-2012 OpenStack, LLC. +# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,9 +16,9 @@ import unittest from mock import patch from swift.proxy.controllers.base import headers_to_container_info, \ - headers_to_account_info, get_container_info, get_container_memcache_key, \ - get_account_info, get_account_memcache_key, _get_cache_key, get_info, \ - Controller + headers_to_account_info, headers_to_object_info, get_container_info, \ + get_container_memcache_key, get_account_info, get_account_memcache_key, \ + get_object_env_key, _get_cache_key, get_info, get_object_info, Controller from swift.common.swob import Request from swift.common.utils import split_path from test.unit import fake_http_connect, FakeRing, FakeMemcache @@ -29,12 +29,18 @@ FakeResponse_status_int = 201 class FakeResponse(object): - def __init__(self, headers, env, account, container): + def __init__(self, headers, env, account, container, obj): self.headers = headers self.status_int = FakeResponse_status_int self.environ = env - cache_key, env_key = _get_cache_key(account, container) - if container: + if obj: + env_key = get_object_env_key(account, container, obj) + else: + cache_key, env_key = _get_cache_key(account, container) + + if account and container and obj: + info = headers_to_object_info(headers, FakeResponse_status_int) + elif account and container: info = headers_to_container_info(headers, FakeResponse_status_int) else: info = headers_to_account_info(headers, FakeResponse_status_int) @@ -42,18 +48,27 @@ class FakeResponse(object): class FakeRequest(object): - def __init__(self, env, path): + def __init__(self, env, path, swift_source=None): self.environ = env (version, account, container, obj) = split_path(path, 2, 4, True) self.account = account self.container = container - stype = container and 'container' or 'account' - self.headers = {'x-%s-object-count' % (stype): 1000, - 'x-%s-bytes-used' % (stype): 6666} + self.obj = obj + if obj: + stype = 'object' + self.headers = {'content-length': 5555, + 'content-type': 'text/plain'} + else: + stype = container and 'container' or 'account' + self.headers = {'x-%s-object-count' % (stype): 1000, + 'x-%s-bytes-used' % (stype): 6666} + if swift_source: + meta = 'x-%s-meta-fakerequest-swift-source' % stype + self.headers[meta] = swift_source def get_response(self, app): return FakeResponse(self.headers, self.environ, self.account, - self.container) + self.container, self.obj) class FakeCache(object): @@ -73,6 +88,21 @@ class TestFuncs(unittest.TestCase): def test_GETorHEAD_base(self): base = Controller(self.app) + req = Request.blank('/a/c/o/with/slashes') + with patch('swift.proxy.controllers.base.' + 'http_connect', fake_http_connect(200)): + resp = base.GETorHEAD_base(req, 'object', FakeRing(), 'part', + '/a/c/o/with/slashes') + self.assertTrue('swift.object/a/c/o/with/slashes' in resp.environ) + self.assertEqual( + resp.environ['swift.object/a/c/o/with/slashes']['status'], 200) + req = Request.blank('/a/c/o') + with patch('swift.proxy.controllers.base.' + 'http_connect', fake_http_connect(200)): + resp = base.GETorHEAD_base(req, 'object', FakeRing(), 'part', + '/a/c/o') + self.assertTrue('swift.object/a/c/o' in resp.environ) + self.assertEqual(resp.environ['swift.object/a/c/o']['status'], 200) req = Request.blank('/a/c') with patch('swift.proxy.controllers.base.' 'http_connect', fake_http_connect(200)): @@ -101,7 +131,7 @@ class TestFuncs(unittest.TestCase): self.assertEquals(info_a['bytes'], 6666) self.assertEquals(info_a['total_object_count'], 1000) # Make sure the env cache is set - self.assertEquals(env, {'swift.account/a': info_a}) + self.assertEquals(env.get('swift.account/a'), info_a) # Do an env cached call to account info_a = get_info(None, env, 'a') @@ -110,7 +140,7 @@ class TestFuncs(unittest.TestCase): self.assertEquals(info_a['bytes'], 6666) self.assertEquals(info_a['total_object_count'], 1000) # Make sure the env cache is set - self.assertEquals(env, {'swift.account/a': info_a}) + self.assertEquals(env.get('swift.account/a'), info_a) # This time do env cached call to account and non cached to container with patch('swift.proxy.controllers.base.' @@ -121,11 +151,12 @@ class TestFuncs(unittest.TestCase): self.assertEquals(info_c['bytes'], 6666) self.assertEquals(info_c['object_count'], 1000) # Make sure the env cache is set - self.assertEquals(env['swift.account/a'], info_a) - self.assertEquals(env['swift.container/a/c'], info_c) + self.assertEquals(env.get('swift.account/a'), info_a) + self.assertEquals(env.get('swift.container/a/c'), info_c) - # This time do a non cached call to account than non cached to container - env = {} # abandon previous call to env + # This time do a non cached call to account than non cached to + # container + env = {} # abandon previous call to env with patch('swift.proxy.controllers.base.' '_prepare_pre_auth_info_request', FakeRequest): info_c = get_info(None, env, 'a', 'c') @@ -134,10 +165,11 @@ class TestFuncs(unittest.TestCase): self.assertEquals(info_c['bytes'], 6666) self.assertEquals(info_c['object_count'], 1000) # Make sure the env cache is set - self.assertEquals(env['swift.account/a'], info_a) - self.assertEquals(env['swift.container/a/c'], info_c) + self.assertEquals(env.get('swift.account/a'), info_a) + self.assertEquals(env.get('swift.container/a/c'), info_c) - # This time do an env cached call to container while account is not cached + # This time do an env cached call to container while account is not + # cached del(env['swift.account/a']) info_c = get_info(None, env, 'a', 'c') # Check that you got proper info @@ -145,7 +177,7 @@ class TestFuncs(unittest.TestCase): self.assertEquals(info_c['bytes'], 6666) self.assertEquals(info_c['object_count'], 1000) # Make sure the env cache is set and account still not cached - self.assertEquals(env, {'swift.container/a/c': info_c}) + self.assertEquals(env.get('swift.container/a/c'), info_c) # Do a non cached call to account not found with ret_not_found env = {} @@ -161,7 +193,7 @@ class TestFuncs(unittest.TestCase): self.assertEquals(info_a['bytes'], 6666) self.assertEquals(info_a['total_object_count'], 1000) # Make sure the env cache is set - self.assertEquals(env, {'swift.account/a': info_a}) + self.assertEquals(env.get('swift.account/a'), info_a) # Do a cached call to account not found with ret_not_found info_a = get_info(None, env, 'a', ret_not_found=True) @@ -170,7 +202,7 @@ class TestFuncs(unittest.TestCase): self.assertEquals(info_a['bytes'], 6666) self.assertEquals(info_a['total_object_count'], 1000) # Make sure the env cache is set - self.assertEquals(env, {'swift.account/a': info_a}) + self.assertEquals(env.get('swift.account/a'), info_a) # Do a non cached call to account not found without ret_not_found env = {} @@ -191,6 +223,21 @@ class TestFuncs(unittest.TestCase): self.assertEquals(info_a, None) self.assertEquals(env['swift.account/a']['status'], 404) + def test_get_container_info_swift_source(self): + req = Request.blank("/v1/a/c", environ={'swift.cache': FakeCache({})}) + with patch('swift.proxy.controllers.base.' + '_prepare_pre_auth_info_request', FakeRequest): + resp = get_container_info(req.environ, 'app', swift_source='MC') + self.assertEquals(resp['meta']['fakerequest-swift-source'], 'MC') + + def test_get_object_info_swift_source(self): + req = Request.blank("/v1/a/c/o", + environ={'swift.cache': FakeCache({})}) + with patch('swift.proxy.controllers.base.' + '_prepare_pre_auth_info_request', FakeRequest): + resp = get_object_info(req.environ, 'app', swift_source='LU') + self.assertEquals(resp['meta']['fakerequest-swift-source'], 'LU') + def test_get_container_info_no_cache(self): req = Request.blank("/v1/AUTH_account/cont", environ={'swift.cache': FakeCache({})}) @@ -217,11 +264,18 @@ class TestFuncs(unittest.TestCase): cache_key = get_container_memcache_key("account", "cont") env_key = 'swift.%s' % cache_key req = Request.blank("/v1/account/cont", - environ={ env_key: {'bytes': 3867}, - 'swift.cache': FakeCache({})}) + environ={env_key: {'bytes': 3867}, + 'swift.cache': FakeCache({})}) resp = get_container_info(req.environ, 'xxx') self.assertEquals(resp['bytes'], 3867) + def test_get_account_info_swift_source(self): + req = Request.blank("/v1/a", environ={'swift.cache': FakeCache({})}) + with patch('swift.proxy.controllers.base.' + '_prepare_pre_auth_info_request', FakeRequest): + resp = get_account_info(req.environ, 'a', swift_source='MC') + self.assertEquals(resp['meta']['fakerequest-swift-source'], 'MC') + def test_get_account_info_no_cache(self): req = Request.blank("/v1/AUTH_account", environ={'swift.cache': FakeCache({})}) @@ -266,11 +320,33 @@ class TestFuncs(unittest.TestCase): cache_key = get_account_memcache_key("account") env_key = 'swift.%s' % cache_key req = Request.blank("/v1/account", - environ={ env_key: {'bytes': 3867}, - 'swift.cache': FakeCache({})}) + environ={env_key: {'bytes': 3867}, + 'swift.cache': FakeCache({})}) resp = get_account_info(req.environ, 'xxx') self.assertEquals(resp['bytes'], 3867) + def test_get_object_info_env(self): + cached = {'status': 200, + 'length': 3333, + 'type': 'application/json', + 'meta': {}} + env_key = get_object_env_key("account", "cont", "obj") + req = Request.blank("/v1/account/cont/obj", + environ={env_key: cached, + 'swift.cache': FakeCache({})}) + resp = get_object_info(req.environ, 'xxx') + self.assertEquals(resp['length'], 3333) + self.assertEquals(resp['type'], 'application/json') + + def test_get_object_info_no_env(self): + req = Request.blank("/v1/account/cont/obj", + environ={'swift.cache': FakeCache({})}) + with patch('swift.proxy.controllers.base.' + '_prepare_pre_auth_info_request', FakeRequest): + resp = get_object_info(req.environ, 'xxx') + self.assertEquals(resp['length'], 5555) + self.assertEquals(resp['type'], 'text/plain') + def test_headers_to_container_info_missing(self): resp = headers_to_container_info({}, 404) self.assertEquals(resp['status'], 404) @@ -329,3 +405,31 @@ class TestFuncs(unittest.TestCase): self.assertEquals( resp, headers_to_account_info(headers.items(), 200)) + + def test_headers_to_object_info_missing(self): + resp = headers_to_object_info({}, 404) + self.assertEquals(resp['status'], 404) + self.assertEquals(resp['length'], None) + self.assertEquals(resp['etag'], None) + + def test_headers_to_object_info_meta(self): + headers = {'X-Object-Meta-Whatevs': 14, + 'x-object-meta-somethingelse': 0} + resp = headers_to_object_info(headers.items(), 200) + self.assertEquals(len(resp['meta']), 2) + self.assertEquals(resp['meta']['whatevs'], 14) + self.assertEquals(resp['meta']['somethingelse'], 0) + + def test_headers_to_object_info_values(self): + headers = { + 'content-length': '1024', + 'content-type': 'application/json', + } + resp = headers_to_object_info(headers.items(), 200) + self.assertEquals(resp['length'], '1024') + self.assertEquals(resp['type'], 'application/json') + + headers['x-unused-header'] = 'blahblahblah' + self.assertEquals( + resp, + headers_to_object_info(headers.items(), 200)) diff --git a/test/unit/proxy/controllers/test_container.py b/test/unit/proxy/controllers/test_container.py index c2e9483..63e6b0e 100644 --- a/test/unit/proxy/controllers/test_container.py +++ b/test/unit/proxy/controllers/test_container.py @@ -1,4 +1,4 @@ -# Copyright (c) 2010-2012 OpenStack, LLC. +# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -40,6 +40,28 @@ class TestContainerController(unittest.TestCase): self.assertEqual(headers_to_container_info(resp.headers), resp.environ['swift.container/a/c']) + def test_swift_owner(self): + owner_headers = { + 'x-container-read': 'value', 'x-container-write': 'value', + 'x-container-sync-key': 'value', 'x-container-sync-to': 'value'} + controller = proxy_server.ContainerController(self.app, 'a', 'c') + + req = Request.blank('/a/c') + with mock.patch('swift.proxy.controllers.base.http_connect', + fake_http_connect(200, 200, headers=owner_headers)): + resp = controller.HEAD(req) + self.assertEquals(2, resp.status_int // 100) + for key in owner_headers: + self.assertTrue(key not in resp.headers) + + req = Request.blank('/a/c', environ={'swift_owner': True}) + with mock.patch('swift.proxy.controllers.base.http_connect', + fake_http_connect(200, 200, headers=owner_headers)): + resp = controller.HEAD(req) + self.assertEquals(2, resp.status_int // 100) + for key in owner_headers: + self.assertTrue(key in resp.headers) + if __name__ == '__main__': unittest.main() diff --git a/test/unit/proxy/controllers/test_obj.py b/test/unit/proxy/controllers/test_obj.py index e4af789..cae62b0 100755 --- a/test/unit/proxy/controllers/test_obj.py +++ b/test/unit/proxy/controllers/test_obj.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2010-2012 OpenStack, LLC. +# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,9 +15,28 @@ # limitations under the License. import unittest +from contextlib import contextmanager +import mock + +import swift from swift.proxy import server as proxy_server -from test.unit import FakeRing, FakeMemcache +from test.unit import FakeRing, FakeMemcache, fake_http_connect + + +@contextmanager +def set_http_connect(*args, **kwargs): + old_connect = swift.proxy.controllers.base.http_connect + new_connect = fake_http_connect(*args, **kwargs) + swift.proxy.controllers.base.http_connect = new_connect + swift.proxy.controllers.obj.http_connect = new_connect + swift.proxy.controllers.account.http_connect = new_connect + swift.proxy.controllers.container.http_connect = new_connect + yield new_connect + swift.proxy.controllers.base.http_connect = old_connect + swift.proxy.controllers.obj.http_connect = old_connect + swift.proxy.controllers.account.http_connect = old_connect + swift.proxy.controllers.container.http_connect = old_connect class TestObjControllerWriteAffinity(unittest.TestCase): @@ -44,7 +63,8 @@ class TestObjControllerWriteAffinity(unittest.TestCase): def test_iter_nodes_local_first_moves_locals_first(self): controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') - self.app.write_affinity_is_local_fn = (lambda node: node['region'] == 1) + self.app.write_affinity_is_local_fn = ( + lambda node: node['region'] == 1) self.app.write_affinity_node_count = lambda ring: 4 all_nodes = self.app.object_ring.get_part_nodes(1) @@ -59,6 +79,44 @@ class TestObjControllerWriteAffinity(unittest.TestCase): # we don't skip any nodes self.assertEqual(sorted(all_nodes), sorted(local_first_nodes)) + def test_connect_put_node_timeout(self): + controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') + self.app.conn_timeout = 0.1 + with set_http_connect(200, slow_connect=True): + nodes = [dict(ip='', port='', device='')] + res = controller._connect_put_node(nodes, '', '', {}, ('', '')) + self.assertTrue(res is None) + + +class TestObjController(unittest.TestCase): + + def test_PUT_log_info(self): + # mock out enough to get to the area of the code we want to test + with mock.patch('swift.proxy.controllers.obj.check_object_creation', + mock.MagicMock(return_value=None)): + app = mock.MagicMock() + app.container_ring.get_nodes.return_value = (1, [2]) + app.object_ring.get_nodes.return_value = (1, [2]) + controller = proxy_server.ObjectController(app, 'a', 'c', 'o') + controller.container_info = mock.MagicMock(return_value={ + 'partition': 1, + 'nodes': [{}], + 'write_acl': None, + 'sync_key': None, + 'versions': None}) + # and now test that we add the header to log_info + req = swift.common.swob.Request.blank('/v1/a/c/o') + req.headers['x-copy-from'] = 'somewhere' + controller.PUT(req) + self.assertEquals( + req.environ.get('swift.log_info'), ['x-copy-from:somewhere']) + # and then check that we don't do that for originating POSTs + req = swift.common.swob.Request.blank('/v1/a/c/o') + req.method = 'POST' + req.headers['x-copy-from'] = 'elsewhere' + controller.PUT(req) + self.assertEquals(req.environ.get('swift.log_info'), None) + if __name__ == '__main__': unittest.main() diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 488f253..490e8fc 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -1,19 +1,4 @@ -# Copyright (c) 2010-2013 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Copyright (c) 2013 Red Hat, Inc. +# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -34,16 +19,17 @@ import logging import os import sys import unittest -from nose import SkipTest import urlparse -import signal +from nose import SkipTest from contextlib import contextmanager, nested, closing from gzip import GzipFile from shutil import rmtree +import gc import time from urllib import quote from hashlib import md5 from tempfile import mkdtemp +import weakref import mock from eventlet import sleep, spawn, wsgi, listen @@ -53,18 +39,18 @@ import gluster.swift.common.Glusterfs as gfs gfs.RUN_DIR = mkdtemp() from test.unit import connect_tcp, readuntil2crlfs, FakeLogger, \ - fake_http_connect, FakeRing, FakeMemcache -from gluster.swift.proxy.server import server as proxy_server -from gluster.swift.obj import server as object_server + fake_http_connect, FakeRing, FakeMemcache, DebugLogger +from gluster.swift.proxy import server as proxy_server from gluster.swift.account import server as account_server from gluster.swift.container import server as container_server +from gluster.swift.obj import server as object_server from swift.common import ring from swift.common.exceptions import ChunkReadTimeout, SegmentError from swift.common.constraints import MAX_META_NAME_LENGTH, \ MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE, \ MAX_FILE_SIZE, MAX_ACCOUNT_NAME_LENGTH, MAX_CONTAINER_NAME_LENGTH from swift.common import utils -from swift.common.utils import mkdirs, normalize_timestamp, NullLogger +from swift.common.utils import mkdirs, normalize_timestamp from swift.common.wsgi import monkey_patch_mimetools from swift.proxy.controllers.obj import SegmentedIterable from swift.proxy.controllers.base import get_container_memcache_key, \ @@ -78,22 +64,15 @@ logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) STATIC_TIME = time.time() -_request_instances = 0 +_request_instances = weakref.WeakKeyDictionary() _test_coros = _test_servers = _test_sockets = _orig_container_listing_limit = \ _testdir = None def request_init(self, *args, **kwargs): - global _request_instances self._orig_init(*args, **kwargs) - _request_instances += 1 - -def request_del(self): - global _request_instances - if self._orig_del: - self._orig_del() - _request_instances -= 1 + _request_instances[self] = None def setup(): @@ -102,8 +81,6 @@ def setup(): _orig_container_listing_limit, _test_coros Request._orig_init = Request.__init__ Request.__init__ = request_init - Request._orig_del = getattr(Request, '__del__', None) - Request.__del__ = request_del monkey_patch_mimetools() # Since we're starting up a lot here, we're going to test more than # just chunked puts; we're also going to test parts of @@ -132,8 +109,8 @@ def setup(): obj2lis = listen(('localhost', 0)) _test_sockets = \ (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) - with closing(GzipFile(os.path.join(_testdir, 'account.ring.gz'), 'wb')) \ - as f: + account_ring_path = os.path.join(_testdir, 'account.ring.gz') + with closing(GzipFile(account_ring_path, 'wb')) as f: pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', 'port': acc1lis.getsockname()[1]}, @@ -143,8 +120,8 @@ def setup(): {'id': 1, 'zone': 1, 'device': 'a', 'ip': '127.0.0.1', 'port': acc2lis.getsockname()[1]}], 30), f) - with closing(GzipFile(os.path.join(_testdir, 'container.ring.gz'), 'wb')) \ - as f: + container_ring_path = os.path.join(_testdir, 'container.ring.gz') + with closing(GzipFile(container_ring_path, 'wb')) as f: pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', 'port': con1lis.getsockname()[1]}, @@ -154,8 +131,8 @@ def setup(): {'id': 1, 'zone': 1, 'device': 'a', 'ip': '127.0.0.1', 'port': con2lis.getsockname()[1]}], 30), f) - with closing(GzipFile(os.path.join(_testdir, 'object.ring.gz'), 'wb')) \ - as f: + object_ring_path = os.path.join(_testdir, 'object.ring.gz') + with closing(GzipFile(object_ring_path, 'wb')) as f: pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', 'port': obj1lis.getsockname()[1]}, @@ -174,9 +151,6 @@ def setup(): obj2srv = object_server.ObjectController(conf) _test_servers = \ (prosrv, acc1srv, acc2srv, con1srv, con2srv, obj1srv, obj2srv) - # Use DebugLogger() when trying to figure out what failed in the spawned - # servers. - from test.unit import DebugLogger nl = DebugLogger() prospa = spawn(wsgi.server, prolis, prosrv, nl) acc1spa = spawn(wsgi.server, acc1lis, acc1srv, nl) @@ -212,7 +186,8 @@ def setup(): fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' - assert headers[:len(exp)] == exp, "Expected '%s', encountered '%s'" % (exp, headers[:len(exp)]) + assert headers[:len(exp)] == exp, "Expected '%s', encountered '%s'" % ( + exp, headers[:len(exp)]) def teardown(): @@ -222,8 +197,6 @@ def teardown(): _orig_container_listing_limit rmtree(os.path.dirname(_testdir)) Request.__init__ = Request._orig_init - if Request._orig_del: - Request.__del__ = Request._orig_del def sortHeaderNames(headerNames): @@ -391,7 +364,7 @@ class TestController(unittest.TestCase): # 'container_count' changed from 0 to None cache_key = get_account_memcache_key(self.account) account_info = {'status': 404, - 'container_count': None, # internally keep None + 'container_count': None, # internally keep None 'total_object_count': None, 'bytes': None, 'meta': {}} @@ -459,8 +432,8 @@ class TestController(unittest.TestCase): with save_globals(): headers = {'x-container-read': self.read_acl, 'x-container-write': self.write_acl} - set_http_connect(200, # account_info is found - 200, headers=headers) # container_info is found + set_http_connect(200, # account_info is found + 200, headers=headers) # container_info is found ret = self.controller.container_info( self.account, self.container, self.request) self.check_container_info_return(ret) @@ -482,8 +455,8 @@ class TestController(unittest.TestCase): return True, True, 0 with save_globals(): - set_http_connect(503, 204, # account_info found - 504, 404, 404) # container_info 'NotFound' + set_http_connect(503, 204, # account_info found + 504, 404, 404) # container_info 'NotFound' ret = self.controller.container_info( self.account, self.container, self.request) self.check_container_info_return(ret, True) @@ -499,7 +472,7 @@ class TestController(unittest.TestCase): self.account, self.container, self.request) self.check_container_info_return(ret, True) - set_http_connect(503, 404, 404)# account_info 'NotFound' + set_http_connect(503, 404, 404) # account_info 'NotFound' ret = self.controller.container_info( self.account, self.container, self.request) self.check_container_info_return(ret, True) @@ -720,41 +693,31 @@ class TestObjectController(unittest.TestCase): self.assertEquals(res.status_int, expected) def test_GET_newest_large_file(self): - calls = [0] - - def handler(_junk1, _junk2): - calls[0] += 1 - - old_handler = signal.signal(signal.SIGPIPE, handler) - try: - prolis = _test_sockets[0] - prosrv = _test_servers[0] - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - obj = 'a' * (1024 * 1024) - path = '/v1/a/c/o.large' - fd.write('PUT %s HTTP/1.1\r\n' - 'Host: localhost\r\n' - 'Connection: close\r\n' - 'X-Storage-Token: t\r\n' - 'Content-Length: %s\r\n' - 'Content-Type: application/octet-stream\r\n' - '\r\n%s' % (path, str(len(obj)), obj)) - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 201' - self.assertEqual(headers[:len(exp)], exp) - req = Request.blank(path, - environ={'REQUEST_METHOD': 'GET'}, - headers={'Content-Type': - 'application/octet-stream', - 'X-Newest': 'true'}) - res = req.get_response(prosrv) - self.assertEqual(res.status_int, 200) - self.assertEqual(res.body, obj) - self.assertEqual(calls[0], 0) - finally: - signal.signal(signal.SIGPIPE, old_handler) + prolis = _test_sockets[0] + prosrv = _test_servers[0] + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + obj = 'a' * (1024 * 1024) + path = '/v1/a/c/o.large' + fd.write('PUT %s HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'X-Storage-Token: t\r\n' + 'Content-Length: %s\r\n' + 'Content-Type: application/octet-stream\r\n' + '\r\n%s' % (path, str(len(obj)), obj)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEqual(headers[:len(exp)], exp) + req = Request.blank(path, + environ={'REQUEST_METHOD': 'GET'}, + headers={'Content-Type': + 'application/octet-stream', + 'X-Newest': 'true'}) + res = req.get_response(prosrv) + self.assertEqual(res.status_int, 200) + self.assertEqual(res.body, obj) def test_PUT_expect_header_zero_content_length(self): test_errors = [] @@ -860,8 +823,8 @@ class TestObjectController(unittest.TestCase): proxy_server.ObjectController(self.app, 'a', 'c', 'o.jpg') controller.error_limit( self.app.object_ring.get_part_nodes(1)[0], 'test') - set_http_connect(200, 200, # account, container - 201, 201, 201, # 3 working backends + set_http_connect(200, 200, # account, container + 201, 201, 201, # 3 working backends give_connect=test_connect) req = Request.blank('/a/c/o.jpg', {}) req.content_length = 1 @@ -874,7 +837,7 @@ class TestObjectController(unittest.TestCase): # this is kind of a hokey test, but in FakeRing, the port is even when # the region is 0, and odd when the region is 1, so this test asserts # that we wrote to 2 nodes in region 0, then went to 1 non-r0 node. - self.assertEqual(0, written_to[0][1] % 2) # it's (ip, port, device) + self.assertEqual(0, written_to[0][1] % 2) # it's (ip, port, device) self.assertEqual(0, written_to[1][1] % 2) self.assertNotEqual(0, written_to[2][1] % 2) @@ -1085,7 +1048,7 @@ class TestObjectController(unittest.TestCase): response_bodies = ( '', # HEAD /a '', # HEAD /a/c - '', # GET manifest + simplejson.dumps([]), # GET manifest simplejson.dumps([])) # GET empty listing with save_globals(): @@ -1279,8 +1242,8 @@ class TestObjectController(unittest.TestCase): "content_type": "application/octet-stream"}] json_listing = simplejson.dumps(listing) response_bodies = ( - '', # HEAD /a - '', # HEAD /a/c + '', # HEAD /a + '', # HEAD /a/c json_listing) # GET manifest with save_globals(): controller = proxy_server.ObjectController( @@ -1432,7 +1395,7 @@ class TestObjectController(unittest.TestCase): {"hash": "8681fb3ada2715c8754706ee5f23d4f8", "last_modified": "2012-11-08T04:05:37.846710", "bytes": 4, - "name": "/d2/sub_manifest", + "name": u"/d2/sub_manifest \u2661", "sub_slo": True, "content_type": "application/octet-stream"}, {"hash": "419af6d362a14b7a789ba1c7e772bbae", "last_modified": "2012-11-08T04:05:37.866820", @@ -1455,8 +1418,8 @@ class TestObjectController(unittest.TestCase): '', # HEAD /a '', # HEAD /a/c simplejson.dumps(listing), # GET manifest - 'Aa', # GET seg01 simplejson.dumps(sub_listing), # GET sub_manifest + 'Aa', # GET seg01 'Bb', # GET seg02 'Cc', # GET seg03 'Dd') # GET seg04 @@ -1478,12 +1441,12 @@ class TestObjectController(unittest.TestCase): 200, # HEAD /a 200, # HEAD /a/c 200, # GET listing1 - 200, # GET seg01 200, # GET sub listing1 + 200, # GET seg01 200, # GET seg02 200, # GET seg03 200, # GET seg04 - headers=[{}, {}, slob_headers, {}, slob_headers, {}, {}, {}], + headers=[{}, {}, slob_headers, slob_headers, {}, {}, {}, {}], body_iter=response_bodies, give_connect=capture_requested_paths) req = Request.blank('/a/c/manifest') @@ -1496,7 +1459,8 @@ class TestObjectController(unittest.TestCase): requested, [['HEAD', '/a', {}], ['HEAD', '/a/c', {}], - ['GET', '/a/c/manifest', {}]]) + ['GET', '/a/c/manifest', {}], + ['GET', '/a/d2/sub_manifest \xe2\x99\xa1', {}]]) # iterating over body will retrieve manifest and sub manifest's # objects self.assertEqual(resp.body, 'AaBbCcDd') @@ -1505,12 +1469,116 @@ class TestObjectController(unittest.TestCase): [['HEAD', '/a', {}], ['HEAD', '/a/c', {}], ['GET', '/a/c/manifest', {}], + ['GET', '/a/d2/sub_manifest \xe2\x99\xa1', {}], ['GET', '/a/d1/seg01', {}], - ['GET', '/a/d2/sub_manifest', {}], ['GET', '/a/d1/seg02', {}], ['GET', '/a/d2/seg03', {}], ['GET', '/a/d1/seg04', {}]]) + def test_GET_nested_manifest_slo_with_range(self): + """ + Original whole slo is Aa1234Bb where 1234 is a sub-manifests. I'm + pulling out 34Bb + """ + listing = [{"hash": "98568d540134639be4655198a36614a4", # Aa + "last_modified": "2012-11-08T04:05:37.866820", + "bytes": 2, + "name": "/d1/seg01", + "content_type": "application/octet-stream"}, + {"hash": "7b4b0ffa275d404bdc2fc6384916714f", # SubManifest1 + "last_modified": "2012-11-08T04:05:37.866820", + "bytes": 4, "sub_slo": True, + "name": "/d2/subManifest01", + "content_type": "application/octet-stream"}, + {"hash": "d526f1c8ef6c1e4e980e2b8471352d23", # Bb + "last_modified": "2012-11-08T04:05:37.866820", + "bytes": 2, + "name": "/d1/seg02", + "content_type": "application/octet-stream"}] + + sublisting = [{"hash": "c20ad4d76fe97759aa27a0c99bff6710", # 12 + "last_modified": "2012-11-08T04:05:37.866820", + "bytes": 2, + "name": "/d2/subSeg01", + "content_type": "application/octet-stream"}, + {"hash": "e369853df766fa44e1ed0ff613f563bd", # 34 + "last_modified": "2012-11-08T04:05:37.866820", + "bytes": 2, + "name": "/d2/subSeg02", + "content_type": "application/octet-stream"}] + + response_bodies = ( + '', # HEAD /a + '', # HEAD /a/c + simplejson.dumps(listing)[1:1], # GET incomplete manifest + simplejson.dumps(listing), # GET complete manifest + simplejson.dumps(sublisting), # GET complete submanifest + '34', # GET subseg02 + 'Bb') # GET seg02 + etag_iter = ['', '', '', '', '', + 'e369853df766fa44e1ed0ff613f563bd', # subSeg02 + 'd526f1c8ef6c1e4e980e2b8471352d23'] # seg02 + headers = [{}, {}, + {'X-Static-Large-Object': 'True', + 'content-type': 'text/html; swift_bytes=4'}, + {'X-Static-Large-Object': 'True', + 'content-type': 'text/html; swift_bytes=4'}, + {'X-Static-Large-Object': 'True', + 'content-type': 'text/html; swift_bytes=4'}, + {}, {}] + self.assertTrue(len(response_bodies) == len(etag_iter) == len(headers)) + with save_globals(): + controller = proxy_server.ObjectController( + self.app, 'a', 'c', 'manifest') + + requested = [] + + def capture_requested_paths(ipaddr, port, device, partition, + method, path, headers=None, + query_string=None): + qs_dict = dict(urlparse.parse_qsl(query_string or '')) + requested.append([method, path, qs_dict]) + + set_http_connect( + 200, # HEAD /a + 200, # HEAD /a/c + 206, # GET incomplete listing + 200, # GET complete listing + 200, # GET complete sublisting + 200, # GET subSeg02 + 200, # GET seg02 + headers=headers, + etags=etag_iter, + body_iter=response_bodies, + give_connect=capture_requested_paths) + + req = Request.blank('/a/c/manifest') + req.range = 'bytes=4-7' + resp = controller.GET(req) + got_called = [False, ] + + def fake_start_response(*args, **kwargs): + got_called[0] = True + self.assertTrue(args[0].startswith('206')) + + app_iter = resp(req.environ, fake_start_response) + resp_body = ''.join(app_iter) # read in entire resp + self.assertEqual(resp.status_int, 206) + self.assertEqual(resp_body, '34Bb') + self.assertTrue(got_called[0]) + self.assertEqual(resp.content_length, 4) + self.assertEqual(resp.content_type, 'text/html') + + self.assertEqual( + requested, + [['HEAD', '/a', {}], + ['HEAD', '/a/c', {}], + ['GET', '/a/c/manifest', {}], # for incomplete manifest + ['GET', '/a/c/manifest', {}], + ['GET', '/a/d2/subManifest01', {}], + ['GET', '/a/d2/subSeg02', {}], + ['GET', '/a/d1/seg02', {}]]) + def test_GET_bad_404_manifest_slo(self): listing = [{"hash": "98568d540134639be4655198a36614a4", "last_modified": "2012-11-08T04:05:37.866820", @@ -1777,16 +1845,14 @@ class TestObjectController(unittest.TestCase): def test_POST(self): with save_globals(): self.app.object_post_as_copy = False - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') def test_status_map(statuses, expected): set_http_connect(*statuses) self.app.memcache.store = {} - req = Request.blank('/a/c/o', {}, + req = Request.blank('/v1/a/c/o', {}, method='POST', headers={'Content-Type': 'foo/bar'}) self.app.update_request(req) - res = controller.POST(req) + res = req.get_response(self.app) expected = str(expected) self.assertEquals(res.status[:len(expected)], expected) test_status_map((200, 200, 202, 202, 202), 202) @@ -1799,16 +1865,13 @@ class TestObjectController(unittest.TestCase): def test_POST_as_copy(self): with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') - def test_status_map(statuses, expected): set_http_connect(*statuses) self.app.memcache.store = {} - req = Request.blank('/a/c/o', {}, + req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'POST'}, headers={'Content-Type': 'foo/bar'}) self.app.update_request(req) - res = controller.POST(req) + res = req.get_response(self.app) expected = str(expected) self.assertEquals(res.status[:len(expected)], expected) test_status_map((200, 200, 200, 200, 200, 202, 202, 202), 202) @@ -1821,15 +1884,12 @@ class TestObjectController(unittest.TestCase): def test_DELETE(self): with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') - def test_status_map(statuses, expected): set_http_connect(*statuses) self.app.memcache.store = {} - req = Request.blank('/a/c/o', {}) + req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'DELETE'}) self.app.update_request(req) - res = controller.DELETE(req) + res = req.get_response(self.app) self.assertEquals(res.status[:len(str(expected))], str(expected)) test_status_map((200, 200, 204, 204, 204), 204) @@ -1841,15 +1901,12 @@ class TestObjectController(unittest.TestCase): def test_HEAD(self): with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') - def test_status_map(statuses, expected): set_http_connect(*statuses) self.app.memcache.store = {} - req = Request.blank('/a/c/o', {}) + req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'HEAD'}) self.app.update_request(req) - res = controller.HEAD(req) + res = req.get_response(self.app) self.assertEquals(res.status[:len(str(expected))], str(expected)) if expected < 400: @@ -1867,16 +1924,14 @@ class TestObjectController(unittest.TestCase): def test_HEAD_newest(self): with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') - def test_status_map(statuses, expected, timestamps, expected_timestamp): set_http_connect(*statuses, timestamps=timestamps) self.app.memcache.store = {} - req = Request.blank('/a/c/o', {}, headers={'x-newest': 'true'}) + req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'HEAD'}, + headers={'x-newest': 'true'}) self.app.update_request(req) - res = controller.HEAD(req) + res = req.get_response(self.app) self.assertEquals(res.status[:len(str(expected))], str(expected)) self.assertEquals(res.headers.get('last-modified'), @@ -1898,16 +1953,14 @@ class TestObjectController(unittest.TestCase): def test_GET_newest(self): with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') - def test_status_map(statuses, expected, timestamps, expected_timestamp): set_http_connect(*statuses, timestamps=timestamps) self.app.memcache.store = {} - req = Request.blank('/a/c/o', {}, headers={'x-newest': 'true'}) + req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'GET'}, + headers={'x-newest': 'true'}) self.app.update_request(req) - res = controller.GET(req) + res = req.get_response(self.app) self.assertEquals(res.status[:len(str(expected))], str(expected)) self.assertEquals(res.headers.get('last-modified'), @@ -1927,16 +1980,13 @@ class TestObjectController(unittest.TestCase): None, '1'), '1') with save_globals(): - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') - def test_status_map(statuses, expected, timestamps, expected_timestamp): set_http_connect(*statuses, timestamps=timestamps) self.app.memcache.store = {} - req = Request.blank('/a/c/o', {}) + req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'HEAD'}) self.app.update_request(req) - res = controller.HEAD(req) + res = req.get_response(self.app) self.assertEquals(res.status[:len(str(expected))], str(expected)) self.assertEquals(res.headers.get('last-modified'), @@ -1957,123 +2007,118 @@ class TestObjectController(unittest.TestCase): with save_globals(): limit = MAX_META_VALUE_LENGTH self.app.object_post_as_copy = False - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') + proxy_server.ObjectController(self.app, 'account', + 'container', 'object') set_http_connect(200, 200, 202, 202, 202) # acct cont obj obj obj - req = Request.blank('/a/c/o', {}, headers={ - 'Content-Type': 'foo/bar', - 'X-Object-Meta-Foo': 'x' * limit}) + req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'POST'}, + headers={'Content-Type': 'foo/bar', + 'X-Object-Meta-Foo': 'x' * limit}) self.app.update_request(req) - res = controller.POST(req) + res = req.get_response(self.app) self.assertEquals(res.status_int, 202) set_http_connect(202, 202, 202) - req = Request.blank('/a/c/o', {}, headers={ - 'Content-Type': 'foo/bar', - 'X-Object-Meta-Foo': 'x' * (limit + 1)}) + req = Request.blank( + '/v1/a/c/o', {'REQUEST_METHOD': 'POST'}, + headers={'Content-Type': 'foo/bar', + 'X-Object-Meta-Foo': 'x' * (limit + 1)}) self.app.update_request(req) - res = controller.POST(req) + res = req.get_response(self.app) self.assertEquals(res.status_int, 400) def test_POST_as_copy_meta_val_len(self): with save_globals(): limit = MAX_META_VALUE_LENGTH - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') set_http_connect(200, 200, 200, 200, 200, 202, 202, 202) # acct cont objc objc objc obj obj obj - req = Request.blank('/a/c/o', {}, headers={ - 'Content-Type': 'foo/bar', - 'X-Object-Meta-Foo': 'x' * limit}) + req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'POST'}, + headers={'Content-Type': 'foo/bar', + 'X-Object-Meta-Foo': 'x' * limit}) self.app.update_request(req) - res = controller.POST(req) + res = req.get_response(self.app) self.assertEquals(res.status_int, 202) set_http_connect(202, 202, 202) - req = Request.blank('/a/c/o', {}, headers={ - 'Content-Type': 'foo/bar', - 'X-Object-Meta-Foo': 'x' * (limit + 1)}) + req = Request.blank( + '/v1/a/c/o', {'REQUEST_METHOD': 'POST'}, + headers={'Content-Type': 'foo/bar', + 'X-Object-Meta-Foo': 'x' * (limit + 1)}) self.app.update_request(req) - res = controller.POST(req) + res = req.get_response(self.app) self.assertEquals(res.status_int, 400) def test_POST_meta_key_len(self): with save_globals(): limit = MAX_META_NAME_LENGTH self.app.object_post_as_copy = False - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') set_http_connect(200, 200, 202, 202, 202) # acct cont obj obj obj - req = Request.blank('/a/c/o', {}, headers={ - 'Content-Type': 'foo/bar', - ('X-Object-Meta-' + 'x' * limit): 'x'}) + req = Request.blank( + '/v1/a/c/o', {'REQUEST_METHOD': 'POST'}, + headers={'Content-Type': 'foo/bar', + ('X-Object-Meta-' + 'x' * limit): 'x'}) self.app.update_request(req) - res = controller.POST(req) + res = req.get_response(self.app) self.assertEquals(res.status_int, 202) set_http_connect(202, 202, 202) - req = Request.blank('/a/c/o', {}, headers={ - 'Content-Type': 'foo/bar', - ('X-Object-Meta-' + 'x' * (limit + 1)): 'x'}) + req = Request.blank( + '/v1/a/c/o', {'REQUEST_METHOD': 'POST'}, + headers={'Content-Type': 'foo/bar', + ('X-Object-Meta-' + 'x' * (limit + 1)): 'x'}) self.app.update_request(req) - res = controller.POST(req) + res = req.get_response(self.app) self.assertEquals(res.status_int, 400) def test_POST_as_copy_meta_key_len(self): with save_globals(): limit = MAX_META_NAME_LENGTH - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') set_http_connect(200, 200, 200, 200, 200, 202, 202, 202) # acct cont objc objc objc obj obj obj - req = Request.blank('/a/c/o', {}, headers={ - 'Content-Type': 'foo/bar', - ('X-Object-Meta-' + 'x' * limit): 'x'}) + req = Request.blank( + '/v1/a/c/o', {'REQUEST_METHOD': 'POST'}, + headers={'Content-Type': 'foo/bar', + ('X-Object-Meta-' + 'x' * limit): 'x'}) self.app.update_request(req) - res = controller.POST(req) + res = req.get_response(self.app) self.assertEquals(res.status_int, 202) set_http_connect(202, 202, 202) - req = Request.blank('/a/c/o', {}, headers={ - 'Content-Type': 'foo/bar', - ('X-Object-Meta-' + 'x' * (limit + 1)): 'x'}) + req = Request.blank( + '/v1/a/c/o', {'REQUEST_METHOD': 'POST'}, + headers={'Content-Type': 'foo/bar', + ('X-Object-Meta-' + 'x' * (limit + 1)): 'x'}) self.app.update_request(req) - res = controller.POST(req) + res = req.get_response(self.app) self.assertEquals(res.status_int, 400) def test_POST_meta_count(self): with save_globals(): limit = MAX_META_COUNT - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') headers = dict( (('X-Object-Meta-' + str(i), 'a') for i in xrange(limit + 1))) headers.update({'Content-Type': 'foo/bar'}) set_http_connect(202, 202, 202) - req = Request.blank('/a/c/o', {}, headers=headers) + req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'POST'}, + headers=headers) self.app.update_request(req) - res = controller.POST(req) + res = req.get_response(self.app) self.assertEquals(res.status_int, 400) def test_POST_meta_size(self): with save_globals(): limit = MAX_META_OVERALL_SIZE - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') count = limit / 256 # enough to cause the limit to be reached headers = dict( (('X-Object-Meta-' + str(i), 'a' * 256) for i in xrange(count + 1))) headers.update({'Content-Type': 'foo/bar'}) set_http_connect(202, 202, 202) - req = Request.blank('/a/c/o', {}, headers=headers) + req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'POST'}, + headers=headers) self.app.update_request(req) - res = controller.POST(req) + res = req.get_response(self.app) self.assertEquals(res.status_int, 400) def test_PUT_not_autodetect_content_type(self): with save_globals(): - controller = proxy_server.ObjectController( - self.app, 'a', 'c', 'o.html') - headers = {'Content-Type': 'something/right', 'Content-Length': 0} it_worked = [] @@ -2086,17 +2131,15 @@ class TestObjectController(unittest.TestCase): set_http_connect(204, 204, 201, 201, 201, give_connect=verify_content_type) - req = Request.blank('/a/c/o.html', {}, headers=headers) + req = Request.blank('/v1/a/c/o.html', {'REQUEST_METHOD': 'PUT'}, + headers=headers) self.app.update_request(req) - res = controller.PUT(req) + req.get_response(self.app) self.assertNotEquals(it_worked, []) self.assertTrue(all(it_worked)) def test_PUT_autodetect_content_type(self): with save_globals(): - controller = proxy_server.ObjectController( - self.app, 'a', 'c', 'o.html') - headers = {'Content-Type': 'something/wrong', 'Content-Length': 0, 'X-Detect-Content-Type': 'True'} it_worked = [] @@ -2110,9 +2153,10 @@ class TestObjectController(unittest.TestCase): set_http_connect(204, 204, 201, 201, 201, give_connect=verify_content_type) - req = Request.blank('/a/c/o.html', {}, headers=headers) + req = Request.blank('/v1/a/c/o.html', {'REQUEST_METHOD': 'PUT'}, + headers=headers) self.app.update_request(req) - res = controller.PUT(req) + req.get_response(self.app) self.assertNotEquals(it_worked, []) self.assertTrue(all(it_worked)) @@ -2143,20 +2187,18 @@ class TestObjectController(unittest.TestCase): return ' ' return '' - req = Request.blank('/a/c/o', + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': SlowBody()}, headers={'Content-Length': '4', 'Content-Type': 'text/plain'}) self.app.update_request(req) - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') set_http_connect(200, 200, 201, 201, 201) # acct cont obj obj obj - resp = controller.PUT(req) + resp = req.get_response(self.app) self.assertEquals(resp.status_int, 201) self.app.client_timeout = 0.1 - req = Request.blank('/a/c/o', + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': SlowBody()}, headers={'Content-Length': '4', @@ -2164,7 +2206,7 @@ class TestObjectController(unittest.TestCase): self.app.update_request(req) set_http_connect(201, 201, 201) # obj obj obj - resp = controller.PUT(req) + resp = req.get_response(self.app) self.assertEquals(resp.status_int, 408) def test_client_disconnect(self): @@ -2190,17 +2232,15 @@ class TestObjectController(unittest.TestCase): def read(self, size=-1): raise Exception('Disconnected') - req = Request.blank('/a/c/o', + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': SlowBody()}, headers={'Content-Length': '4', 'Content-Type': 'text/plain'}) self.app.update_request(req) - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') set_http_connect(200, 200, 201, 201, 201) # acct cont obj obj obj - resp = controller.PUT(req) + resp = req.get_response(self.app) self.assertEquals(resp.status_int, 499) def test_node_read_timeout(self): @@ -2217,13 +2257,11 @@ class TestObjectController(unittest.TestCase): for dev in self.app.object_ring.devs.values(): dev['ip'] = '127.0.0.1' dev['port'] = 1 - req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'GET'}) + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) self.app.update_request(req) - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') set_http_connect(200, 200, 200, slow=True) req.sent_size = 0 - resp = controller.GET(req) + resp = req.get_response(self.app) got_exc = False try: resp.body @@ -2232,7 +2270,7 @@ class TestObjectController(unittest.TestCase): self.assert_(not got_exc) self.app.node_timeout = 0.1 set_http_connect(200, 200, 200, slow=True) - resp = controller.GET(req) + resp = req.get_response(self.app) got_exc = False try: resp.body @@ -2254,26 +2292,24 @@ class TestObjectController(unittest.TestCase): for dev in self.app.object_ring.devs.values(): dev['ip'] = '127.0.0.1' dev['port'] = 1 - req = Request.blank('/a/c/o', + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '4', 'Content-Type': 'text/plain'}, body=' ') self.app.update_request(req) - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') set_http_connect(200, 200, 201, 201, 201, slow=True) - resp = controller.PUT(req) + resp = req.get_response(self.app) self.assertEquals(resp.status_int, 201) self.app.node_timeout = 0.1 set_http_connect(201, 201, 201, slow=True) - req = Request.blank('/a/c/o', + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '4', 'Content-Type': 'text/plain'}, body=' ') self.app.update_request(req) - resp = controller.PUT(req) + resp = req.get_response(self.app) self.assertEquals(resp.status_int, 503) def test_iter_nodes(self): @@ -2422,20 +2458,18 @@ class TestObjectController(unittest.TestCase): def test_proxy_passes_content_type(self): with save_globals(): - req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'GET'}) + req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) self.app.update_request(req) - controller = proxy_server.ObjectController(self.app, 'account', - 'container', 'object') set_http_connect(200, 200, 200) - resp = controller.GET(req) + resp = req.get_response(self.app) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.content_type, 'x-application/test') set_http_connect(200, 200, 200) - resp = controller.GET(req) + resp = req.get_response(self.app) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.content_length, 0) set_http_connect(200, 200, 200, slow=True) - resp = controller.GET(req) + resp = req.get_response(self.app) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.content_length, 4) @@ -2865,7 +2899,7 @@ class TestObjectController(unittest.TestCase): copy_from_obj_body = LargeResponseBody() set_http_connect(200, 200, 200, 200, 200, 201, 201, 201, - body=copy_from_obj_body) + body=copy_from_obj_body) self.app.memcache.store = {} resp = controller.PUT(req) self.assertEquals(resp.status_int, 413) @@ -2995,7 +3029,7 @@ class TestObjectController(unittest.TestCase): copy_from_obj_body = LargeResponseBody() set_http_connect(200, 200, 200, 200, 200, 201, 201, 201, - body=copy_from_obj_body) + body=copy_from_obj_body) self.app.memcache.store = {} resp = controller.COPY(req) self.assertEquals(resp.status_int, 413) @@ -3347,16 +3381,24 @@ class TestObjectController(unittest.TestCase): body = fd.read() self.assertEquals(body, 'oh hai123456789abcdef') - def test_version_manifest(self): + def test_version_manifest(self, oc='versions', vc='vers', o='name'): versions_to_create = 3 # Create a container for our versioned object testing (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) = _test_sockets sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() - fd.write('PUT /v1/a/versions HTTP/1.1\r\nHost: localhost\r\n' + pre = quote('%03x' % len(o)) + osub = '%s.sub' % o + presub = quote('%03x' % len(osub)) + osub = quote(osub) + presub = quote(presub) + oc = quote(oc) + vc = quote(vc) + fd.write('PUT /v1/a/%s HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Storage-Token: t\r\n' - 'Content-Length: 0\r\nX-Versions-Location: vers\r\n\r\n') + 'Content-Length: 0\r\nX-Versions-Location: %s\r\n\r\n' + % (oc, vc)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' @@ -3364,19 +3406,19 @@ class TestObjectController(unittest.TestCase): # check that the header was set sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() - fd.write('GET /v1/a/versions HTTP/1.1\r\nHost: localhost\r\n' - 'Connection: close\r\nX-Storage-Token: t\r\n\r\n\r\n') + fd.write('GET /v1/a/%s HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Storage-Token: t\r\n\r\n\r\n' % oc) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 2' # 2xx series response self.assertEquals(headers[:len(exp)], exp) - self.assert_('X-Versions-Location: vers' in headers) + self.assert_('X-Versions-Location: %s' % vc in headers) # make the container for the object versions sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() - fd.write('PUT /v1/a/vers HTTP/1.1\r\nHost: localhost\r\n' + fd.write('PUT /v1/a/%s HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Storage-Token: t\r\n' - 'Content-Length: 0\r\n\r\n') + 'Content-Length: 0\r\n\r\n' % vc) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' @@ -3384,10 +3426,10 @@ class TestObjectController(unittest.TestCase): # Create the versioned file sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() - fd.write('PUT /v1/a/versions/name HTTP/1.1\r\nHost: ' + fd.write('PUT /v1/a/%s/%s HTTP/1.1\r\nHost: ' 'localhost\r\nConnection: close\r\nX-Storage-Token: ' 't\r\nContent-Length: 5\r\nContent-Type: text/jibberish0\r\n' - 'X-Object-Meta-Foo: barbaz\r\n\r\n00000\r\n') + 'X-Object-Meta-Foo: barbaz\r\n\r\n00000\r\n' % (oc, o)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' @@ -3397,10 +3439,10 @@ class TestObjectController(unittest.TestCase): sleep(.01) # guarantee that the timestamp changes sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() - fd.write('PUT /v1/a/versions/name HTTP/1.1\r\nHost: ' + fd.write('PUT /v1/a/%s/%s HTTP/1.1\r\nHost: ' 'localhost\r\nConnection: close\r\nX-Storage-Token: ' 't\r\nContent-Length: 5\r\nContent-Type: text/jibberish%s' - '\r\n\r\n%05d\r\n' % (segment, segment)) + '\r\n\r\n%05d\r\n' % (oc, o, segment, segment)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' @@ -3408,9 +3450,9 @@ class TestObjectController(unittest.TestCase): # Ensure retrieving the manifest file gets the latest version sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() - fd.write('GET /v1/a/versions/name HTTP/1.1\r\nHost: ' + fd.write('GET /v1/a/%s/%s HTTP/1.1\r\nHost: ' 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n' - '\r\n') + '\r\n' % (oc, o)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 200' @@ -3422,8 +3464,9 @@ class TestObjectController(unittest.TestCase): # Ensure we have the right number of versions saved sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() - fd.write('GET /v1/a/vers?prefix=004name/ HTTP/1.1\r\nHost: ' - 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n') + fd.write('GET /v1/a/%s?prefix=%s%s/ HTTP/1.1\r\nHost: ' + 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n' + % (vc, pre, o)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 200' @@ -3434,18 +3477,19 @@ class TestObjectController(unittest.TestCase): # copy a version and make sure the version info is stripped sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() - fd.write('COPY /v1/a/versions/name HTTP/1.1\r\nHost: ' + fd.write('COPY /v1/a/%s/%s HTTP/1.1\r\nHost: ' 'localhost\r\nConnection: close\r\nX-Auth-Token: ' - 't\r\nDestination: versions/copied_name\r\n' - 'Content-Length: 0\r\n\r\n') + 't\r\nDestination: %s/copied_name\r\n' + 'Content-Length: 0\r\n\r\n' % (oc, o, oc)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 2' # 2xx series response to the COPY self.assertEquals(headers[:len(exp)], exp) sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() - fd.write('GET /v1/a/versions/copied_name HTTP/1.1\r\nHost: ' - 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n') + fd.write('GET /v1/a/%s/copied_name HTTP/1.1\r\nHost: ' + 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n' + % oc) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 200' @@ -3455,18 +3499,19 @@ class TestObjectController(unittest.TestCase): # post and make sure it's updated sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() - fd.write('POST /v1/a/versions/name HTTP/1.1\r\nHost: ' + fd.write('POST /v1/a/%s/%s HTTP/1.1\r\nHost: ' 'localhost\r\nConnection: close\r\nX-Auth-Token: ' 't\r\nContent-Type: foo/bar\r\nContent-Length: 0\r\n' - 'X-Object-Meta-Bar: foo\r\n\r\n') + 'X-Object-Meta-Bar: foo\r\n\r\n' % (oc, o)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 2' # 2xx series response to the POST self.assertEquals(headers[:len(exp)], exp) sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() - fd.write('GET /v1/a/versions/name HTTP/1.1\r\nHost: ' - 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n') + fd.write('GET /v1/a/%s/%s HTTP/1.1\r\nHost: ' + 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n' + % (oc, o)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 200' @@ -3479,8 +3524,9 @@ class TestObjectController(unittest.TestCase): for segment in xrange(versions_to_create - 1, 0, -1): sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() - fd.write('DELETE /v1/a/versions/name HTTP/1.1\r\nHost: localhost\r' - '\nConnection: close\r\nX-Storage-Token: t\r\n\r\n') + fd.write('DELETE /v1/a/%s/%s HTTP/1.1\r\nHost: localhost\r' + '\nConnection: close\r\nX-Storage-Token: t\r\n\r\n' + % (oc, o)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 2' # 2xx series response @@ -3488,8 +3534,9 @@ class TestObjectController(unittest.TestCase): # Ensure retrieving the manifest file gets the latest version sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() - fd.write('GET /v1/a/versions/name HTTP/1.1\r\nHost: localhost\r\n' - 'Connection: close\r\nX-Auth-Token: t\r\n\r\n') + fd.write('GET /v1/a/%s/%s HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Auth-Token: t\r\n\r\n' + % (oc, o)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 200' @@ -3501,9 +3548,9 @@ class TestObjectController(unittest.TestCase): # Ensure we have the right number of versions saved sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() - fd.write('GET /v1/a/vers?prefix=004name/ HTTP/1.1\r\nHost: ' + fd.write('GET /v1/a/%s?prefix=%s%s/ HTTP/1.1\r\nHost: ' 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r' - '\n') + '\n' % (vc, pre, o)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 2' # 2xx series response @@ -3515,8 +3562,9 @@ class TestObjectController(unittest.TestCase): # Ensure we have no saved versions sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() - fd.write('GET /v1/a/vers?prefix=004name/ HTTP/1.1\r\nHost: ' - 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n') + fd.write('GET /v1/a/%s?prefix=%s%s/ HTTP/1.1\r\nHost: ' + 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n' + % (vc, pre, o)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 204 No Content' @@ -3524,8 +3572,8 @@ class TestObjectController(unittest.TestCase): # delete the last verision sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() - fd.write('DELETE /v1/a/versions/name HTTP/1.1\r\nHost: localhost\r\n' - 'Connection: close\r\nX-Storage-Token: t\r\n\r\n') + fd.write('DELETE /v1/a/%s/%s HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Storage-Token: t\r\n\r\n' % (oc, o)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 2' # 2xx series response @@ -3533,8 +3581,9 @@ class TestObjectController(unittest.TestCase): # Ensure it's all gone sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() - fd.write('GET /v1/a/versions/name HTTP/1.1\r\nHost: ' - 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n') + fd.write('GET /v1/a/%s/%s HTTP/1.1\r\nHost: ' + 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n' + % (oc, o)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 404' @@ -3543,10 +3592,11 @@ class TestObjectController(unittest.TestCase): # make sure manifest files don't get versioned sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() - fd.write('PUT /v1/a/versions/name HTTP/1.1\r\nHost: ' + fd.write('PUT /v1/a/%s/%s HTTP/1.1\r\nHost: ' 'localhost\r\nConnection: close\r\nX-Storage-Token: ' 't\r\nContent-Length: 0\r\nContent-Type: text/jibberish0\r\n' - 'Foo: barbaz\r\nX-Object-Manifest: vers/foo_\r\n\r\n') + 'Foo: barbaz\r\nX-Object-Manifest: %s/foo_\r\n\r\n' + % (oc, vc, o)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' @@ -3554,55 +3604,68 @@ class TestObjectController(unittest.TestCase): # Ensure we have no saved versions sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() - fd.write('GET /v1/a/vers?prefix=004name/ HTTP/1.1\r\nHost: ' - 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n') + fd.write('GET /v1/a/%s?prefix=%s%s/ HTTP/1.1\r\nHost: ' + 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n' + % (vc, pre, o)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 204 No Content' self.assertEquals(headers[:len(exp)], exp) - # DELETE v1/a/c/dir shouldn't delete v1/a/c/dir/sub versions + # DELETE v1/a/c/obj shouldn't delete v1/a/c/obj/sub versions + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/%s/%s HTTP/1.1\r\nHost: ' + 'localhost\r\nConnection: close\r\nX-Storage-Token: ' + 't\r\nContent-Length: 5\r\nContent-Type: text/jibberish0\r\n' + 'Foo: barbaz\r\n\r\n00000\r\n' % (oc, o)) + fd.flush() + headers = readuntil2crlfs(fd) + exp = 'HTTP/1.1 201' + self.assertEquals(headers[:len(exp)], exp) sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() - fd.write('PUT /v1/a/versions/dir HTTP/1.1\r\nHost: ' + fd.write('PUT /v1/a/%s/%s HTTP/1.1\r\nHost: ' 'localhost\r\nConnection: close\r\nX-Storage-Token: ' - 't\r\nContent-Length: 0\r\nContent-Type: application/directory\r\n\r\n') + 't\r\nContent-Length: 5\r\nContent-Type: text/jibberish0\r\n' + 'Foo: barbaz\r\n\r\n00001\r\n' % (oc, o)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' self.assertEquals(headers[:len(exp)], exp) sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() - fd.write('PUT /v1/a/versions/dir/sub HTTP/1.1\r\nHost: ' + fd.write('PUT /v1/a/%s/%s HTTP/1.1\r\nHost: ' 'localhost\r\nConnection: close\r\nX-Storage-Token: ' 't\r\nContent-Length: 4\r\nContent-Type: text/jibberish0\r\n' - 'Foo: barbaz\r\n\r\nsub1\r\n') + 'Foo: barbaz\r\n\r\nsub1\r\n' % (oc, osub)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' self.assertEquals(headers[:len(exp)], exp) sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() - fd.write('PUT /v1/a/versions/dir/sub HTTP/1.1\r\nHost: ' + fd.write('PUT /v1/a/%s/%s HTTP/1.1\r\nHost: ' 'localhost\r\nConnection: close\r\nX-Storage-Token: ' 't\r\nContent-Length: 4\r\nContent-Type: text/jibberish0\r\n' - 'Foo: barbaz\r\n\r\nsub2\r\n') + 'Foo: barbaz\r\n\r\nsub2\r\n' % (oc, osub)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' self.assertEquals(headers[:len(exp)], exp) sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() - fd.write('DELETE /v1/a/versions/dir HTTP/1.1\r\nHost: localhost\r\n' - 'Connection: close\r\nX-Storage-Token: t\r\n\r\n') + fd.write('DELETE /v1/a/%s/%s HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Storage-Token: t\r\n\r\n' % (oc, o)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 2' # 2xx series response self.assertEquals(headers[:len(exp)], exp) sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() - fd.write('GET /v1/a/vers?prefix=007dir/sub/ HTTP/1.1\r\nHost: ' - 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n') + fd.write('GET /v1/a/%s?prefix=%s%s/ HTTP/1.1\r\nHost: ' + 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n' + % (vc, presub, osub)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 2' # 2xx series response @@ -3614,9 +3677,9 @@ class TestObjectController(unittest.TestCase): # Check for when the versions target container doesn't exist sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() - fd.write('PUT /v1/a/whoops HTTP/1.1\r\nHost: localhost\r\n' + fd.write('PUT /v1/a/%swhoops HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Storage-Token: t\r\n' - 'Content-Length: 0\r\nX-Versions-Location: none\r\n\r\n') + 'Content-Length: 0\r\nX-Versions-Location: none\r\n\r\n' % oc) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' @@ -3624,9 +3687,9 @@ class TestObjectController(unittest.TestCase): # Create the versioned file sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() - fd.write('PUT /v1/a/whoops/foo HTTP/1.1\r\nHost: ' + fd.write('PUT /v1/a/%swhoops/foo HTTP/1.1\r\nHost: ' 'localhost\r\nConnection: close\r\nX-Storage-Token: ' - 't\r\nContent-Length: 5\r\n\r\n00000\r\n') + 't\r\nContent-Length: 5\r\n\r\n00000\r\n' % oc) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' @@ -3634,9 +3697,9 @@ class TestObjectController(unittest.TestCase): # Create another version sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() - fd.write('PUT /v1/a/whoops/foo HTTP/1.1\r\nHost: ' + fd.write('PUT /v1/a/%swhoops/foo HTTP/1.1\r\nHost: ' 'localhost\r\nConnection: close\r\nX-Storage-Token: ' - 't\r\nContent-Length: 5\r\n\r\n00001\r\n') + 't\r\nContent-Length: 5\r\n\r\n00001\r\n' % oc) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 412' @@ -3644,13 +3707,102 @@ class TestObjectController(unittest.TestCase): # Delete the object sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() - fd.write('DELETE /v1/a/whoops/foo HTTP/1.1\r\nHost: localhost\r\n' - 'Connection: close\r\nX-Storage-Token: t\r\n\r\n') + fd.write('DELETE /v1/a/%swhoops/foo HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Storage-Token: t\r\n\r\n' % oc) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 2' # 2xx response self.assertEquals(headers[:len(exp)], exp) + def test_version_manifest_utf8(self): + oc = '0_oc_non_ascii\xc2\xa3' + vc = '0_vc_non_ascii\xc2\xa3' + o = '0_o_non_ascii\xc2\xa3' + self.test_version_manifest(oc, vc, o) + + def test_version_manifest_utf8_container(self): + oc = '1_oc_non_ascii\xc2\xa3' + vc = '1_vc_ascii' + o = '1_o_ascii' + self.test_version_manifest(oc, vc, o) + + def test_version_manifest_utf8_version_container(self): + oc = '2_oc_ascii' + vc = '2_vc_non_ascii\xc2\xa3' + o = '2_o_ascii' + self.test_version_manifest(oc, vc, o) + + def test_version_manifest_utf8_containers(self): + oc = '3_oc_non_ascii\xc2\xa3' + vc = '3_vc_non_ascii\xc2\xa3' + o = '3_o_ascii' + self.test_version_manifest(oc, vc, o) + + def test_version_manifest_utf8_object(self): + oc = '4_oc_ascii' + vc = '4_vc_ascii' + o = '4_o_non_ascii\xc2\xa3' + self.test_version_manifest(oc, vc, o) + + def test_version_manifest_utf8_version_container_utf_object(self): + oc = '5_oc_ascii' + vc = '5_vc_non_ascii\xc2\xa3' + o = '5_o_non_ascii\xc2\xa3' + self.test_version_manifest(oc, vc, o) + + def test_version_manifest_utf8_container_utf_object(self): + oc = '6_oc_non_ascii\xc2\xa3' + vc = '6_vc_ascii' + o = '6_o_non_ascii\xc2\xa3' + self.test_version_manifest(oc, vc, o) + + def test_conditional_range_get(self): + (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) = \ + _test_sockets + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + + # make a container + fd = sock.makefile() + fd.write('PUT /v1/a/con HTTP/1.1\r\nHost: localhost\r\n' + 'Connection: close\r\nX-Storage-Token: t\r\n' + 'Content-Length: 0\r\n\r\n') + fd.flush() + exp = 'HTTP/1.1 201' + headers = readuntil2crlfs(fd) + self.assertEquals(headers[:len(exp)], exp) + + # put an object in it + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('PUT /v1/a/con/o HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'X-Storage-Token: t\r\n' + 'Content-Length: 10\r\n' + 'Content-Type: text/plain\r\n' + '\r\n' + 'abcdefghij\r\n') + fd.flush() + exp = 'HTTP/1.1 201' + headers = readuntil2crlfs(fd) + self.assertEquals(headers[:len(exp)], exp) + + # request with both If-None-Match and Range + etag = md5("abcdefghij").hexdigest() + sock = connect_tcp(('localhost', prolis.getsockname()[1])) + fd = sock.makefile() + fd.write('GET /v1/a/con/o HTTP/1.1\r\n' + + 'Host: localhost\r\n' + + 'Connection: close\r\n' + + 'X-Storage-Token: t\r\n' + + 'If-None-Match: "' + etag + '"\r\n' + + 'Range: bytes=3-8\r\n' + + '\r\n') + fd.flush() + exp = 'HTTP/1.1 304' + headers = readuntil2crlfs(fd) + self.assertEquals(headers[:len(exp)], exp) + def test_chunked_put_lobjects_with_nonzero_size_manifest_file(self): raise SkipTest("Not until we support pure object requests") # Create a container for our segmented/manifest object testing @@ -4408,7 +4560,6 @@ class TestObjectController(unittest.TestCase): self.assertTrue('X-Delete-At in past' in resp.body) def test_leak_1(self): - global _request_instances prolis = _test_sockets[0] prosrv = _test_servers[0] obj_len = prosrv.client_chunk_size * 2 @@ -4426,8 +4577,11 @@ class TestObjectController(unittest.TestCase): headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' self.assertEqual(headers[:len(exp)], exp) - # Remember Request instance count - before_request_instances = _request_instances + # Remember Request instance count, make sure the GC is run for pythons + # without reference counting. + for i in xrange(4): + gc.collect() + before_request_instances = len(_request_instances) # GET test file, but disconnect early sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() @@ -4443,7 +4597,10 @@ class TestObjectController(unittest.TestCase): fd.read(1) fd.close() sock.close() - self.assertEquals(before_request_instances, _request_instances) + # Make sure the GC is run again for pythons without reference counting + for i in xrange(4): + gc.collect() + self.assertEquals(before_request_instances, len(_request_instances)) def test_OPTIONS(self): with save_globals(): @@ -4630,7 +4787,8 @@ class TestObjectController(unittest.TestCase): seen_headers = self._gather_x_container_headers( controller.PUT, req, 200, 200, 201, 201, 201) # HEAD HEAD PUT PUT PUT - self.assertEqual(seen_headers, [ + self.assertEqual( + seen_headers, [ {'X-Container-Host': '10.0.0.0:1000', 'X-Container-Partition': '1', 'X-Container-Device': 'sda'}, @@ -4651,7 +4809,8 @@ class TestObjectController(unittest.TestCase): controller.PUT, req, 200, 200, 201, 201, 201) # HEAD HEAD PUT PUT PUT - self.assertEqual(seen_headers, [ + self.assertEqual( + seen_headers, [ {'X-Container-Host': '10.0.0.0:1000', 'X-Container-Partition': '1', 'X-Container-Device': 'sda'}, @@ -4672,7 +4831,8 @@ class TestObjectController(unittest.TestCase): controller.PUT, req, 200, 200, 201, 201, 201) # HEAD HEAD PUT PUT PUT - self.assertEqual(seen_headers, [ + self.assertEqual( + seen_headers, [ {'X-Container-Host': '10.0.0.0:1000,10.0.0.3:1003', 'X-Container-Partition': '1', 'X-Container-Device': 'sda,sdd'}, @@ -4694,7 +4854,8 @@ class TestObjectController(unittest.TestCase): controller.POST, req, 200, 200, 200, 200, 200) # HEAD HEAD POST POST POST - self.assertEqual(seen_headers, [ + self.assertEqual( + seen_headers, [ {'X-Container-Host': '10.0.0.0:1000,10.0.0.3:1003', 'X-Container-Partition': '1', 'X-Container-Device': 'sda,sdd'}, @@ -4859,8 +5020,9 @@ class TestContainerController(unittest.TestCase): self.assertEquals(res.headers['x-works'], 'yes') if c_expected: self.assertTrue('swift.container/a/c' in res.environ) - self.assertEquals(res.environ['swift.container/a/c']['status'], - c_expected) + self.assertEquals( + res.environ['swift.container/a/c']['status'], + c_expected) else: self.assertTrue('swift.container/a/c' not in res.environ) if a_expected: @@ -4882,8 +5044,9 @@ class TestContainerController(unittest.TestCase): self.assertEquals(res.headers['x-works'], 'yes') if c_expected: self.assertTrue('swift.container/a/c' in res.environ) - self.assertEquals(res.environ['swift.container/a/c']['status'], - c_expected) + self.assertEquals( + res.environ['swift.container/a/c']['status'], + c_expected) else: self.assertTrue('swift.container/a/c' not in res.environ) if a_expected: @@ -4909,7 +5072,8 @@ class TestContainerController(unittest.TestCase): # In all the following tests cache 404 for account # return 404 (as account is not found) and dont cache container test_status_map((404, 404, 404), 404, None, 404) - self.app.account_autocreate = True # This should make no difference + # This should make no difference + self.app.account_autocreate = True test_status_map((404, 404, 404), 404, None, 404) def test_PUT(self): @@ -4934,41 +5098,41 @@ class TestContainerController(unittest.TestCase): self.assertFalse(self.app.account_autocreate) test_status_map((404, 404, 404), 404, missing_container=True) self.app.account_autocreate = True - #fail to retrieve account info + # fail to retrieve account info test_status_map( - (503, 503, 503), # account_info fails on 503 - 404, missing_container=True) + (503, 503, 503), # account_info fails on 503 + 404, missing_container=True) # account fail after creation test_status_map( - (404, 404, 404, # account_info fails on 404 - 201, 201, 201, # PUT account - 404, 404, 404), # account_info fail - 404, missing_container=True) + (404, 404, 404, # account_info fails on 404 + 201, 201, 201, # PUT account + 404, 404, 404), # account_info fail + 404, missing_container=True) test_status_map( - (503, 503, 404, # account_info fails on 404 - 503, 503, 503, # PUT account - 503, 503, 404), # account_info fail - 404, missing_container=True) - #put fails + (503, 503, 404, # account_info fails on 404 + 503, 503, 503, # PUT account + 503, 503, 404), # account_info fail + 404, missing_container=True) + # put fails test_status_map( - (404, 404, 404, # account_info fails on 404 - 201, 201, 201, # PUT account - 200, # account_info success - 503, 503, 201), # put container fail - 503, missing_container=True) + (404, 404, 404, # account_info fails on 404 + 201, 201, 201, # PUT account + 200, # account_info success + 503, 503, 201), # put container fail + 503, missing_container=True) # all goes according to plan test_status_map( - (404, 404, 404, # account_info fails on 404 - 201, 201, 201, # PUT account - 200, # account_info success - 201, 201, 201), # put container success - 201, missing_container=True) + (404, 404, 404, # account_info fails on 404 + 201, 201, 201, # PUT account + 200, # account_info success + 201, 201, 201), # put container success + 201, missing_container=True) test_status_map( - (503, 404, 404, # account_info fails on 404 - 503, 201, 201, # PUT account - 503, 200, # account_info success - 503, 201, 201), # put container success - 201, missing_container=True) + (503, 404, 404, # account_info fails on 404 + 503, 201, 201, # PUT account + 503, 200, # account_info success + 503, 201, 201), # put container success + 201, missing_container=True) def test_POST(self): with save_globals(): @@ -5382,7 +5546,8 @@ class TestContainerController(unittest.TestCase): req = Request.blank('/a/c') self.app.update_request(req) res = controller.GET(req) - self.assertEquals(res.environ['swift.container/a/c']['status'], 204) + self.assertEquals( + res.environ['swift.container/a/c']['status'], 204) self.assertEquals(res.content_length, 0) self.assertTrue('transfer-encoding' not in res.headers) @@ -5586,7 +5751,7 @@ class TestContainerController(unittest.TestCase): self.assertEquals(expected_exposed, exposed) def _gather_x_account_headers(self, controller_call, req, *connect_args, - **kwargs): + **kwargs): seen_headers = [] to_capture = ('X-Account-Partition', 'X-Account-Host', 'X-Account-Device') @@ -5736,9 +5901,10 @@ class TestAccountController(unittest.TestCase): # Access-Control-Request-Method headers) self.app.allow_account_management = False controller = proxy_server.AccountController(self.app, 'account') - req = Request.blank('/account', {'REQUEST_METHOD': 'OPTIONS'}, + req = Request.blank( + '/account', {'REQUEST_METHOD': 'OPTIONS'}, headers={'Origin': 'http://foo.com', - 'Access-Control-Request-Method': 'GET'}) + 'Access-Control-Request-Method': 'GET'}) req.content_length = 0 resp = controller.OPTIONS(req) self.assertEquals(200, resp.status_int) @@ -5753,7 +5919,6 @@ class TestAccountController(unittest.TestCase): req.content_length = 0 resp = controller.OPTIONS(req) self.assertEquals(200, resp.status_int) - print resp.headers['Allow'] for verb in 'OPTIONS GET POST PUT DELETE HEAD'.split(): self.assertTrue( verb in resp.headers['Allow']) @@ -5847,16 +6012,19 @@ class TestAccountController(unittest.TestCase): # first test with autocreate being False self.assertFalse(self.app.account_autocreate) self.assert_status_map(controller.POST, - (404, 404, 404), 404) + (404, 404, 404), 404) # next turn it on and test account being created than updated controller.app.account_autocreate = True - self.assert_status_map(controller.POST, + self.assert_status_map( + controller.POST, (404, 404, 404, 202, 202, 202, 201, 201, 201), 201) # account_info PUT account POST account - self.assert_status_map(controller.POST, - (404, 404, 503, 201, 201, 503, 204, 204, 504), 204) + self.assert_status_map( + controller.POST, + (404, 404, 503, 201, 201, 503, 204, 204, 504), 204) # what if create fails - self.assert_status_map(controller.POST, + self.assert_status_map( + controller.POST, (404, 404, 404, 403, 403, 403, 400, 400, 400), 400) def test_connection_refused(self): @@ -6121,20 +6289,21 @@ class TestAccountControllerFakeGetResponse(unittest.TestCase): have to match the responses for empty accounts that really exist. """ def setUp(self): - self.app = proxy_server.Application(None, FakeMemcache(), + conf = {'account_autocreate': 'yes'} + self.app = proxy_server.Application(conf, FakeMemcache(), account_ring=FakeRing(), container_ring=FakeRing(), object_ring=FakeRing) self.app.memcache = FakeMemcacheReturnsNone() - self.controller = proxy_server.AccountController(self.app, 'acc') - self.controller.app.account_autocreate = True def test_GET_autocreate_accept_json(self): with save_globals(): - set_http_connect(404) # however many backends we ask, they all 404 - req = Request.blank('/a', headers={'Accept': 'application/json'}) - - resp = self.controller.GET(req) + set_http_connect(*([404] * 100)) # nonexistent: all backends 404 + req = Request.blank( + '/v1/a', headers={'Accept': 'application/json'}, + environ={'REQUEST_METHOD': 'GET', + 'PATH_INFO': '/v1/a'}) + resp = req.get_response(self.app) self.assertEqual(200, resp.status_int) self.assertEqual('application/json; charset=utf-8', resp.headers['Content-Type']) @@ -6142,10 +6311,12 @@ class TestAccountControllerFakeGetResponse(unittest.TestCase): def test_GET_autocreate_format_json(self): with save_globals(): - set_http_connect(404) # however many backends we ask, they all 404 - req = Request.blank('/a?format=json') - - resp = self.controller.GET(req) + set_http_connect(*([404] * 100)) # nonexistent: all backends 404 + req = Request.blank('/v1/a?format=json', + environ={'REQUEST_METHOD': 'GET', + 'PATH_INFO': '/v1/a', + 'QUERY_STRING': 'format=json'}) + resp = req.get_response(self.app) self.assertEqual(200, resp.status_int) self.assertEqual('application/json; charset=utf-8', resp.headers['Content-Type']) @@ -6153,30 +6324,54 @@ class TestAccountControllerFakeGetResponse(unittest.TestCase): def test_GET_autocreate_accept_xml(self): with save_globals(): - set_http_connect(404) # however many backends we ask, they all 404 - req = Request.blank('/a', headers={"Accept": "text/xml"}) + set_http_connect(*([404] * 100)) # nonexistent: all backends 404 + req = Request.blank('/v1/a', headers={"Accept": "text/xml"}, + environ={'REQUEST_METHOD': 'GET', + 'PATH_INFO': '/v1/a'}) - resp = self.controller.GET(req) + resp = req.get_response(self.app) self.assertEqual(200, resp.status_int) + self.assertEqual('text/xml; charset=utf-8', resp.headers['Content-Type']) empty_xml_listing = ('\n' - '\n') + '\n') self.assertEqual(empty_xml_listing, resp.body) def test_GET_autocreate_format_xml(self): with save_globals(): - set_http_connect(404) # however many backends we ask, they all 404 - req = Request.blank('/a?format=xml') - - resp = self.controller.GET(req) + set_http_connect(*([404] * 100)) # nonexistent: all backends 404 + req = Request.blank('/v1/a?format=xml', + environ={'REQUEST_METHOD': 'GET', + 'PATH_INFO': '/v1/a', + 'QUERY_STRING': 'format=xml'}) + resp = req.get_response(self.app) self.assertEqual(200, resp.status_int) self.assertEqual('application/xml; charset=utf-8', resp.headers['Content-Type']) empty_xml_listing = ('\n' - '\n') + '\n') self.assertEqual(empty_xml_listing, resp.body) + def test_GET_autocreate_accept_unknown(self): + with save_globals(): + set_http_connect(*([404] * 100)) # nonexistent: all backends 404 + req = Request.blank('/v1/a', headers={"Accept": "mystery/meat"}, + environ={'REQUEST_METHOD': 'GET', + 'PATH_INFO': '/v1/a'}) + resp = req.get_response(self.app) + self.assertEqual(406, resp.status_int) + + def test_GET_autocreate_format_invalid_utf8(self): + with save_globals(): + set_http_connect(*([404] * 100)) # nonexistent: all backends 404 + req = Request.blank('/v1/a?format=\xff\xfe', + environ={'REQUEST_METHOD': 'GET', + 'PATH_INFO': '/v1/a', + 'QUERY_STRING': 'format=\xff\xfe'}) + resp = req.get_response(self.app) + self.assertEqual(400, resp.status_int) + class FakeObjectController(object): @@ -6350,7 +6545,8 @@ class TestSegmentedIterable(unittest.TestCase): segit.ratelimit_index = 0 segit.listing.next() segit._load_next_segment() - self.assertEquals(self.controller.GETorHEAD_base_args[-1][4], '/a/lc/o2') + self.assertEquals( + self.controller.GETorHEAD_base_args[-1][4], '/a/lc/o2') data = ''.join(segit.segment_iter) self.assertEquals(data, '22') @@ -6362,9 +6558,11 @@ class TestSegmentedIterable(unittest.TestCase): segit.listing.next() segit.seek = 1 segit._load_next_segment() - self.assertEquals(self.controller.GETorHEAD_base_args[-1][4], '/a/lc/o2') - self.assertEquals(str(self.controller.GETorHEAD_base_args[-1][0].range), - 'bytes=1-') + self.assertEquals( + self.controller.GETorHEAD_base_args[-1][4], '/a/lc/o2') + self.assertEquals( + str(self.controller.GETorHEAD_base_args[-1][0].range), + 'bytes=1-') data = ''.join(segit.segment_iter) self.assertEquals(data, '2') -- cgit