diff options
author | Peter Portante <peter.portante@redhat.com> | 2013-05-28 06:29:22 -0400 |
---|---|---|
committer | Luis Pabon <lpabon@redhat.com> | 2013-06-03 14:01:29 -0700 |
commit | a574805398d9bf28be6f357e26a2ba299a5b90e1 (patch) | |
tree | 1207d03d0419d14d99204b8313d95b991d73df8b /test/unit | |
parent | 3c0cdd74ec1d8df3e452a6653f20bad6252b971c (diff) |
Bring initial DiskDir module coverage to 97%.
This is quite ugly. Sorry. We ported a set of test from OpenStack
Swift's test/unit/commont/test_db.py, the testAccountBroker and
testContainerBroker, but because of the divergent nature of the
current attempt UFO (Unified File and Object) implementation, it was
not possible to use the interface calls directly.
Instead, we are using these tests to make sure most of the code paths
are exercised, and to define much of the expected behavior. Further
refactoring, unit tests and functional test work will help us bring
the code base and these tests closer to the originals in upstream
OpenStack Swift (as of Havana).
Change-Id: I095bb03619de6e7e1378b5252913e39b1ea8bf27
Signed-off-by: Peter Portante <peter.portante@redhat.com>
Reviewed-on: http://review.gluster.org/5135
Reviewed-by: Luis Pabon <lpabon@redhat.com>
Tested-by: Luis Pabon <lpabon@redhat.com>
Diffstat (limited to 'test/unit')
-rw-r--r-- | test/unit/common/test_diskdir.py | 715 | ||||
-rw-r--r-- | test/unit/common/test_utils.py | 47 |
2 files changed, 735 insertions, 27 deletions
diff --git a/test/unit/common/test_diskdir.py b/test/unit/common/test_diskdir.py index 1058367..733cc17 100644 --- a/test/unit/common/test_diskdir.py +++ b/test/unit/common/test_diskdir.py @@ -22,6 +22,8 @@ import cPickle as pickle import unittest import shutil import tarfile +import hashlib +from time import time from nose import SkipTest from swift.common.utils import normalize_timestamp from gluster.swift.common import utils @@ -29,12 +31,23 @@ import gluster.swift.common.Glusterfs from test_utils import _initxattr, _destroyxattr, _setxattr, _getxattr from test.unit import FakeLogger -gluster.swift.common.Glusterfs.RUN_DIR = '/tmp/gluster_unit_tests/run' -try: - os.makedirs(gluster.swift.common.Glusterfs.RUN_DIR) -except OSError as e: - if e.errno != errno.EEXIST: - raise +def setup(): + global _saved_RUN_DIR, _saved_do_getsize + _saved_do_getsize = gluster.swift.common.Glusterfs._do_getsize + gluster.swift.common.Glusterfs._do_getsize = True + _saved_RUN_DIR = gluster.swift.common.Glusterfs.RUN_DIR + gluster.swift.common.Glusterfs.RUN_DIR = '/tmp/gluster_unit_tests/run' + try: + os.makedirs(gluster.swift.common.Glusterfs.RUN_DIR) + except OSError as e: + if e.errno != errno.EEXIST: + raise + +def teardown(): + shutil.rmtree(gluster.swift.common.Glusterfs.RUN_DIR) + gluster.swift.common.Glusterfs.RUN_DIR = _saved_RUN_DIR + gluster.swift.common.Glusterfs._do_getsize = _saved_do_getsize + import gluster.swift.common.DiskDir as dd @@ -243,6 +256,7 @@ class TestDiskDirModuleFunctions(unittest.TestCase): l = list(dd.filter_limit([1,2,3], 4)) assert l == [1,2,3] + class TestDiskCommon(unittest.TestCase): """ Tests for gluster.swift.common.DiskDir.DiskCommon """ @@ -393,6 +407,678 @@ class TestDiskDir(unittest.TestCase): self.fail("Implement me") +class TestContainerBroker(unittest.TestCase): + """ + Tests for DiskDir.DiskDir class (duck-typed + swift.common.db.ContainerBroker). + """ + + def __init__(self, *args, **kwargs): + super(TestContainerBroker, self).__init__(*args, **kwargs) + self.initial_ts = normalize_timestamp('1') + + def setUp(self): + _initxattr() + self.path = tempfile.mkdtemp() + self.drive = 'drv' + self.container = None + + def tearDown(self): + self.container = None + _destroyxattr() + shutil.rmtree(self.path) + + def _get_broker(self, account=None, container=None): + assert account is not None + assert container is not None + self.container = os.path.join(self.path, self.drive, container) + return dd.DiskDir(self.path, self.drive, account=account, + container=container, logger=FakeLogger()) + + def _create_file(self, p): + fullname = os.path.join(self.container, p) + dirs = os.path.dirname(fullname) + try: + os.makedirs(dirs) + except OSError as e: + if e.errno != errno.EEXIST: + raise + with open(fullname, 'w') as fp: + fp.write("file path: %s\n" % fullname) + return fullname + + def test_creation(self): + # Test swift.common.db.ContainerBroker.__init__ + broker = self._get_broker(account='a', container='c') + self.assertEqual(broker.db_file, dd._db_file) + self.assertEqual(os.path.basename(broker.db_file), 'db_file.db') + broker.initialize(self.initial_ts) + self.assertTrue(os.path.isdir(self.container)) + self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP]) + self.assertFalse(broker.is_deleted()) + + def test_creation_existing(self): + # Test swift.common.db.ContainerBroker.__init__ + os.makedirs(os.path.join(self.path, self.drive, 'c')) + broker = self._get_broker(account='a', container='c') + self.assertEqual(broker.db_file, dd._db_file) + self.assertEqual(os.path.basename(broker.db_file), 'db_file.db') + broker.initialize(self.initial_ts) + self.assertTrue(os.path.isdir(self.container)) + self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP]) + self.assertFalse(broker.is_deleted()) + + def test_creation_existing_bad_metadata(self): + # Test swift.common.db.ContainerBroker.__init__ + container = os.path.join(self.path, self.drive, 'c') + os.makedirs(container) + utils.write_metadata(container, dict(a=1, b=2)) + broker = self._get_broker(account='a', container='c') + self.assertEqual(broker.db_file, dd._db_file) + self.assertEqual(os.path.basename(broker.db_file), 'db_file.db') + broker.initialize(self.initial_ts) + self.assertTrue(os.path.isdir(self.container)) + self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP]) + self.assertFalse(broker.is_deleted()) + + def test_empty(self): + # Test swift.common.db.ContainerBroker.empty + broker = self._get_broker(account='a', container='c') + broker.initialize(self.initial_ts) + self.assert_(broker.empty()) + obj = self._create_file('o.txt') + self.assert_(not broker.empty()) + os.unlink(obj) + self.assert_(broker.empty()) + + def test_put_object(self): + broker = self._get_broker(account='a', container='c') + broker.initialize(self.initial_ts) + self.assert_(broker.empty()) + broker.put_object('o', normalize_timestamp(time()), 0, 'text/plain', + 'd41d8cd98f00b204e9800998ecf8427e') + # put_object() should be a NOOP + self.assert_(broker.empty()) + + def test_delete_object(self): + broker = self._get_broker(account='a', container='c') + broker.initialize(self.initial_ts) + self.assert_(broker.empty()) + obj = self._create_file('o.txt') + self.assert_(not broker.empty()) + broker.delete_object('o', normalize_timestamp(time())) + # delete_object() should be a NOOP + self.assert_(not broker.empty()) + os.unlink(obj) + self.assert_(broker.empty()) + + def test_get_info(self): + # Test swift.common.db.ContainerBroker.get_info + broker = self._get_broker(account='test1', + container='test2') + broker.initialize(normalize_timestamp('1')) + + info = broker.get_info() + self.assertEquals(info['account'], 'test1') + self.assertEquals(info['container'], 'test2') + + info = broker.get_info() + self.assertEquals(info['object_count'], 0) + self.assertEquals(info['bytes_used'], 0) + + obj1 = os.path.join(self.container, 'o1') + with open(obj1, 'w') as fp: + fp.write("%s\n" % ('x' * 122)) + info = broker.get_info() + self.assertEquals(info['object_count'], 1) + self.assertEquals(info['bytes_used'], 123) + + obj2 = os.path.join(self.container, 'o2') + with open(obj2, 'w') as fp: + fp.write("%s\n" % ('x' * 122)) + info = broker.get_info() + self.assertEquals(info['object_count'], 2) + self.assertEquals(info['bytes_used'], 246) + + with open(obj2, 'w') as fp: + fp.write("%s\n" % ('x' * 999)) + info = broker.get_info() + self.assertEquals(info['object_count'], 2) + self.assertEquals(info['bytes_used'], 1123) + + os.unlink(obj1) + info = broker.get_info() + self.assertEquals(info['object_count'], 1) + self.assertEquals(info['bytes_used'], 1000) + + os.unlink(obj2) + info = broker.get_info() + self.assertEquals(info['object_count'], 0) + self.assertEquals(info['bytes_used'], 0) + + info = broker.get_info() + self.assertEquals(info['x_container_sync_point1'], -1) + self.assertEquals(info['x_container_sync_point2'], -1) + + def test_set_x_syncs(self): + broker = self._get_broker(account='test1', + container='test2') + broker.initialize(normalize_timestamp('1')) + + info = broker.get_info() + self.assertEquals(info['x_container_sync_point1'], -1) + self.assertEquals(info['x_container_sync_point2'], -1) + + broker.set_x_container_sync_points(1, 2) + info = broker.get_info() + self.assertEquals(info['x_container_sync_point1'], 1) + self.assertEquals(info['x_container_sync_point2'], 2) + + def test_list_objects_iter(self): + # Test swift.common.db.ContainerBroker.list_objects_iter + broker = self._get_broker(account='a', container='c') + broker.initialize(normalize_timestamp('1')) + + for obj1 in xrange(4): + for obj2 in xrange(125): + self._create_file('%d.d/%04d' % (obj1, obj2)) + for obj in xrange(125): + self._create_file('2.d/0051.d/%04d' % obj) + for obj in xrange(125): + self._create_file('3.d/%04d.d/0049' % obj) + + listing = broker.list_objects_iter(100, '', None, None, '') + self.assertEquals(len(listing), 100) + self.assertEquals(listing[0][0], '0.d/0000') + self.assertEquals(listing[-1][0], '0.d/0099') + + listing = broker.list_objects_iter(100, '', '0.d/0050', None, '') + self.assertEquals(len(listing), 50) + self.assertEquals(listing[0][0], '0.d/0000') + self.assertEquals(listing[-1][0], '0.d/0049') + + listing = broker.list_objects_iter(100, '0.d/0099', None, None, '') + self.assertEquals(len(listing), 100) + self.assertEquals(listing[0][0], '0.d/0100') + self.assertEquals(listing[-1][0], '1.d/0074') + + listing = broker.list_objects_iter(55, '1.d/0074', None, None, '') + self.assertEquals(len(listing), 55) + self.assertEquals(listing[0][0], '1.d/0075') + self.assertEquals(listing[-1][0], '2.d/0004') + + listing = broker.list_objects_iter(10, '', None, '0.d/01', '') + self.assertEquals(len(listing), 10) + self.assertEquals(listing[0][0], '0.d/0100') + self.assertEquals(listing[-1][0], '0.d/0109') + + listing = broker.list_objects_iter(10, '', None, '0.d/', '/') + self.assertEquals(len(listing), 10) + self.assertEquals(listing[0][0], '0.d/0000') + self.assertEquals(listing[-1][0], '0.d/0009') + + listing = broker.list_objects_iter(10, '', None, None, '', '0.d') + self.assertEquals(len(listing), 10) + self.assertEquals(listing[0][0], '0.d/0000') + self.assertEquals(listing[-1][0], '0.d/0009') + + listing = broker.list_objects_iter(10, '', None, '', '/') + self.assertEquals(len(listing), 0) + + listing = broker.list_objects_iter(10, '2', None, None, '/') + self.assertEquals(len(listing), 0) + + listing = broker.list_objects_iter(10, '2.d/', None, None, '/') + self.assertEquals(len(listing), 0) + + listing = broker.list_objects_iter(10, '2.d/0050', None, '2.d/', '/') + self.assertEquals(len(listing), 9) + self.assertEquals(listing[0][0], '2.d/0051') + self.assertEquals(listing[1][0], '2.d/0052') + self.assertEquals(listing[-1][0], '2.d/0059') + + listing = broker.list_objects_iter(10, '3.d/0045', None, '3.d/', '/') + self.assertEquals(len(listing), 5) + self.assertEquals([row[0] for row in listing], + ['3.d/0046', '3.d/0047', + '3.d/0048', '3.d/0049', + '3.d/0050']) + + # FIXME + #broker.put_object('3/0049/', normalize_timestamp(time()), 0, + # 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + #listing = broker.list_objects_iter(10, '3/0048', None, None, None) + #self.assertEquals(len(listing), 10) + #self.assertEquals([row[0] for row in listing], + # ['3.d/0048.d/0049', '3.d/0049', '3.d/0049.d/', + # '3.d/0049.d/0049', '3.d/0050', '3.d/0050.d/0049', '3.d/0051', '3.d/0051.d/0049', + # '3.d/0052', '3.d/0052.d/0049']) + + listing = broker.list_objects_iter(10, '3.d/0048', None, '3.d/', '/') + self.assertEquals(len(listing), 5) + self.assertEquals([row[0] for row in listing], + ['3.d/0049', '3.d/0050', + '3.d/0051', '3.d/0052', '3.d/0053']) + + listing = broker.list_objects_iter(10, None, None, '3.d/0049.d/', '/') + self.assertEquals(len(listing), 1) + self.assertEquals([row[0] for row in listing], + ['3.d/0049.d/0049']) + + # FIXME + #listing = broker.list_objects_iter(10, None, None, None, None, + # '3.d/0049') + #self.assertEquals(len(listing), 1) + #self.assertEquals([row[0] for row in listing], ['3.d/0049.d/0049']) + + listing = broker.list_objects_iter(2, None, None, '3.d/', '/') + self.assertEquals(len(listing), 1) + self.assertEquals([row[0] for row in listing], ['3.d/0000']) + + # FIXME + #listing = broker.list_objects_iter(2, None, None, None, None, '3') + #self.assertEquals(len(listing), 2) + #self.assertEquals([row[0] for row in listing], ['3.d/0000', '3.d/0001']) + + def test_list_objects_iter_prefix_delim(self): + # Test swift.common.db.ContainerBroker.list_objects_iter + broker = self._get_broker(account='a', container='c') + broker.initialize(normalize_timestamp('1')) + + os.mkdir(os.path.join(self.container, 'pets')) + os.mkdir(os.path.join(self.container, 'pets', 'dogs')) + obj1 = os.path.join(self.container, 'pets', 'dogs', '1') + with open(obj1, 'w') as fp: + fp.write("one\n") + obj2 = os.path.join(self.container, 'pets', 'dogs', '2') + with open(obj2, 'w') as fp: + fp.write("two\n") + os.mkdir(os.path.join(self.container, 'pets', 'fish')) + obja = os.path.join(self.container, 'pets', 'fish', 'a') + with open(obja, 'w') as fp: + fp.write("A\n") + objb = os.path.join(self.container, 'pets', 'fish', 'b') + with open(objb, 'w') as fp: + fp.write("B\n") + objf = os.path.join(self.container, 'pets', 'fish_info.txt') + with open(objf, 'w') as fp: + fp.write("one fish\n") + objs = os.path.join(self.container, 'snakes') + with open(objs, 'w') as fp: + fp.write("slither\n") + + listing = broker.list_objects_iter(100, None, None, 'pets/f', '/') + self.assertEquals([row[0] for row in listing], + ['pets/fish_info.txt']) + listing = broker.list_objects_iter(100, None, None, 'pets/fish', '/') + self.assertEquals([row[0] for row in listing], + ['pets/fish_info.txt']) + listing = broker.list_objects_iter(100, None, None, 'pets/fish/', '/') + self.assertEquals([row[0] for row in listing], + ['pets/fish/a', 'pets/fish/b']) + + def test_double_check_trailing_delimiter(self): + # Test swift.common.db.ContainerBroker.list_objects_iter for a + # container that has an odd file with a trailing delimiter + broker = self._get_broker(account='a', container='c') + broker.initialize(normalize_timestamp('1')) + + self._create_file('a') + self._create_file('a.d/a') + self._create_file('a.d/a.d/a') + self._create_file('a.d/a.d/b') + self._create_file('a.d/b') + self._create_file('b') + self._create_file('b.d/a') + self._create_file('b.d/b') + self._create_file('c') + self._create_file('a.d/0') + self._create_file('0') + self._create_file('00') + self._create_file('0.d/0') + self._create_file('0.d/00') + self._create_file('0.d/1') + self._create_file('0.d/1.d/0') + self._create_file('1') + self._create_file('1.d/0') + + listing = broker.list_objects_iter(25, None, None, None, None) + self.assertEquals(len(listing), 18) + self.assertEquals([row[0] for row in listing], + ['0', '0.d/0', '0.d/00', '0.d/1', '0.d/1.d/0', '00', + '1', '1.d/0', 'a', 'a.d/0', 'a.d/a', 'a.d/a.d/a', + 'a.d/a.d/b', 'a.d/b', 'b', 'b.d/a', 'b.d/b', 'c']) + listing = broker.list_objects_iter(25, None, None, '', '/') + self.assertEquals(len(listing), 6) + self.assertEquals([row[0] for row in listing], + ['0', '00', '1', 'a', 'b', 'c']) + listing = broker.list_objects_iter(25, None, None, 'a.d/', '/') + self.assertEquals(len(listing), 3) + self.assertEquals([row[0] for row in listing], + ['a.d/0', 'a.d/a', 'a.d/b']) + listing = broker.list_objects_iter(25, None, None, '0.d/', '/') + self.assertEquals(len(listing), 3) + self.assertEquals([row[0] for row in listing], + ['0.d/0', '0.d/00', '0.d/1']) + listing = broker.list_objects_iter(25, None, None, '0.d/1.d/', '/') + self.assertEquals(len(listing), 1) + self.assertEquals([row[0] for row in listing], ['0.d/1.d/0']) + listing = broker.list_objects_iter(25, None, None, 'b.d/', '/') + self.assertEquals(len(listing), 2) + self.assertEquals([row[0] for row in listing], ['b.d/a', 'b.d/b']) + + def test_metadata(self): + # Initializes a good broker for us + broker = self._get_broker(account='a', container='c') + broker.initialize(normalize_timestamp('1')) + + # Add our first item + first_timestamp = normalize_timestamp(1) + first_value = '1' + broker.update_metadata({'First': [first_value, first_timestamp]}) + self.assert_('First' in broker.metadata) + self.assertEquals(broker.metadata['First'], + [first_value, first_timestamp]) + # Add our second item + second_timestamp = normalize_timestamp(2) + second_value = '2' + broker.update_metadata({'Second': [second_value, second_timestamp]}) + self.assert_('First' in broker.metadata) + self.assertEquals(broker.metadata['First'], + [first_value, first_timestamp]) + self.assert_('Second' in broker.metadata) + self.assertEquals(broker.metadata['Second'], + [second_value, second_timestamp]) + # Update our first item + first_timestamp = normalize_timestamp(3) + first_value = '1b' + broker.update_metadata({'First': [first_value, first_timestamp]}) + self.assert_('First' in broker.metadata) + self.assertEquals(broker.metadata['First'], + [first_value, first_timestamp]) + self.assert_('Second' in broker.metadata) + self.assertEquals(broker.metadata['Second'], + [second_value, second_timestamp]) + # Delete our second item (by setting to empty string) + second_timestamp = normalize_timestamp(4) + second_value = '' + broker.update_metadata({'Second': [second_value, second_timestamp]}) + self.assert_('First' in broker.metadata) + self.assertEquals(broker.metadata['First'], + [first_value, first_timestamp]) + self.assert_('Second' in broker.metadata) + self.assertEquals(broker.metadata['Second'], + [second_value, second_timestamp]) + + def test_delete_db(self): + broker = self._get_broker(account='a', container='c') + self.assertEqual(broker.db_file, dd._db_file) + self.assertEqual(os.path.basename(broker.db_file), 'db_file.db') + broker.initialize(self.initial_ts) + self.assertTrue(os.path.isdir(self.container)) + self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP]) + self.assertFalse(broker.is_deleted()) + broker.delete_db(normalize_timestamp(time())) + self.assertTrue(broker.is_deleted()) + + +class TestAccountBroker(unittest.TestCase): + """ + Tests for DiskDir.DiskAccount class (duck-typed + swift.common.db.AccountBroker). + """ + + def __init__(self, *args, **kwargs): + super(TestAccountBroker, self).__init__(*args, **kwargs) + self.initial_ts = normalize_timestamp('1') + + def setUp(self): + _initxattr() + self.path = tempfile.mkdtemp() + self.drive = 'drv' + self.drive_fullpath = os.path.join(self.path, self.drive) + os.mkdir(self.drive_fullpath) + self.account = None + + def tearDown(self): + self.account = None + _destroyxattr() + shutil.rmtree(self.path) + + def _get_broker(self, account=None): + assert account is not None + self.account = account + return dd.DiskAccount(self.path, self.drive, account=account, + logger=FakeLogger()) + + def _create_container(self, name): + cont = os.path.join(self.drive_fullpath, name) + try: + os.mkdir(cont) + except OSError as e: + if e.errno != errno.EEXIST: + raise + return cont + + def test_creation(self): + # Test swift.common.db.AccountBroker.__init__ + broker = self._get_broker(account='a') + self.assertEqual(broker.db_file, dd._db_file) + self.assertEqual(os.path.basename(broker.db_file), 'db_file.db') + broker.initialize(self.initial_ts) + self.assertTrue(os.path.isdir(self.drive_fullpath)) + self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP]) + self.assertFalse(broker.is_deleted()) + + def test_creation_bad_metadata(self): + # Test swift.common.db.AccountBroker.__init__ + utils.write_metadata(self.drive_fullpath, dict(a=1, b=2)) + broker = self._get_broker(account='a') + self.assertEqual(broker.db_file, dd._db_file) + self.assertEqual(os.path.basename(broker.db_file), 'db_file.db') + broker.initialize(self.initial_ts) + self.assertTrue(os.path.isdir(self.drive_fullpath)) + self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP]) + self.assertFalse(broker.is_deleted()) + + def test_empty(self): + # Test swift.common.db.AccountBroker.empty + broker = self._get_broker(account='a') + broker.initialize(self.initial_ts) + self.assert_(broker.empty()) + c1 = self._create_container('c1') + self.assert_(not broker.empty()) + os.rmdir(c1) + self.assert_(broker.empty()) + + def test_put_container(self): + broker = self._get_broker(account='a') + broker.initialize(self.initial_ts) + self.assert_(broker.empty()) + broker.put_container('c1', normalize_timestamp(time()), 0, 0, 0) + # put_container() should be a NOOP + self.assert_(broker.empty()) + + def test_put_container_for_deletes(self): + broker = self._get_broker(account='a') + broker.initialize(self.initial_ts) + self.assert_(broker.empty()) + c1 = self._create_container('c1') + self.assert_(not broker.empty()) + broker.put_container('c1', 0, normalize_timestamp(time()), 0, 0) + # put_container() should be a NOOP + self.assert_(not broker.empty()) + os.rmdir(c1) + self.assert_(broker.empty()) + + def test_get_info(self): + # Test swift.common.db.AccountBroker.get_info + broker = self._get_broker(account='test1') + broker.initialize(normalize_timestamp('1')) + + info = broker.get_info() + self.assertEquals(info['account'], 'test1') + + info = broker.get_info() + self.assertEquals(info['container_count'], 0) + + c1 = self._create_container('c1') + info = broker.get_info() + self.assertEquals(info['container_count'], 1) + + c2 = self._create_container('c2') + info = broker.get_info() + self.assertEquals(info['container_count'], 2) + + c2 = self._create_container('c2') + info = broker.get_info() + self.assertEquals(info['container_count'], 2) + + os.rmdir(c1) + info = broker.get_info() + self.assertEquals(info['container_count'], 1) + + os.rmdir(c2) + info = broker.get_info() + self.assertEquals(info['container_count'], 0) + + def test_list_containers_iter(self): + # Test swift.common.db.AccountBroker.list_containers_iter + broker = self._get_broker(account='a') + broker.initialize(normalize_timestamp('1')) + for cont1 in xrange(4): + for cont2 in xrange(125): + self._create_container('%d-%04d' % (cont1, cont2)) + for cont in xrange(125): + self._create_container('2-0051-%04d' % cont) + for cont in xrange(125): + self._create_container('3-%04d-0049' % cont) + + listing = broker.list_containers_iter(100, '', None, None, '') + self.assertEquals(len(listing), 100) + self.assertEquals(listing[0][0], '0-0000') + self.assertEquals(listing[-1][0], '0-0099') + + listing = broker.list_containers_iter(100, '', '0-0050', None, '') + self.assertEquals(len(listing), 50) + self.assertEquals(listing[0][0], '0-0000') + self.assertEquals(listing[-1][0], '0-0049') + + listing = broker.list_containers_iter(100, '0-0099', None, None, '') + self.assertEquals(len(listing), 100) + self.assertEquals(listing[0][0], '0-0100') + self.assertEquals(listing[-1][0], '1-0074') + + listing = broker.list_containers_iter(55, '1-0074', None, None, '') + self.assertEquals(len(listing), 55) + self.assertEquals(listing[0][0], '1-0075') + self.assertEquals(listing[-1][0], '2-0004') + + listing = broker.list_containers_iter(10, '', None, '0-01', '') + self.assertEquals(len(listing), 10) + self.assertEquals(listing[0][0], '0-0100') + self.assertEquals(listing[-1][0], '0-0109') + + listing = broker.list_containers_iter(10, '', None, '0-01', '-') + self.assertEquals(len(listing), 10) + self.assertEquals(listing[0][0], '0-0100') + self.assertEquals(listing[-1][0], '0-0109') + + listing = broker.list_containers_iter(10, '', None, '0-', '-') + self.assertEquals(len(listing), 10) + self.assertEquals(listing[0][0], '0-0000') + self.assertEquals(listing[-1][0], '0-0009') + + listing = broker.list_containers_iter(10, '', None, '', '-') + self.assertEquals(len(listing), 4) + self.assertEquals([row[0] for row in listing], + ['0', '1', '2', '3']) + + listing = broker.list_containers_iter(10, '2-', None, None, '-') + self.assertEquals(len(listing), 1) + self.assertEquals([row[0] for row in listing], ['3']) + + listing = broker.list_containers_iter(10, '', None, '2', '-') + self.assertEquals(len(listing), 1) + self.assertEquals([row[0] for row in listing], ['2']) + + listing = broker.list_containers_iter(10, '2-0050', None, '2-', '-') + self.assertEquals(len(listing), 10) + self.assertEquals(listing[0][0], '2-0051') + self.assertEquals(listing[1][0], '2-0052') + self.assertEquals(listing[-1][0], '2-0060') + + listing = broker.list_containers_iter(10, '3-0045', None, '3-', '-') + self.assertEquals(len(listing), 10) + self.assertEquals([row[0] for row in listing], + ['3-0046', '3-0047', '3-0048', '3-0049', '3-0050', + '3-0051', '3-0052', '3-0053', '3-0054', '3-0055']) + + self._create_container('3-0049-') + listing = broker.list_containers_iter(10, '3-0048', None, None, None) + self.assertEquals(len(listing), 10) + self.assertEquals([row[0] for row in listing], + ['3-0048-0049', '3-0049', '3-0049-', '3-0049-0049', + '3-0050', '3-0050-0049', '3-0051', '3-0051-0049', + '3-0052', '3-0052-0049']) + + listing = broker.list_containers_iter(10, '3-0048', None, '3-', '-') + self.assertEquals(len(listing), 10) + self.assertEquals([row[0] for row in listing], + ['3-0049', '3-0050', '3-0051', '3-0052', '3-0053', + '3-0054', '3-0055', '3-0056', '3-0057', '3-0058']) + + listing = broker.list_containers_iter(10, None, None, '3-0049-', '-') + self.assertEquals(len(listing), 2) + self.assertEquals([row[0] for row in listing], + ['3-0049-', '3-0049-0049']) + + def test_double_check_trailing_delimiter(self): + # Test swift.common.db.AccountBroker.list_containers_iter for an + # account that has an odd file with a trailing delimiter + broker = self._get_broker(account='a') + broker.initialize(normalize_timestamp('1')) + self._create_container('a') + self._create_container('a-') + self._create_container('a-a') + self._create_container('a-a-a') + self._create_container('a-a-b') + self._create_container('a-b') + self._create_container('b') + self._create_container('b-a') + self._create_container('b-b') + self._create_container('c') + listing = broker.list_containers_iter(15, None, None, None, None) + self.assertEquals(len(listing), 10) + self.assertEquals([row[0] for row in listing], + ['a', 'a-', 'a-a', 'a-a-a', 'a-a-b', 'a-b', 'b', + 'b-a', 'b-b', 'c']) + listing = broker.list_containers_iter(15, None, None, '', '-') + self.assertEquals(len(listing), 3) + self.assertEquals([row[0] for row in listing], + ['a', 'b', 'c']) + listing = broker.list_containers_iter(15, None, None, 'a-', '-') + self.assertEquals(len(listing), 3) + self.assertEquals([row[0] for row in listing], + ['a-', 'a-a', 'a-b']) + listing = broker.list_containers_iter(15, None, None, 'b-', '-') + self.assertEquals(len(listing), 2) + self.assertEquals([row[0] for row in listing], ['b-a', 'b-b']) + + def test_delete_db(self): + broker = self._get_broker(account='a') + broker.initialize(normalize_timestamp('1')) + self.assertEqual(broker.db_file, dd._db_file) + self.assertEqual(os.path.basename(broker.db_file), 'db_file.db') + broker.initialize(self.initial_ts) + self.assertTrue(os.path.isdir(self.drive_fullpath)) + self.assertEquals(self.initial_ts, broker.metadata[utils.X_TIMESTAMP]) + self.assertFalse(broker.is_deleted()) + broker.delete_db(normalize_timestamp(time())) + # Deleting the "db" should be a NOOP + self.assertFalse(broker.is_deleted()) + + class TestDiskAccount(unittest.TestCase): """ Tests for gluster.swift.common.DiskDir.DiskAccount """ @@ -524,23 +1210,6 @@ class TestDiskAccount(unittest.TestCase): assert data['hash'] == '' assert data['id'] == '' - def test_get_container_timestamp(self): - tf = tarfile.open("common/data/account_tree.tar.bz2", "r:bz2") - orig_cwd = os.getcwd() - datadir = os.path.join(self.td, self.fake_drives[0]) - os.chdir(datadir) - try: - tf.extractall() - finally: - os.chdir(orig_cwd) - md = dd.create_container_metadata(os.path.join(datadir, 'c2')) - assert 'X-PUT-Timestamp' in md, repr(md) - da = dd.DiskAccount(self.td, self.fake_drives[0], - self.fake_accounts[0], self.fake_logger) - raise SkipTest - cts = da.get_container_timestamp('c2') - assert md['X-PUT-Timestamp'][0] == cts, repr(cts) - def test_update_put_timestamp_not_updated(self): da = dd.DiskAccount(self.td, self.fake_drives[0], self.fake_accounts[0], self.fake_logger) diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 0f6dba3..06f1a46 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -38,9 +38,11 @@ _xattr_set_err = {} _xattr_get_err = {} _xattr_rem_err = {} + def _xkey(path, key): return "%s:%s" % (path, key) + def _setxattr(path, key, value, *args, **kwargs): _xattr_op_cnt['set'] += 1 xkey = _xkey(path, key) @@ -51,6 +53,7 @@ def _setxattr(path, key, value, *args, **kwargs): global _xattrs _xattrs[xkey] = value + def _getxattr(path, key, *args, **kwargs): _xattr_op_cnt['get'] += 1 xkey = _xkey(path, key) @@ -67,6 +70,7 @@ def _getxattr(path, key, *args, **kwargs): raise e return ret_val + def _removexattr(path, key, *args, **kwargs): _xattr_op_cnt['remove'] += 1 xkey = _xkey(path, key) @@ -82,6 +86,7 @@ def _removexattr(path, key, *args, **kwargs): e.errno = errno.ENODATA raise e + def _initxattr(): global _xattrs _xattrs = {} @@ -102,6 +107,7 @@ def _initxattr(): xattr.getxattr = _getxattr xattr.removexattr = _removexattr + def _destroyxattr(): # Restore the current methods just in case global _xattr_set; xattr.setxattr = _xattr_set @@ -789,6 +795,40 @@ class TestUtils(unittest.TestCase): cd = utils._get_container_details_from_fs(td) assert cd.bytes_used == 0, repr(cd.bytes_used) + # Should not include the directories + assert cd.object_count == 5, repr(cd.object_count) + assert set(cd.obj_list) == set(['file1', 'file3', 'file2', + 'dir1/file1', 'dir1/file2' + ]), repr(cd.obj_list) + + full_dir1 = os.path.join(td, 'dir1') + full_dir2 = os.path.join(td, 'dir2') + full_dir3 = os.path.join(td, 'dir3') + exp_dir_dict = { td: os.path.getmtime(td), + full_dir1: os.path.getmtime(full_dir1), + full_dir2: os.path.getmtime(full_dir2), + full_dir3: os.path.getmtime(full_dir3), + } + for d,m in cd.dir_list: + assert d in exp_dir_dict + assert exp_dir_dict[d] == m + finally: + os.chdir(orig_cwd) + shutil.rmtree(td) + + def test_get_container_details_from_fs_ufo(self): + orig_cwd = os.getcwd() + __obj_only = Glusterfs.OBJECT_ONLY + td = tempfile.mkdtemp() + try: + tf = tarfile.open("common/data/container_tree.tar.bz2", "r:bz2") + os.chdir(td) + tf.extractall() + + Glusterfs.OBJECT_ONLY = False + + cd = utils._get_container_details_from_fs(td) + assert cd.bytes_used == 0, repr(cd.bytes_used) assert cd.object_count == 8, repr(cd.object_count) assert set(cd.obj_list) == set(['file1', 'file3', 'file2', 'dir3', 'dir1', 'dir2', @@ -809,24 +849,23 @@ class TestUtils(unittest.TestCase): finally: os.chdir(orig_cwd) shutil.rmtree(td) - + Glusterfs.OBJECT_ONLY = __obj_only def test_get_container_details_from_fs_do_getsize_true(self): orig_cwd = os.getcwd() + __do_getsize = Glusterfs._do_getsize td = tempfile.mkdtemp() try: tf = tarfile.open("common/data/container_tree.tar.bz2", "r:bz2") os.chdir(td) tf.extractall() - __do_getsize = Glusterfs._do_getsize Glusterfs._do_getsize = True cd = utils._get_container_details_from_fs(td) assert cd.bytes_used == 30, repr(cd.bytes_used) - assert cd.object_count == 8, repr(cd.object_count) + assert cd.object_count == 5, repr(cd.object_count) assert set(cd.obj_list) == set(['file1', 'file3', 'file2', - 'dir3', 'dir1', 'dir2', 'dir1/file1', 'dir1/file2' ]), repr(cd.obj_list) |