diff options
| -rwxr-xr-x | .unittests | 5 | ||||
| -rwxr-xr-x | swift/1.4.8/.unittests | 6 | ||||
| -rw-r--r-- | swift/1.4.8/plugins/utils.py | 53 | ||||
| -rw-r--r-- | swift/1.4.8/test/unit/plugins/test_utils.py | 258 |
4 files changed, 281 insertions, 41 deletions
diff --git a/.unittests b/.unittests index 5be925d5829..d13fd45a368 100755 --- a/.unittests +++ b/.unittests @@ -1,6 +1,5 @@ #!/bin/bash -cd swift/1.4.8/test/unit -nosetests --exe --with-coverage --cover-package plugins --cover-erase $@ -rm -f .coverage +cd swift/1.4.8/ +./.unittests cd - diff --git a/swift/1.4.8/.unittests b/swift/1.4.8/.unittests new file mode 100755 index 00000000000..113e5dd1f0f --- /dev/null +++ b/swift/1.4.8/.unittests @@ -0,0 +1,6 @@ +#!/bin/bash + +cd test/unit +nosetests --exe --with-coverage --cover-package plugins --cover-erase $@ +rm -f .coverage +cd - diff --git a/swift/1.4.8/plugins/utils.py b/swift/1.4.8/plugins/utils.py index ef5f9d62f61..289420cb93f 100644 --- a/swift/1.4.8/plugins/utils.py +++ b/swift/1.4.8/plugins/utils.py @@ -98,7 +98,6 @@ def do_makedirs(path): raise return True - def do_listdir(path): try: buf = os.listdir(path) @@ -278,7 +277,6 @@ def dir_empty(path): if not os.path.exists(path): return True - def get_device_from_account(account): if account.startswith(RESELLER_PREFIX): device = account.replace(RESELLER_PREFIX, '', 1) @@ -517,7 +515,6 @@ def get_account_details_from_memcache(acc_path, memcache=None): container_count = memcache.get(strip_obj_storage_path(acc_path)+'_container_count') return container_list, container_count - def get_account_details(acc_path, memcache=None): """ Return container_list and container_count. @@ -527,8 +524,6 @@ def get_account_details(acc_path, memcache=None): else: return get_account_details_from_fs(acc_path, memcache) - - def get_etag(path): etag = None if os.path.exists(path): @@ -613,45 +608,28 @@ def get_account_metadata(acc_path, memcache=None): X_CONTAINER_COUNT: container_count} return _add_timestamp(metadata) -def restore_object(obj_path, metadata): - meta = read_metadata(obj_path) - if meta: - meta.update(metadata) - write_metadata(obj_path, meta) +def restore_metadata(path, metadata): + meta_orig = read_metadata(path) + if meta_orig: + meta_new = meta_orig.copy() + meta_new.update(metadata) else: - write_metadata(obj_path, metadata) - -def restore_container(cont_path, metadata): - meta = read_metadata(cont_path) - if meta: - meta.update(metadata) - write_metadata(cont_path, meta) - else: - write_metadata(cont_path, metadata) - -def restore_account(acc_path, metadata): - meta = read_metadata(acc_path) - if meta: - meta.update(metadata) - write_metadata(acc_path, meta) - else: - write_metadata(acc_path, metadata) + meta_new = metadata + if meta_orig != meta_new: + write_metadata(path, meta_new) + return meta_new def create_object_metadata(obj_path): - meta = get_object_metadata(obj_path) - restore_object(obj_path, meta) - return meta + metadata = get_object_metadata(obj_path) + return restore_metadata(obj_path, metadata) def create_container_metadata(cont_path, memcache=None): - meta = get_container_metadata(cont_path, memcache) - restore_container(cont_path, meta) - return meta + metadata = get_container_metadata(cont_path, memcache) + return restore_metadata(cont_path, metadata) def create_account_metadata(acc_path, memcache=None): - meta = get_account_metadata(acc_path, memcache) - restore_account(acc_path, meta) - return meta - + metadata = get_account_metadata(acc_path, memcache) + return restore_metadata(acc_path, metadata) def check_account_exists(account, fs_object): if account not in get_account_list(fs_object): @@ -666,7 +644,6 @@ def get_account_list(fs_object): account_list = fs_object.get_export_list() return account_list - def get_account_id(account): return RESELLER_PREFIX + md5(account + HASH_PATH_SUFFIX).hexdigest() diff --git a/swift/1.4.8/test/unit/plugins/test_utils.py b/swift/1.4.8/test/unit/plugins/test_utils.py index daecc425cab..944d118926a 100644 --- a/swift/1.4.8/test/unit/plugins/test_utils.py +++ b/swift/1.4.8/test/unit/plugins/test_utils.py @@ -15,10 +15,14 @@ """ Tests for plugins.utils """ +import os import unittest import errno import xattr import cPickle as pickle +import tempfile +import hashlib +from swift.common.utils import normalize_timestamp from collections import defaultdict from plugins import utils @@ -229,3 +233,257 @@ class TestUtils(unittest.TestCase): assert res_d == {} assert _xattr_op_cnt['get'] == 3, "%r" % _xattr_op_cnt assert len(_xattrs.keys()) == 0, "Expected 0 keys, found %d" % len(_xattrs.keys()) + + def test_restore_metadata_none(self): + # No initial metadata + path = "/tmp/foo/i" + res_d = utils.restore_metadata(path, { 'b': 'y' }) + expected_d = { 'b': 'y' } + assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d) + assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt + assert _xattr_op_cnt['set'] == 1, "%r" % _xattr_op_cnt + + def test_restore_metadata(self): + # Initial metadata + path = "/tmp/foo/i" + initial_d = { 'a': 'z' } + xkey = _xkey(path, utils.METADATA_KEY) + _xattrs[xkey] = pickle.dumps(initial_d, utils.PICKLE_PROTOCOL) + res_d = utils.restore_metadata(path, { 'b': 'y' }) + expected_d = { 'a': 'z', 'b': 'y' } + assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d) + assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt + assert _xattr_op_cnt['set'] == 1, "%r" % _xattr_op_cnt + + def test_restore_metadata_nochange(self): + # Initial metadata but no changes + path = "/tmp/foo/i" + initial_d = { 'a': 'z' } + xkey = _xkey(path, utils.METADATA_KEY) + _xattrs[xkey] = pickle.dumps(initial_d, utils.PICKLE_PROTOCOL) + res_d = utils.restore_metadata(path, {}) + expected_d = { 'a': 'z' } + assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d) + assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt + assert _xattr_op_cnt['set'] == 0, "%r" % _xattr_op_cnt + + def test_add_timestamp_empty(self): + orig = {} + res = utils._add_timestamp(orig) + assert res == {} + + def test_add_timestamp_none(self): + orig = { 'a': 1, 'b': 2, 'c': 3 } + exp = { 'a': (1, 0), 'b': (2, 0), 'c': (3, 0) } + res = utils._add_timestamp(orig) + assert res == exp + + def test_add_timestamp_mixed(self): + orig = { 'a': 1, 'b': (2, 1), 'c': 3 } + exp = { 'a': (1, 0), 'b': (2, 1), 'c': (3, 0) } + res = utils._add_timestamp(orig) + assert res == exp + + def test_add_timestamp_all(self): + orig = { 'a': (1, 0), 'b': (2, 1), 'c': (3, 0) } + res = utils._add_timestamp(orig) + assert res == orig + + def test_get_etag_empty(self): + tf = tempfile.NamedTemporaryFile() + hd = utils.get_etag(tf.name) + assert hd == hashlib.md5().hexdigest() + + def test_get_etag(self): + tf = tempfile.NamedTemporaryFile() + tf.file.write('123' * utils.CHUNK_SIZE) + tf.file.flush() + hd = utils.get_etag(tf.name) + tf.file.seek(0) + md5 = hashlib.md5() + while True: + chunk = tf.file.read(utils.CHUNK_SIZE) + if not chunk: + break + md5.update(chunk) + assert hd == md5.hexdigest() + + def test_get_object_metadata_dne(self): + md = utils.get_object_metadata("/tmp/doesNotEx1st") + assert md == {} + + obj_keys = (utils.X_TIMESTAMP, utils.X_CONTENT_TYPE, utils.X_ETAG, + utils.X_CONTENT_LENGTH, utils.X_TYPE, utils.X_OBJECT_TYPE) + + def test_get_object_metadata_file(self): + tf = tempfile.NamedTemporaryFile() + tf.file.write('123'); tf.file.flush() + md = utils.get_object_metadata(tf.name) + for key in self.obj_keys: + assert key in md, "Expected key %s in %r" % (key, md) + assert md[utils.X_TYPE] == utils.OBJECT + assert md[utils.X_OBJECT_TYPE] == utils.FILE + assert md[utils.X_CONTENT_TYPE] == utils.FILE_TYPE + assert md[utils.X_CONTENT_LENGTH] == os.path.getsize(tf.name) + assert md[utils.X_TIMESTAMP] == normalize_timestamp(os.path.getctime(tf.name)) + assert md[utils.X_ETAG] == utils.get_etag(tf.name) + + def test_get_object_metadata_dir(self): + td = tempfile.mkdtemp() + try: + md = utils.get_object_metadata(td) + for key in self.obj_keys: + assert key in md, "Expected key %s in %r" % (key, md) + assert md[utils.X_TYPE] == utils.OBJECT + assert md[utils.X_OBJECT_TYPE] == utils.DIR + assert md[utils.X_CONTENT_TYPE] == utils.DIR_TYPE + assert md[utils.X_CONTENT_LENGTH] == 0 + assert md[utils.X_TIMESTAMP] == normalize_timestamp(os.path.getctime(td)) + assert md[utils.X_ETAG] == hashlib.md5().hexdigest() + finally: + os.rmdir(td) + + def test_create_object_metadata_file(self): + tf = tempfile.NamedTemporaryFile() + tf.file.write('4567'); tf.file.flush() + r_md = utils.create_object_metadata(tf.name) + + xkey = _xkey(tf.name, utils.METADATA_KEY) + assert len(_xattrs.keys()) == 1 + assert xkey in _xattrs + assert _xattr_op_cnt['get'] == 1 + assert _xattr_op_cnt['set'] == 1 + md = pickle.loads(_xattrs[xkey]) + assert r_md == md + + for key in self.obj_keys: + assert key in md, "Expected key %s in %r" % (key, md) + assert md[utils.X_TYPE] == utils.OBJECT + assert md[utils.X_OBJECT_TYPE] == utils.FILE + assert md[utils.X_CONTENT_TYPE] == utils.FILE_TYPE + assert md[utils.X_CONTENT_LENGTH] == os.path.getsize(tf.name) + assert md[utils.X_TIMESTAMP] == normalize_timestamp(os.path.getctime(tf.name)) + assert md[utils.X_ETAG] == utils.get_etag(tf.name) + + def test_create_object_metadata_dir(self): + td = tempfile.mkdtemp() + try: + r_md = utils.create_object_metadata(td) + + xkey = _xkey(td, utils.METADATA_KEY) + assert len(_xattrs.keys()) == 1 + assert xkey in _xattrs + assert _xattr_op_cnt['get'] == 1 + assert _xattr_op_cnt['set'] == 1 + md = pickle.loads(_xattrs[xkey]) + assert r_md == md + + for key in self.obj_keys: + assert key in md, "Expected key %s in %r" % (key, md) + assert md[utils.X_TYPE] == utils.OBJECT + assert md[utils.X_OBJECT_TYPE] == utils.DIR + assert md[utils.X_CONTENT_TYPE] == utils.DIR_TYPE + assert md[utils.X_CONTENT_LENGTH] == 0 + assert md[utils.X_TIMESTAMP] == normalize_timestamp(os.path.getctime(td)) + assert md[utils.X_ETAG] == hashlib.md5().hexdigest() + finally: + os.rmdir(td) + + def test_get_container_metadata(self): + def _mock_get_container_details(path, memcache=None): + o_list = [ 'a', 'b', 'c' ] + o_count = 3 + b_used = 47 + return o_list, o_count, b_used + td = tempfile.mkdtemp() + orig_gcd = utils.get_container_details + utils.get_container_details = _mock_get_container_details + try: + exp_md = { + utils.X_TYPE: (utils.CONTAINER, 0), + utils.X_TIMESTAMP: (normalize_timestamp(os.path.getctime(td)), 0), + utils.X_PUT_TIMESTAMP: (normalize_timestamp(os.path.getmtime(td)), 0), + utils.X_OBJECTS_COUNT: (3, 0), + utils.X_BYTES_USED: (47, 0), + } + md = utils.get_container_metadata(td) + assert md == exp_md + finally: + utils.get_container_details = orig_gcd + os.rmdir(td) + + def test_get_account_metadata(self): + def _mock_get_account_details(path, memcache=None): + c_list = [ '123', 'abc' ] + c_count = 2 + return c_list, c_count + td = tempfile.mkdtemp() + orig_gad = utils.get_account_details + utils.get_account_details = _mock_get_account_details + try: + exp_md = { + utils.X_TYPE: (utils.ACCOUNT, 0), + utils.X_TIMESTAMP: (normalize_timestamp(os.path.getctime(td)), 0), + utils.X_PUT_TIMESTAMP: (normalize_timestamp(os.path.getmtime(td)), 0), + utils.X_OBJECTS_COUNT: (0, 0), + utils.X_BYTES_USED: (0, 0), + utils.X_CONTAINER_COUNT: (2, 0), + } + md = utils.get_account_metadata(td) + assert md == exp_md + finally: + utils.get_account_details = orig_gad + os.rmdir(td) + + cont_keys = [utils.X_TYPE, utils.X_TIMESTAMP, utils.X_PUT_TIMESTAMP, + utils.X_OBJECTS_COUNT, utils.X_BYTES_USED] + + def test_create_container_metadata(self): + td = tempfile.mkdtemp() + try: + r_md = utils.create_container_metadata(td) + + xkey = _xkey(td, utils.METADATA_KEY) + assert len(_xattrs.keys()) == 1 + assert xkey in _xattrs + assert _xattr_op_cnt['get'] == 1 + assert _xattr_op_cnt['set'] == 1 + md = pickle.loads(_xattrs[xkey]) + assert r_md == md + + for key in self.cont_keys: + assert key in md, "Expected key %s in %r" % (key, md) + assert md[utils.X_TYPE] == (utils.CONTAINER, 0) + assert md[utils.X_TIMESTAMP] == (normalize_timestamp(os.path.getctime(td)), 0) + assert md[utils.X_PUT_TIMESTAMP] == (normalize_timestamp(os.path.getmtime(td)), 0) + assert md[utils.X_OBJECTS_COUNT] == (0, 0) + assert md[utils.X_BYTES_USED] == (0, 0) + finally: + os.rmdir(td) + + acct_keys = [val for val in cont_keys] + acct_keys.append(utils.X_CONTAINER_COUNT) + + def test_create_account_metadata(self): + td = tempfile.mkdtemp() + try: + r_md = utils.create_account_metadata(td) + + xkey = _xkey(td, utils.METADATA_KEY) + assert len(_xattrs.keys()) == 1 + assert xkey in _xattrs + assert _xattr_op_cnt['get'] == 1 + assert _xattr_op_cnt['set'] == 1 + md = pickle.loads(_xattrs[xkey]) + assert r_md == md + + for key in self.acct_keys: + assert key in md, "Expected key %s in %r" % (key, md) + assert md[utils.X_TYPE] == (utils.ACCOUNT, 0) + assert md[utils.X_TIMESTAMP] == (normalize_timestamp(os.path.getctime(td)), 0) + assert md[utils.X_PUT_TIMESTAMP] == (normalize_timestamp(os.path.getmtime(td)), 0) + assert md[utils.X_OBJECTS_COUNT] == (0, 0) + assert md[utils.X_BYTES_USED] == (0, 0) + assert md[utils.X_CONTAINER_COUNT] == (0, 0) + finally: + os.rmdir(td) |