diff options
Diffstat (limited to 'ufo')
| -rw-r--r-- | ufo/gluster/swift/common/DiskFile.py | 130 | ||||
| -rw-r--r-- | ufo/test/unit/common/test_diskfile.py | 140 |
2 files changed, 138 insertions, 132 deletions
diff --git a/ufo/gluster/swift/common/DiskFile.py b/ufo/gluster/swift/common/DiskFile.py index e3f00a01ad7..26abbbef6ed 100644 --- a/ufo/gluster/swift/common/DiskFile.py +++ b/ufo/gluster/swift/common/DiskFile.py @@ -32,9 +32,7 @@ import logging from swift.obj.server import DiskFile -DATADIR = 'objects' -ASYNCDIR = 'async_pending' -KEEP_CACHE_SIZE = (5 * 1024 * 1024) +DEFAULT_DISK_CHUNK_SIZE = 65536 # keep these lower-case DISALLOWED_HEADERS = set('content-length content-type deleted etag'.split()) @@ -53,6 +51,7 @@ class Gluster_DiskFile(DiskFile): :param account: account name for the object :param container: container name for the object :param obj: object name for the object + :param logger: logger object for writing out log file messages :param keep_data_fp: if True, don't close the fp, otherwise close it :param disk_chunk_Size: size of chunks on file reads :param uid: user ID disk object should assume (file or directory) @@ -60,59 +59,64 @@ class Gluster_DiskFile(DiskFile): """ def __init__(self, path, device, partition, account, container, obj, - logger, keep_data_fp=False, disk_chunk_size=65536, - uid=DEFAULT_UID, gid=DEFAULT_GID): + logger, keep_data_fp=False, + disk_chunk_size=DEFAULT_DISK_CHUNK_SIZE, + uid=DEFAULT_UID, gid=DEFAULT_GID, iter_hook=None): self.disk_chunk_size = disk_chunk_size - #Don't support obj_name ending/begining with '/', like /a, a/, /a/b/ etc - obj = obj.strip('/') - if '/' in obj: - self.obj_path, self.obj = obj.rsplit('/', 1) + self.iter_hook = iter_hook + # Don't support obj_name ending/begining with '/', like /a, a/, /a/b/, + # etc. + obj = obj.strip(os.path.sep) + if os.path.sep in obj: + self._obj_path, self._obj = os.path.split(obj) else: - self.obj_path = '' - self.obj = obj + self._obj_path = '' + self._obj = obj - if self.obj_path: - self.name = '/'.join((container, self.obj_path)) + if self._obj_path: + self.name = os.path.join(container, self._obj_path) else: self.name = container - #Absolute path for obj directory. + # Absolute path for object directory. self.datadir = os.path.join(path, device, self.name) self.device_path = os.path.join(path, device) - self.container_path = os.path.join(path, device, container) + self._container_path = os.path.join(path, device, container) + self._is_dir = False self.tmpdir = os.path.join(path, device, 'tmp') self.logger = logger self.metadata = {} - self.data_file = None + self.meta_file = None self.fp = None self.iter_etag = None self.started_at_0 = False self.read_to_eof = False self.quarantined_dir = None self.keep_cache = False - self.is_dir = False - self.is_valid = True self.uid = int(uid) self.gid = int(gid) - if not os.path.exists(self.datadir + '/' + self.obj): + + # Don't store a value for data_file until we know it exists. + self.data_file = None + data_file = os.path.join(self.datadir, self._obj) + if not os.path.exists(data_file): return - self.data_file = os.path.join(self.datadir, self.obj) - self.metadata = read_metadata(self.datadir + '/' + self.obj) + self.data_file = os.path.join(data_file) + self.metadata = read_metadata(data_file) if not self.metadata: - create_object_metadata(self.datadir + '/' + self.obj) - self.metadata = read_metadata(self.datadir + '/' + self.obj) + create_object_metadata(data_file) + self.metadata = read_metadata(data_file) if not validate_object(self.metadata): - create_object_metadata(self.datadir + '/' + self.obj) - self.metadata = read_metadata(self.datadir + '/' + - self.obj) + create_object_metadata(data_file) + self.metadata = read_metadata(data_file) self.filter_metadata() - if os.path.isdir(self.datadir + '/' + self.obj): - self.is_dir = True + if os.path.isdir(data_file): + self._is_dir = True else: - self.fp = do_open(self.data_file, 'rb') + self.fp = do_open(data_file, 'rb') if not keep_data_fp: self.close(verify_file=False) @@ -124,7 +128,7 @@ class Gluster_DiskFile(DiskFile): file to see if it needs quarantining. """ #Marker directory - if self.is_dir: + if self._is_dir: return if self.fp: do_close(self.fp) @@ -150,8 +154,8 @@ class Gluster_DiskFile(DiskFile): create_object_metadata(dir_path) def put_metadata(self, metadata): - obj_path = self.datadir + '/' + self.obj - write_metadata(obj_path, metadata) + assert self.data_file is not None, "put_metadata: no file to put metadata into" + write_metadata(self.data_file, metadata) self.metadata = metadata def put(self, fd, tmppath, metadata, extension=''): @@ -175,10 +179,11 @@ class Gluster_DiskFile(DiskFile): content_type = metadata['Content-Type'] if not content_type: - # FIXME: How can this be some object that evaluates to False? + # FIXME: How can this be that our caller supplied us with metadata + # that has a content type that evaluates to False? # # FIXME: If the file exists, we would already know it is a - # directory. + # directory. So why are we assuming it is a file object? metadata['Content-Type'] = FILE_TYPE x_object_type = FILE else: @@ -199,43 +204,37 @@ class Gluster_DiskFile(DiskFile): extension = '' if metadata[X_OBJECT_TYPE] == MARKER_DIR: - # FIXME: If we know it already exists, why call - # create_dir_object()? - self.create_dir_object(os.path.join(self.datadir, self.obj)) + if not self.data_file: + self.data_file = os.path.join(self.datadir, self._obj) + self.create_dir_object(self.data_file) self.put_metadata(metadata) - self.data_file = self.datadir + '/' + self.obj return # Check if directory already exists. - if self.is_dir: + if self._is_dir: # FIXME: How can we have a directory and it not be marked as a # MARKER_DIR (see above)? - raise AlreadyExistsAsDir('File object already exists ' \ - 'as a directory: %s/%s' % (self.datadir , self.obj)) + msg = 'File object exists as a directory: %s' % self.data_file + raise AlreadyExistsAsDir(msg) timestamp = normalize_timestamp(metadata[X_TIMESTAMP]) write_metadata(tmppath, metadata) if X_CONTENT_LENGTH in metadata: self.drop_cache(fd, 0, int(metadata[X_CONTENT_LENGTH])) tpool.execute(os.fsync, fd) - if self.obj_path: - dir_objs = self.obj_path.split('/') + if self._obj_path: + dir_objs = self._obj_path.split('/') assert len(dir_objs) >= 1 - tmp_path = '' + tmp_path = self._container_path for dir_name in dir_objs: - if tmp_path: - tmp_path = tmp_path + '/' + dir_name - else: - tmp_path = dir_name - self.create_dir_object( - os.path.join(self.container_path, tmp_path)) - - renamer(tmppath, os.path.join(self.datadir, - self.obj + extension)) - do_chown(os.path.join(self.datadir, self.obj + extension), \ - self.uid, self.gid) + tmp_path = os.path.join(tmp_path, dir_name) + self.create_dir_object(tmp_path) + + newpath = os.path.join(self.datadir, self._obj) + renamer(tmppath, newpath) + do_chown(newpath, self.uid, self.gid) self.metadata = metadata - self.data_file = self.datadir + '/' + self.obj + extension + self.data_file = newpath return def unlinkold(self, timestamp): @@ -248,16 +247,16 @@ class Gluster_DiskFile(DiskFile): if not self.metadata or self.metadata['X-Timestamp'] >= timestamp: return - if self.is_dir: + if self._is_dir: # Marker directory object - if not rmdirs(os.path.join(self.datadir, self.obj)): - logging.error('Unable to delete dir %s' % os.path.join(self.datadir, self.obj)) + if not rmdirs(self.data_file): + logging.error('Unable to delete dir object: %s' % self.data_file) return else: # File object for fname in do_listdir(self.datadir): - if fname == self.obj: - do_unlink(os.path.join(self.datadir, fname)) + if os.path.join(self.datadir, fname) == self.data_file: + do_unlink(self.data_file) self.metadata = {} self.data_file = None @@ -273,7 +272,7 @@ class Gluster_DiskFile(DiskFile): :raises DiskFileNotExist: on file not existing (including deleted) """ #Marker directory. - if self.is_dir: + if self._is_dir: return 0 try: file_size = 0 @@ -283,7 +282,7 @@ class Gluster_DiskFile(DiskFile): metadata_size = int(self.metadata[X_CONTENT_LENGTH]) if file_size != metadata_size: self.metadata[X_CONTENT_LENGTH] = file_size - self.update_object(self.metadata) + write_metadata(self.data_file, self.metadata) return file_size except OSError as err: @@ -291,11 +290,6 @@ class Gluster_DiskFile(DiskFile): raise raise DiskFileNotExist('Data File does not exist.') - def update_object(self, metadata): - obj_path = self.datadir + '/' + self.obj - write_metadata(obj_path, metadata) - self.metadata = metadata - def filter_metadata(self): if X_TYPE in self.metadata: self.metadata.pop(X_TYPE) diff --git a/ufo/test/unit/common/test_diskfile.py b/ufo/test/unit/common/test_diskfile.py index 8c4756a7b69..264c656a5f1 100644 --- a/ufo/test/unit/common/test_diskfile.py +++ b/ufo/test/unit/common/test_diskfile.py @@ -102,18 +102,20 @@ class TestDiskFile(unittest.TestCase): assert not os.path.exists("/tmp/foo") gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", "z", self.lg) - assert gdf.obj == "z" - assert gdf.obj_path == "" + assert gdf._obj == "z" + assert gdf._obj_path == "" assert gdf.name == "bar" assert gdf.datadir == "/tmp/foo/vol0/bar" assert gdf.device_path == "/tmp/foo/vol0" - assert gdf.container_path == "/tmp/foo/vol0/bar" + assert gdf._container_path == "/tmp/foo/vol0/bar" assert gdf.tmpdir == "/tmp/foo/vol0/tmp" assert gdf.disk_chunk_size == 65536 + assert gdf.iter_hook == None assert gdf.logger == self.lg assert gdf.uid == DEFAULT_UID assert gdf.gid == DEFAULT_GID assert gdf.metadata == {} + assert gdf.meta_file == None assert gdf.data_file == None assert gdf.fp == None assert gdf.iter_etag == None @@ -121,15 +123,14 @@ class TestDiskFile(unittest.TestCase): assert not gdf.read_to_eof assert gdf.quarantined_dir == None assert not gdf.keep_cache - assert not gdf.is_dir - assert gdf.is_valid + assert not gdf._is_dir def test_constructor_leadtrail_slash(self): assert not os.path.exists("/tmp/foo") gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", "/b/a/z/", self.lg) - assert gdf.obj == "z" - assert gdf.obj_path == "b/a" + assert gdf._obj == "z" + assert gdf._obj_path == "b/a" assert gdf.name == "bar/b/a" assert gdf.datadir == "/tmp/foo/vol0/bar/b/a" assert gdf.device_path == "/tmp/foo/vol0" @@ -155,9 +156,9 @@ class TestDiskFile(unittest.TestCase): 'Content-Type': 'application/octet-stream'} gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", "z", self.lg) - assert gdf.obj == "z" + assert gdf._obj == "z" assert gdf.data_file == the_file - assert not gdf.is_dir + assert not gdf._is_dir assert gdf.fp is None assert gdf.metadata == exp_md finally: @@ -184,9 +185,9 @@ class TestDiskFile(unittest.TestCase): del exp_md['X-Object-Type'] gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", "z", self.lg) - assert gdf.obj == "z" + assert gdf._obj == "z" assert gdf.data_file == the_file - assert not gdf.is_dir + assert not gdf._is_dir assert gdf.fp is None assert gdf.metadata == exp_md finally: @@ -208,9 +209,9 @@ class TestDiskFile(unittest.TestCase): fd.write("1234") gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", "z", self.lg) - assert gdf.obj == "z" + assert gdf._obj == "z" assert gdf.data_file == the_file - assert not gdf.is_dir + assert not gdf._is_dir assert gdf.fp is None assert gdf.metadata != inv_md finally: @@ -235,9 +236,9 @@ class TestDiskFile(unittest.TestCase): del exp_md['X-Object-Type'] gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", "d", self.lg, keep_data_fp=True) - assert gdf.obj == "d" + assert gdf._obj == "d" assert gdf.data_file == the_dir - assert gdf.is_dir + assert gdf._is_dir assert gdf.fp is None assert gdf.metadata == exp_md finally: @@ -253,9 +254,9 @@ class TestDiskFile(unittest.TestCase): fd.write("1234") gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", "z", self.lg, keep_data_fp=True) - assert gdf.obj == "z" + assert gdf._obj == "z" assert gdf.data_file == the_file - assert not gdf.is_dir + assert not gdf._is_dir assert gdf.fp is not None finally: shutil.rmtree(td) @@ -266,6 +267,12 @@ class TestDiskFile(unittest.TestCase): "z", self.lg, disk_chunk_size=8192) assert gdf.disk_chunk_size == 8192 + def test_constructor_iter_hook(self): + assert not os.path.exists("/tmp/foo") + gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", + "z", self.lg, iter_hook='hook') + assert gdf.iter_hook == 'hook' + def test_close(self): assert not os.path.exists("/tmp/foo") gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", @@ -273,13 +280,13 @@ class TestDiskFile(unittest.TestCase): # Should be a no-op, as by default is_dir is False, but fp is None gdf.close() - gdf.is_dir = True + gdf._is_dir = True gdf.fp = "123" # Should still be a no-op as is_dir is True (marker directory) gdf.close() assert gdf.fp == "123" - gdf.is_dir = False + gdf._is_dir = False saved_dc = gluster.swift.common.DiskFile.do_close self.called = False def our_do_close(fp): @@ -307,7 +314,7 @@ class TestDiskFile(unittest.TestCase): gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", "dir/z", self.lg) # Not created, dir object path is different, just checking - assert gdf.obj == "z" + assert gdf._obj == "z" gdf.create_dir_object(the_dir) assert os.path.isdir(the_dir) assert the_dir in _metadata @@ -325,7 +332,7 @@ class TestDiskFile(unittest.TestCase): gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", "dir/z", self.lg) # Not created, dir object path is different, just checking - assert gdf.obj == "z" + assert gdf._obj == "z" def _mock_do_chown(p, u, g): assert u == DEFAULT_UID assert g == DEFAULT_GID @@ -343,14 +350,15 @@ class TestDiskFile(unittest.TestCase): def test_put_metadata(self): td = tempfile.mkdtemp() the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "dir") + the_dir = os.path.join(the_path, "z") try: + os.makedirs(the_dir) gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "dir/z", self.lg) + "z", self.lg) md = { 'a': 'b' } gdf.put_metadata(md) assert gdf.metadata == md - assert _metadata[os.path.join(the_dir, "z")] == md + assert _metadata[the_dir] == md finally: shutil.rmtree(td) @@ -434,6 +442,24 @@ class TestDiskFile(unittest.TestCase): finally: shutil.rmtree(td) + def test_put_w_marker_dir_create(self): + td = tempfile.mkdtemp() + the_path = os.path.join(td, "vol0", "bar") + the_dir = os.path.join(the_path, "dir") + try: + gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", + "dir", self.lg) + assert gdf.metadata == {} + newmd = { + 'Content-Length': 0, + 'ETag': 'etag', + 'X-Timestamp': 'ts', + 'Content-Type': 'application/directory'} + gdf.put(None, None, newmd) + assert gdf.metadata == newmd + assert _metadata[the_dir] == newmd + finally: + shutil.rmtree(td) def test_put_is_dir(self): td = tempfile.mkdtemp() @@ -466,8 +492,8 @@ class TestDiskFile(unittest.TestCase): try: gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", "z", self.lg) - assert gdf.obj == "z" - assert gdf.obj_path == "" + assert gdf._obj == "z" + assert gdf._obj_path == "" assert gdf.name == "bar" assert gdf.datadir == os.path.join(td, "vol0", "bar") assert gdf.data_file is None @@ -500,8 +526,8 @@ class TestDiskFile(unittest.TestCase): try: gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", the_file, self.lg) - assert gdf.obj == "z" - assert gdf.obj_path == the_obj_path + assert gdf._obj == "z" + assert gdf._obj_path == the_obj_path assert gdf.name == os.path.join("bar", "b", "a") assert gdf.datadir == os.path.join(td, "vol0", "bar", "b", "a") assert gdf.data_file is None @@ -572,14 +598,14 @@ class TestDiskFile(unittest.TestCase): fd.write("1234") gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", "z", self.lg) - assert gdf.obj == "z" + assert gdf._obj == "z" assert gdf.data_file == the_file - assert not gdf.is_dir + assert not gdf._is_dir later = float(gdf.metadata['X-Timestamp']) + 1 gdf.unlinkold(normalize_timestamp(later)) assert os.path.isdir(gdf.datadir) - assert not os.path.exists(os.path.join(gdf.datadir, gdf.obj)) + assert not os.path.exists(os.path.join(gdf.datadir, gdf._obj)) finally: shutil.rmtree(td) @@ -593,9 +619,9 @@ class TestDiskFile(unittest.TestCase): fd.write("1234") gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", "z", self.lg) - assert gdf.obj == "z" + assert gdf._obj == "z" assert gdf.data_file == the_file - assert not gdf.is_dir + assert not gdf._is_dir # Handle the case the file is not in the directory listing. os.unlink(the_file) @@ -603,7 +629,7 @@ class TestDiskFile(unittest.TestCase): later = float(gdf.metadata['X-Timestamp']) + 1 gdf.unlinkold(normalize_timestamp(later)) assert os.path.isdir(gdf.datadir) - assert not os.path.exists(os.path.join(gdf.datadir, gdf.obj)) + assert not os.path.exists(os.path.join(gdf.datadir, gdf._obj)) finally: shutil.rmtree(td) @@ -617,9 +643,9 @@ class TestDiskFile(unittest.TestCase): fd.write("1234") gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", "z", self.lg) - assert gdf.obj == "z" + assert gdf._obj == "z" assert gdf.data_file == the_file - assert not gdf.is_dir + assert not gdf._is_dir later = float(gdf.metadata['X-Timestamp']) + 1 @@ -637,7 +663,7 @@ class TestDiskFile(unittest.TestCase): os.chmod(the_path, stats.st_mode) assert os.path.isdir(gdf.datadir) - assert os.path.exists(os.path.join(gdf.datadir, gdf.obj)) + assert os.path.exists(os.path.join(gdf.datadir, gdf._obj)) finally: shutil.rmtree(td) @@ -650,12 +676,12 @@ class TestDiskFile(unittest.TestCase): gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", "d", self.lg, keep_data_fp=True) assert gdf.data_file == the_dir - assert gdf.is_dir + assert gdf._is_dir later = float(gdf.metadata['X-Timestamp']) + 1 gdf.unlinkold(normalize_timestamp(later)) assert os.path.isdir(gdf.datadir) - assert not os.path.exists(os.path.join(gdf.datadir, gdf.obj)) + assert not os.path.exists(os.path.join(gdf.datadir, gdf._obj)) finally: shutil.rmtree(td) @@ -668,7 +694,7 @@ class TestDiskFile(unittest.TestCase): gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", "d", self.lg, keep_data_fp=True) assert gdf.data_file == the_dir - assert gdf.is_dir + assert gdf._is_dir stats = os.stat(gdf.datadir) os.chmod(gdf.datadir, 0) @@ -692,9 +718,9 @@ class TestDiskFile(unittest.TestCase): fd.write("1234") gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", "z", self.lg) - assert gdf.obj == "z" + assert gdf._obj == "z" assert gdf.data_file == the_file - assert not gdf.is_dir + assert not gdf._is_dir assert 4 == gdf.get_data_file_size() finally: shutil.rmtree(td) @@ -709,9 +735,9 @@ class TestDiskFile(unittest.TestCase): fd.write("1234") gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", "z", self.lg) - assert gdf.obj == "z" + assert gdf._obj == "z" assert gdf.data_file == the_file - assert not gdf.is_dir + assert not gdf._is_dir assert 4 == gdf.metadata['Content-Length'] gdf.metadata['Content-Length'] = 3 assert 4 == gdf.get_data_file_size() @@ -740,9 +766,9 @@ class TestDiskFile(unittest.TestCase): fd.write("1234") gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", "z", self.lg) - assert gdf.obj == "z" + assert gdf._obj == "z" assert gdf.data_file == the_file - assert not gdf.is_dir + assert not gdf._is_dir gdf.data_file = gdf.data_file + ".dne" try: s = gdf.get_data_file_size() @@ -763,9 +789,9 @@ class TestDiskFile(unittest.TestCase): fd.write("1234") gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", "z", self.lg) - assert gdf.obj == "z" + assert gdf._obj == "z" assert gdf.data_file == the_file - assert not gdf.is_dir + assert not gdf._is_dir stats = os.stat(the_path) os.chmod(the_path, 0) try: @@ -787,27 +813,13 @@ class TestDiskFile(unittest.TestCase): os.makedirs(the_dir) gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", "d", self.lg, keep_data_fp=True) - assert gdf.obj == "d" + assert gdf._obj == "d" assert gdf.data_file == the_dir - assert gdf.is_dir + assert gdf._is_dir assert 0 == gdf.get_data_file_size() finally: shutil.rmtree(td) - def test_update_object(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "dir") - try: - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "dir/z", self.lg) - md = { 'a': 'b' } - gdf.update_object(md) - assert gdf.metadata == md - assert _metadata[os.path.join(the_dir, "z")] == md - finally: - shutil.rmtree(td) - def test_filter_metadata(self): assert not os.path.exists("/tmp/foo") gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", |