summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--gluster/swift/common/utils.py116
-rw-r--r--requirements.txt4
-rw-r--r--test-requirements.txt4
-rw-r--r--test/unit/common/test_utils.py46
4 files changed, 153 insertions, 17 deletions
diff --git a/gluster/swift/common/utils.py b/gluster/swift/common/utils.py
index 70bf551..a27aaf0 100644
--- a/gluster/swift/common/utils.py
+++ b/gluster/swift/common/utils.py
@@ -28,11 +28,18 @@ from gluster.swift.common.exceptions import GlusterFileSystemIOError
from swift.common.exceptions import DiskFileNoSpace
from swift.common.db import utf8encodekeys
from gluster.swift.common.fs_utils import do_getctime, do_getmtime, do_stat, \
- do_listdir, do_walk, do_rmdir, do_log_rl, get_filename_from_fd, do_open, \
+ do_rmdir, do_log_rl, get_filename_from_fd, do_open, \
do_isdir, do_getsize, do_getxattr, do_setxattr, do_removexattr, do_read, \
do_close, do_dup, do_lseek, do_fstat, do_fsync, do_rename
from gluster.swift.common import Glusterfs
+try:
+ import scandir
+ scandir_present = True
+except ImportError:
+ scandir_present = False
+
+
X_CONTENT_TYPE = 'Content-Type'
X_CONTENT_LENGTH = 'Content-Length'
X_TIMESTAMP = 'X-Timestamp'
@@ -306,6 +313,8 @@ def validate_object(metadata, statinfo=None):
def _update_list(path, cont_path, src_list, reg_file=True, object_count=0,
bytes_used=0, obj_list=[]):
+ if isinstance(path, unicode):
+ path = path.encode('utf-8')
# strip the prefix off, also stripping the leading and trailing slashes
obj_path = path.replace(cont_path, '').strip(os.path.sep)
@@ -363,7 +372,7 @@ def get_container_details(cont_path):
obj_list = []
if do_isdir(cont_path):
- for (path, dirs, files) in do_walk(cont_path):
+ for (path, dirs, files) in gf_walk(cont_path):
object_count, bytes_used = update_list(path, cont_path, dirs,
files, object_count,
bytes_used, obj_list)
@@ -436,16 +445,11 @@ def get_account_details(acc_path):
"""
container_list = []
- if do_isdir(acc_path):
- for name in do_listdir(acc_path):
- if name.lower() == TEMP_DIR \
- or name.lower() == ASYNCDIR \
- or name.lower() == TRASHCAN \
- or not do_isdir(os.path.join(acc_path, name)):
- # Do not include .async_pending, .trashcan and all
- # non-directories in containers list
- continue
- container_list.append(name)
+ if os.path.isdir(acc_path):
+ for entry in gf_listdir(acc_path):
+ if entry.is_dir() and \
+ entry.name not in (TEMP_DIR, ASYNCDIR, TRASHCAN):
+ container_list.append(entry.name)
return container_list, len(container_list)
@@ -636,7 +640,7 @@ def rmobjdir(dir_path, marker_dir_check=True):
# We have a directory that is not empty, walk it to see if it is filled
# with empty sub-directories that are not user created objects
# (gratuitously created as a result of other object creations).
- for (path, dirs, files) in do_walk(dir_path, topdown=False):
+ for (path, dirs, files) in gf_walk(dir_path, topdown=False):
for directory in dirs:
fullpath = os.path.join(path, directory)
@@ -717,3 +721,89 @@ def write_pickle(obj, dest, tmp=None, pickle_protocol=0):
fo.flush()
do_fsync(fo)
do_rename(tmppath, dest)
+
+
+DT_UNKNOWN = 0
+DT_DIR = 4
+
+
+class SmallDirEntry(object):
+ """
+ This class is an essential subset of DirEntry class from Python 3.5
+ https://docs.python.org/3.5/library/os.html#os.DirEntry
+ """
+ __slots__ = ('name', '_d_type', '_path', '_stat')
+
+ def __init__(self, path, name, d_type):
+ self.name = name
+ self._path = path # Parent directory
+ self._d_type = d_type
+ self._stat = None # Populated only if is_dir() is invoked.
+
+ def is_dir(self):
+ if self._d_type == DT_UNKNOWN:
+ try:
+ if not self._stat:
+ self._stat = os.lstat(os.path.join(self._path, self.name))
+ except OSError as err:
+ if err.errno != errno.ENOENT:
+ raise
+ return False
+ return stat.S_ISDIR(self._stat.st_mode)
+ else:
+ return self._d_type == DT_DIR
+
+
+def gf_listdir(path):
+ if scandir_present:
+ for entry in scandir.scandir(path):
+ yield entry
+ else:
+ for name in os.listdir(path):
+ yield SmallDirEntry(path, name, DT_UNKNOWN)
+
+
+def _walk(top, topdown=True, onerror=None, followlinks=False):
+ """
+ This is very similar to os.walk() implementation present in Python 2.7.11
+ (https://hg.python.org/cpython/file/v2.7.11/Lib/os.py#l209) but with the
+ following differences:
+ * This internally uses scandir which is a directory iterator instead
+ of os.listdir which returns an actual list.
+ * Makes no stat() calls. To do so, it deliberately chooses to ignore
+ https://bugs.python.org/issue23605#msg237775 issue.
+ """
+ dirs = [] # List of DirEntry objects
+ nondirs = [] # List of names (strings)
+
+ try:
+ for entry in scandir.scandir(top):
+ if entry.is_dir():
+ dirs.append(entry)
+ else:
+ nondirs.append(entry.name)
+ except OSError as err:
+ # As entries_iter is not a list and is a true iterator, it can raise
+ # an exception and blow-up. The try-except block here is to handle it
+ # gracefully and return.
+ if onerror is not None:
+ onerror(err)
+ return
+
+ if topdown:
+ yield top, [d.name for d in dirs], nondirs
+
+ for directory in dirs:
+ if followlinks or not directory.is_symlink():
+ new_path = os.path.join(top, directory.name)
+ for x in _walk(new_path, topdown, onerror, followlinks):
+ yield x
+
+ if not topdown:
+ yield top, [d.name for d in dirs], nondirs
+
+
+if scandir_present:
+ gf_walk = _walk
+else:
+ gf_walk = os.walk
diff --git a/requirements.txt b/requirements.txt
index a50eb9d..81ae220 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,3 +10,7 @@ pastedeploy>=1.3.3
simplejson>=2.0.9
xattr>=0.4
PyECLib==1.0.7 # BSD
+
+# gluster-swift specific requirements
+prettytable # needed by gswauth
+scandir>=1.3
diff --git a/test-requirements.txt b/test-requirements.txt
index 88ca936..27953e7 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -7,10 +7,6 @@ hacking>=0.8.0,<0.9
coverage
nose
nosexcover
-openstack.nose_plugin
nosehtmloutput
-oslosphinx
-sphinx>=1.1.2,<1.2
mock>=1.0
python-swiftclient
-prettytable # SOF
diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py
index 8ba9a2a..4790304 100644
--- a/test/unit/common/test_utils.py
+++ b/test/unit/common/test_utils.py
@@ -34,6 +34,14 @@ from gluster.swift.common.exceptions import GlusterFileSystemOSError,\
GlusterFileSystemIOError
from swift.common.exceptions import DiskFileNoSpace
+from nose import SkipTest
+
+try:
+ import scandir
+ scandir_present = True
+except ImportError:
+ scandir_present = False
+
#
# Somewhat hacky way of emulating the operation of xattr calls. They are made
# against a dictionary that stores the xattr key/value pairs.
@@ -1047,3 +1055,41 @@ class TestUtilsDirObjects(unittest.TestCase):
self.fail("Expected OSError")
finally:
utils.do_rmdir = _orig_rm
+
+ def test_gf_listdir(self):
+ for entry in utils.gf_listdir(self.rootdir):
+ if scandir_present:
+ self.assertFalse(isinstance(entry, utils.SmallDirEntry))
+ else:
+ self.assertTrue(isinstance(entry, utils.SmallDirEntry))
+ if entry.name in ('dir1'):
+ self.assertTrue(entry.is_dir())
+ if not scandir_present:
+ self.assertEqual(entry._d_type, utils.DT_UNKNOWN)
+ elif entry.name in ('file1', 'file2'):
+ self.assertFalse(entry.is_dir())
+
+
+class TestSmallDirEntry(unittest.TestCase):
+
+ def test_does_stat_when_no_d_type(self):
+ e = utils.SmallDirEntry('/root/path', 'name', utils.DT_UNKNOWN)
+ mock_os_lstat = Mock(return_value=Mock(st_mode=16744))
+ with patch('os.lstat', mock_os_lstat):
+ self.assertTrue(e.is_dir())
+ self.assertTrue(e._stat) # Make sure stat gets populated
+ mock_os_lstat.assert_called_once_with('/root/path/name')
+
+ # Subsequent calls to is_dir() should not call os.lstat()
+ mock_os_lstat.reset_mock()
+ with patch('os.lstat', mock_os_lstat):
+ self.assertTrue(e._stat) # Make sure stat is already populated
+ self.assertTrue(e.is_dir())
+ self.assertFalse(mock_os_lstat.called)
+
+ def test_is_dir_file_not_present_should_return_false(self):
+ e = utils.SmallDirEntry('/root/path', 'name', utils.DT_UNKNOWN)
+ mock_os_lstat = Mock(side_effect=OSError(errno.ENOENT,
+ os.strerror(errno.ENOENT)))
+ with patch('os.lstat', mock_os_lstat):
+ self.assertFalse(e.is_dir())