object-storage: turn off stat() for container list

Turn of stat() system calls used to fetch the file size during a
container listing operation since these system calls can swamp Gluster
and the result is most often not used.

When a GET or HEAD request is made on a container, stat() system calls
are made during the Python standard library method, os.walk, to
determine if a given directory entry is another directory to recurse
into, and then utils._update_list() will stat() each file to get it
size, and finally utils.get_container_details_from_fs() will stat()
each directory encountered.

For most installations we have seen so far, we don't need the
container listing to accurately return the size of all the objects in
the container, so we can reduce the number of stat() system calls by
not fetching the size of the object.

For now, turn it off by default, and provide an /etc/swift/fs.conf
configuration parameter to turn it back on:

  accurate_size_in_listing = yes

The default for the above is "no".

Change-Id: I7dde11e14bb32ecafa3eabb08852f1ffc4366b35
BUG: 903396
Signed-off-by: Mohammed Junaid <junaid@redhat.com>
Reviewed-on: http://review.gluster.org/4787
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Kaleb KEITHLEY <kkeithle@redhat.com>
Reviewed-by: Anand Avati <avati@redhat.com>

1 files changed, 40 insertions, 3 deletions

diff --git a/ufo/test/unit/common/test_utils.py b/ufo/test/unit/common/test_utils.py
index 92ce9aef30f..c645509fa24 100644
--- a/ufo/test/unit/common/test_utils.py
+++ b/ufo/test/unit/common/test_utils.py
@@ -26,7 +26,7 @@ import tarfile
 import shutil
 from collections import defaultdict
 from swift.common.utils import normalize_timestamp
-from gluster.swift.common import utils
+from gluster.swift.common import utils, Glusterfs
 
 #
 # Somewhat hacky way of emulating the operation of xattr calls. They are made
@@ -755,7 +755,7 @@ class TestUtils(unittest.TestCase):
             utils._get_account_details_from_fs = orig_gcdff
             utils.do_stat = orig_ds
 
-    def test_get_container_details_from_fs(self):
+    def test_get_account_details_from_fs(self):
         orig_cwd = os.getcwd()
         td = tempfile.mkdtemp()
         try:
@@ -779,7 +779,39 @@ class TestUtils(unittest.TestCase):
         assert cd.obj_list == []
         assert cd.dir_list == []
 
-    def test_get_account_details_from_fs(self):
+    def test_get_container_details_from_fs(self):
+        orig_cwd = os.getcwd()
+        td = tempfile.mkdtemp()
+        try:
+            tf = tarfile.open("common/data/container_tree.tar.bz2", "r:bz2")
+            os.chdir(td)
+            tf.extractall()
+
+            cd = utils._get_container_details_from_fs(td)
+            assert cd.bytes_used == 0, repr(cd.bytes_used)
+            assert cd.object_count == 8, repr(cd.object_count)
+            assert set(cd.obj_list) == set(['file1', 'file3', 'file2',
+                                   'dir3', 'dir1', 'dir2',
+                                   'dir1/file1', 'dir1/file2'
+                                   ]), repr(cd.obj_list)
+
+            full_dir1 = os.path.join(td, 'dir1')
+            full_dir2 = os.path.join(td, 'dir2')
+            full_dir3 = os.path.join(td, 'dir3')
+            exp_dir_dict = { td:        os.path.getmtime(td),
+                             full_dir1: os.path.getmtime(full_dir1),
+                             full_dir2: os.path.getmtime(full_dir2),
+                             full_dir3: os.path.getmtime(full_dir3),
+                             }
+            for d,m in cd.dir_list:
+                assert d in exp_dir_dict
+                assert exp_dir_dict[d] == m
+        finally:
+            os.chdir(orig_cwd)
+            shutil.rmtree(td)
+
+
+    def test_get_container_details_from_fs_do_getsize_true(self):
         orig_cwd = os.getcwd()
         td = tempfile.mkdtemp()
         try:
@@ -787,6 +819,9 @@ class TestUtils(unittest.TestCase):
             os.chdir(td)
             tf.extractall()
 
+            __do_getsize = Glusterfs._do_getsize
+            Glusterfs._do_getsize = True
+
             cd = utils._get_container_details_from_fs(td)
             assert cd.bytes_used == 30, repr(cd.bytes_used)
             assert cd.object_count == 8, repr(cd.object_count)
@@ -794,6 +829,7 @@ class TestUtils(unittest.TestCase):
                                    'dir3', 'dir1', 'dir2',
                                    'dir1/file1', 'dir1/file2'
                                    ]), repr(cd.obj_list)
+
             full_dir1 = os.path.join(td, 'dir1')
             full_dir2 = os.path.join(td, 'dir2')
             full_dir3 = os.path.join(td, 'dir3')
@@ -806,6 +842,7 @@ class TestUtils(unittest.TestCase):
                 assert d in exp_dir_dict
                 assert exp_dir_dict[d] == m
         finally:
+            Glusterfs._do_getsize = __do_getsize
             os.chdir(orig_cwd)
             shutil.rmtree(td)