Enhance object expiration

This change re-introduces object expiration feature with some internal
differences from earlier implementation such as:

* Creation of zero-byte tracker object is performed directly on the
  mount point. Earlier HTTP request was sent to object server
  for the same. This incurred overhead of metadata creation for
  these zero-byte files which isn't necessarry as all required
  information is encoded in the path itself.
* Crawling of zero-byte tracker objects is done by the object
  expirer daemon itself and not container server.
* Deletion of tracker object is performed by the object expiration daemon
  directly on mount point.

Deletion of actual data object is not carried out by object expiration
daemon directly. The object expirer sends a DELETE request to object
server which deletes the actual object. This behaviour is not changed.

There is no change in behaviour in comparison with older implementation.
This is asserted by re-enabling existing functional tests without
any changes.

Change-Id: I01dc77cc4db3be3147d54e3aa2a19ed182498900
Signed-off-by: Prashanth Pai <ppai@redhat.com>
Reviewed-on: http://review.gluster.org/13913
Reviewed-by: Thiago da Silva <thiago@redhat.com>
Tested-by: Thiago da Silva <thiago@redhat.com>

3 files changed, 26 insertions, 15 deletions

diff --git a/test/functional_auth/common_conf/object-expirer.conf b/test/functional_auth/common_conf/object-expirer.conf
index 4449ee2..971367b 100644
--- a/test/functional_auth/common_conf/object-expirer.conf
+++ b/test/functional_auth/common_conf/object-expirer.conf
@@ -2,6 +2,7 @@
 #For now, refer: https://github.com/openstack/swift/blob/master/etc/object-expirer.conf-sample
 
 [DEFAULT]
+devices = /mnt/gluster-object
 
 [object-expirer]
 user = root
@@ -15,13 +16,18 @@ expiring_objects_account_name = expiring
 interval = 30
 
 [pipeline:main]
-pipeline = catch_errors cache proxy-server
+pipeline = catch_errors cache proxy-logging proxy-server
 
 [app:proxy-server]
 use = egg:gluster_swift#proxy
+log_facility = LOG_LOCAL1
+log_level = DEBUG
 
 [filter:cache]
 use = egg:swift#memcache
 
 [filter:catch_errors]
 use = egg:swift#catch_errors
+
+[filter:proxy-logging]
+use = egg:swift#proxy_logging
diff --git a/test/object_expirer_functional/test_object_expirer.py b/test/object_expirer_functional/test_object_expirer.py
index aaec75e..279994f 100644
--- a/test/object_expirer_functional/test_object_expirer.py
+++ b/test/object_expirer_functional/test_object_expirer.py
@@ -15,14 +15,19 @@
 
 import os
 import time
+import logging
 
-from swift.common.manager import Manager
-from swift.common.internal_client import InternalClient
+from gluster.swift.obj.expirer import ObjectExpirer, GlusterSwiftInternalClient
 
-from test.functional.tests import Base, config, Utils
+from swift.common.utils import readconf
+
+from test import get_config
+from test.functional.tests import Base, Utils
 from test.functional.swift_test_client import Account, Connection, \
     ResponseError
 
+config = get_config('func_test')
+
 
 class TestObjectExpirerEnv:
     @classmethod
@@ -39,9 +44,12 @@ class TestObjectExpirerEnv:
         cls.file_size = 8
         cls.root_dir = os.path.join('/mnt/gluster-object',
                                 cls.account.conn.storage_url.split('/')[2].split('_')[1])
-        cls.client = InternalClient('/etc/swift/object-expirer.conf',
-                                     'Test Object Expirer', 1)
-        cls.expirer = Manager(['object-expirer'])
+        devices = config.get('devices', '/mnt/gluster-object')
+        cls.client = GlusterSwiftInternalClient('/etc/swift/object-expirer.conf',
+                                                'Test Object Expirer', 1,
+                                                devices=devices)
+        conf = readconf('/etc/swift/object-expirer.conf', 'object-expirer')
+        cls.expirer = ObjectExpirer(conf)
 
 
 class TestObjectExpirer(Base):
@@ -91,7 +99,7 @@ class TestObjectExpirer(Base):
             self.fail("Tracker object not found.")
 
         # Run expirer daemon once.
-        self.env.expirer.once()
+        self.env.expirer.run_once()
 
         # Ensure object is physically deleted from filesystem.
         self.assertFalse(os.path.exists(os.path.join(self.env.root_dir,
@@ -151,7 +159,7 @@ class TestObjectExpirer(Base):
             self.fail("Tracker object not found.")
 
         # Run expirer daemon once.
-        self.env.expirer.once()
+        self.env.expirer.run_once()
 
         # Ensure object is physically deleted from filesystem.
         self.assertFalse(os.path.exists(os.path.join(self.env.root_dir,
@@ -169,7 +177,6 @@ class TestObjectExpirer(Base):
         # GET on container should no longer list the object.
         self.assertFalse(obj.name in self.env.container.files())
 
-
     def test_object_expiry_X_Delete_At_POST(self):
 
         # Create normal object
@@ -221,8 +228,7 @@ class TestObjectExpirer(Base):
             self.fail("Tracker object not found.")
 
         # Run expirer daemon once.
-        self.env.expirer.once()
-        time.sleep(3)
+        self.env.expirer.run_once()
 
         # Ensure object is physically deleted from filesystem.
         self.assertFalse(os.path.exists(os.path.join(self.env.root_dir,
@@ -291,8 +297,7 @@ class TestObjectExpirer(Base):
             self.fail("Tracker object not found.")
 
         # Run expirer daemon once.
-        self.env.expirer.once()
-        time.sleep(3)
+        self.env.expirer.run_once()
 
         # Ensure object is physically deleted from filesystem.
         self.assertFalse(os.path.exists(os.path.join(self.env.root_dir,
diff --git a/test/unit/common/test_diskdir.py b/test/unit/common/test_diskdir.py
index 3c91016..623164c 100644
--- a/test/unit/common/test_diskdir.py
+++ b/test/unit/common/test_diskdir.py
@@ -461,7 +461,7 @@ class TestContainerBroker(unittest.TestCase):
         self.assert_(broker.empty())
         obj = self._create_file('o.txt')
         self.assert_(not broker.empty())
-        broker.delete_object('o', normalize_timestamp(time()))
+        broker.delete_object('o', normalize_timestamp(time()), 0)
         # delete_object() should be a NOOP
         self.assert_(not broker.empty())
         os.unlink(obj)