summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPrashanth Pai <ppai@redhat.com>2016-04-20 15:10:43 +0530
committerThiago da Silva <thiago@redhat.com>2016-04-20 12:08:03 -0700
commit5a04cede1f5bb44d6c64b186335146dd4e70a6ea (patch)
tree34babe4bcc295562ca3b740c98bc2299669c12f3
parent2bd696e392e420a2521dcca0b8613122d8169025 (diff)
Make swift's expirer compatible with gluster-swift
Swift's object expirer in kilo series was incompatible with gluster-swift. This change does the following: * Optimizes crawl in account and container server for listing requests for containers and tracker objects in gsexpiring volume. * Enables container server to delete tracker objects from gsexpiring volume. Swift's expirer sends request directly to container server to remove tracker object entry. * delete_tracker_object() is now a common utility function that is invoked from container server and gluster-swift's object expirer. * Run functional test to be run against both swift's object expirer and gluster-swift's object expirer Change-Id: Ib5b7f7f08fe7dda574f6dd80be2f38bdfaee32bc Signed-off-by: Prashanth Pai <ppai@redhat.com> Reviewed-on: http://review.gluster.org/14038 Reviewed-by: Thiago da Silva <thiago@redhat.com> Tested-by: Thiago da Silva <thiago@redhat.com>
-rw-r--r--etc/object-expirer.conf-gluster8
-rw-r--r--gluster/swift/common/DiskDir.py97
-rw-r--r--gluster/swift/common/utils.py57
-rw-r--r--gluster/swift/obj/expirer.py40
-rw-r--r--test/object_expirer_functional/test_object_expirer_gluster_swift.py (renamed from test/object_expirer_functional/test_object_expirer.py)0
-rw-r--r--test/object_expirer_functional/test_object_expirer_swift.py335
-rw-r--r--test/unit/common/test_diskdir.py70
7 files changed, 559 insertions, 48 deletions
diff --git a/etc/object-expirer.conf-gluster b/etc/object-expirer.conf-gluster
index 32be5a1..8be8626 100644
--- a/etc/object-expirer.conf-gluster
+++ b/etc/object-expirer.conf-gluster
@@ -14,6 +14,14 @@ log_level = INFO
auto_create_account_prefix = gs
expiring_objects_account_name = expiring
+# The expirer will re-attempt expiring if the source object is not available
+# up to reclaim_age seconds before it gives up and deletes the entry in the
+# queue. In gluster-swift, you'd almost always want to set this to zero.
+reclaim_age = 0
+
+# Do not retry DELETEs on getting 404. Hence default is set to 1.
+request_tries = 1
+
# The swift-object-expirer daemon will run every 'interval' number of seconds
# interval = 300
diff --git a/gluster/swift/common/DiskDir.py b/gluster/swift/common/DiskDir.py
index d314a1f..204ae1d 100644
--- a/gluster/swift/common/DiskDir.py
+++ b/gluster/swift/common/DiskDir.py
@@ -17,19 +17,22 @@ import os
import errno
from gluster.swift.common.fs_utils import dir_empty, mkdirs, do_chown, \
- do_exists, do_touch
+ do_exists, do_touch, do_stat
from gluster.swift.common.utils import validate_account, validate_container, \
get_container_details, get_account_details, create_container_metadata, \
create_account_metadata, DEFAULT_GID, get_container_metadata, \
get_account_metadata, DEFAULT_UID, validate_object, \
create_object_metadata, read_metadata, write_metadata, X_CONTENT_TYPE, \
X_CONTENT_LENGTH, X_TIMESTAMP, X_PUT_TIMESTAMP, X_ETAG, X_OBJECTS_COUNT, \
- X_BYTES_USED, X_CONTAINER_COUNT, DIR_TYPE, rmobjdir, dir_is_object
+ X_BYTES_USED, X_CONTAINER_COUNT, DIR_TYPE, rmobjdir, dir_is_object, \
+ list_objects_gsexpiring_container, normalize_timestamp
from gluster.swift.common import Glusterfs
from gluster.swift.common.exceptions import FileOrDirNotFoundError, \
GlusterFileSystemIOError
+from gluster.swift.obj.expirer import delete_tracker_object
from swift.common.constraints import MAX_META_COUNT, MAX_META_OVERALL_SIZE
from swift.common.swob import HTTPBadRequest
+from swift.common.utils import ThreadPool
DATADIR = 'containers'
@@ -176,6 +179,12 @@ class DiskCommon(object):
self.datadir = os.path.join(root, drive)
self._dir_exists = None
+ # nthread=0 is intentional. This ensures that no green pool is
+ # used. Call to force_run_in_thread() will ensure that the method
+ # passed as arg is run in a real external thread using eventlet.tpool
+ # which has a threadpool of 20 threads (default)
+ self.threadpool = ThreadPool(nthreads=0)
+
def _dir_exists_read_metadata(self):
self._dir_exists = do_exists(self.datadir)
if self._dir_exists:
@@ -342,6 +351,24 @@ class DiskDir(DiskCommon):
self.container = container
self.datadir = os.path.join(self.datadir, self.container)
+ if self.account == 'gsexpiring':
+ # Do not bother crawling the entire container tree just to update
+ # object count and bytes used. Return immediately before metadata
+ # validation and creation happens.
+ info = do_stat(self.datadir)
+ if not info:
+ # Container no longer exists.
+ return
+ semi_fake_md = {
+ 'X-Object-Count': (0, 0),
+ 'X-Timestamp': ((normalize_timestamp(info.st_ctime)), 0),
+ 'X-Type': ('container', 0),
+ 'X-PUT-Timestamp': ((normalize_timestamp(info.st_mtime)), 0),
+ 'X-Bytes-Used': (0, 0)
+ }
+ self.metadata = semi_fake_md
+ return
+
if not self._dir_exists_read_metadata():
return
@@ -387,7 +414,10 @@ class DiskDir(DiskCommon):
container_list = []
- objects = self._update_object_count()
+ if self.account == 'gsexpiring':
+ objects = list_objects_gsexpiring_container(self.datadir)
+ else:
+ objects = self._update_object_count()
if objects:
objects.sort()
else:
@@ -419,12 +449,23 @@ class DiskDir(DiskCommon):
objects = filter_delimiter(objects, delimiter, prefix, marker,
path)
- if out_content_type == 'text/plain':
+ if out_content_type == 'text/plain' or \
+ self.account == 'gsexpiring':
+ # When out_content_type == 'text/plain':
+ #
# The client is only asking for a plain list of objects and NOT
# asking for any extended information about objects such as
# bytes used or etag.
+ #
+ # When self.account == 'gsexpiring':
+ #
+ # This is a JSON request sent by the object expirer to list
+ # tracker objects in a container in gsexpiring volume.
+ # When out_content_type is 'application/json', the caller
+ # expects each record entry to have the following ordered
+ # fields: (name, timestamp, size, content_type, etag)
for obj in objects:
- container_list.append((obj, 0, 0, 0, 0))
+ container_list.append((obj, '0', 0, 'text/plain', ''))
if len(container_list) >= limit:
break
return container_list
@@ -498,7 +539,8 @@ class DiskDir(DiskCommon):
reported_put_timestamp, reported_delete_timestamp,
reported_object_count, and reported_bytes_used.
"""
- if self._dir_exists and Glusterfs._container_update_object_count:
+ if self._dir_exists and Glusterfs._container_update_object_count and \
+ self.account != 'gsexpiring':
self._update_object_count()
data = {'account': self.account, 'container': self.container,
@@ -560,9 +602,15 @@ class DiskDir(DiskCommon):
write_metadata(self.datadir, self.metadata)
def delete_object(self, name, timestamp, obj_policy_index):
- # NOOP - should never be called since object file removal occurs
- # within a directory implicitly.
- return
+ if self.account == 'gsexpiring':
+ # The request originated from object expirer. This should
+ # delete tracker object.
+ self.threadpool.force_run_in_thread(delete_tracker_object,
+ self.datadir, name)
+ else:
+ # NOOP - should never be called since object file removal occurs
+ # within a directory implicitly.
+ return
def delete_db(self, timestamp):
"""
@@ -626,6 +674,22 @@ class DiskAccount(DiskCommon):
super(DiskAccount, self).__init__(root, drive, account, logger,
**kwargs)
+ if self.account == 'gsexpiring':
+ # Do not bother updating object count, container count and bytes
+ # used. Return immediately before metadata validation and
+ # creation happens.
+ info = do_stat(self.datadir)
+ semi_fake_md = {
+ 'X-Object-Count': (0, 0),
+ 'X-Container-Count': (0, 0),
+ 'X-Timestamp': ((normalize_timestamp(info.st_ctime)), 0),
+ 'X-Type': ('Account', 0),
+ 'X-PUT-Timestamp': ((normalize_timestamp(info.st_mtime)), 0),
+ 'X-Bytes-Used': (0, 0)
+ }
+ self.metadata = semi_fake_md
+ return
+
# Since accounts should always exist (given an account maps to a
# gluster volume directly, and the mount has already been checked at
# the beginning of the REST API handling), just assert that that
@@ -750,10 +814,20 @@ class DiskAccount(DiskCommon):
containers = filter_delimiter(containers, delimiter, prefix,
marker)
- if response_content_type == 'text/plain':
+ if response_content_type == 'text/plain' or \
+ self.account == 'gsexpiring':
+ # When response_content_type == 'text/plain':
+ #
# The client is only asking for a plain list of containers and NOT
# asking for any extended information about container such as
# bytes used or object count.
+ #
+ # When self.account == 'gsexpiring':
+ # This is a JSON request sent by the object expirer to list
+ # containers in gsexpiring volume. When out_content_type is
+ # 'application/json', the caller expects each record entry to have
+ # the following ordered fields:
+ # (name, object_count, bytes_used, is_subdir)
for container in containers:
# When response_content_type == 'text/plain', Swift will only
# consume the name of the container (first element of tuple).
@@ -796,7 +870,8 @@ class DiskAccount(DiskCommon):
delete_timestamp, container_count, object_count,
bytes_used, hash, id
"""
- if Glusterfs._account_update_container_count:
+ if Glusterfs._account_update_container_count and \
+ self.account != 'gsexpiring':
self._update_container_count()
data = {'account': self.account, 'created_at': '1',
diff --git a/gluster/swift/common/utils.py b/gluster/swift/common/utils.py
index 1bbc56c..26e8c1b 100644
--- a/gluster/swift/common/utils.py
+++ b/gluster/swift/common/utils.py
@@ -370,6 +370,63 @@ def get_container_details(cont_path):
return obj_list, object_count, bytes_used
+def list_objects_gsexpiring_container(container_path):
+ """
+ This method does a simple walk, unlike get_container_details which
+ walks the filesystem tree and does a getxattr() on every directory
+ to check if it's a directory marker object and stat() on every file
+ to get it's size. These are not required for gsexpiring volume as
+ it can never have directory marker objects in it and all files are
+ zero-byte in size.
+ """
+ obj_list = []
+
+ for (root, dirs, files) in os.walk(container_path):
+ for f in files:
+ obj_path = os.path.join(root, f)
+ obj = obj_path[(len(container_path) + 1):]
+ obj_list.append(obj)
+ # Yield the co-routine cooperatively
+ sleep()
+
+ return obj_list
+
+
+def delete_tracker_object(container_path, obj):
+ """
+ Delete zero-byte tracker object from gsexpiring volume.
+ Called by:
+ - gluster.swift.obj.expirer.ObjectExpirer.pop_queue()
+ - gluster.swift.common.DiskDir.DiskDir.delete_object()
+ """
+ tracker_object_path = os.path.join(container_path, obj)
+
+ try:
+ os.unlink(tracker_object_path)
+ except OSError as err:
+ if err.errno in (errno.ENOENT, errno.ESTALE):
+ # Ignore removal from another entity.
+ return
+ elif err.errno == errno.EISDIR:
+ # Handle race: Was a file during crawl, but now it's a
+ # directory. There are no 'directory marker' objects in
+ # gsexpiring volume.
+ return
+ else:
+ raise
+
+ # This part of code is very similar to DiskFile._unlinkold()
+ dirname = os.path.dirname(tracker_object_path)
+ while dirname and dirname != container_path:
+ if not rmobjdir(dirname, marker_dir_check=False):
+ # If a directory with objects has been found, we can stop
+ # garbage collection
+ break
+ else:
+ # Traverse upwards till the root of container
+ dirname = os.path.dirname(dirname)
+
+
def get_account_details(acc_path):
"""
Return container_list and container_count.
diff --git a/gluster/swift/obj/expirer.py b/gluster/swift/obj/expirer.py
index 564a2c9..38f870e 100644
--- a/gluster/swift/obj/expirer.py
+++ b/gluster/swift/obj/expirer.py
@@ -20,7 +20,7 @@ import gluster.swift.common.constraints # noqa
import errno
import os
-from gluster.swift.common.utils import rmobjdir
+from gluster.swift.common.utils import delete_tracker_object
from swift.obj.expirer import ObjectExpirer as SwiftObjectExpirer
from swift.common.http import HTTP_NOT_FOUND
@@ -95,45 +95,17 @@ class ObjectExpirer(SwiftObjectExpirer):
# which has a threadpool of 20 threads (default)
self.threadpool = ThreadPool(nthreads=0)
- def _delete_tracker_object(self, container, obj):
- container_path = os.path.join(self.devices,
- self.expiring_objects_account,
- container)
- tracker_object_path = os.path.join(container_path, obj)
-
- try:
- os.unlink(tracker_object_path)
- except OSError as err:
- if err.errno in (errno.ENOENT, errno.ESTALE):
- # Ignore removal from another entity.
- return
- elif err.errno == errno.EISDIR:
- # Handle race: Was a file during crawl, but now it's a
- # directory. There are no 'directory marker' objects in
- # gsexpiring volume.
- return
- else:
- raise
-
- # This part of code is very similar to DiskFile._unlinkold()
- dirname = os.path.dirname(tracker_object_path)
- while dirname and dirname != container_path:
- if not rmobjdir(dirname, marker_dir_check=False):
- # If a directory with objects has been found, we can stop
- # garbage collection
- break
- else:
- # Traverse upwards till the root of container
- dirname = os.path.dirname(dirname)
-
def pop_queue(self, container, obj):
"""
In Swift, this method removes tracker object entry directly from
container database. In gluster-swift, this method deletes tracker
object directly from filesystem.
"""
- self.threadpool.force_run_in_thread(self._delete_tracker_object,
- container, obj)
+ container_path = os.path.join(self.devices,
+ self.expiring_objects_account,
+ container)
+ self.threadpool.force_run_in_thread(delete_tracker_object,
+ container_path, obj)
def delete_actual_object(self, actual_obj, timestamp):
"""
diff --git a/test/object_expirer_functional/test_object_expirer.py b/test/object_expirer_functional/test_object_expirer_gluster_swift.py
index 279994f..279994f 100644
--- a/test/object_expirer_functional/test_object_expirer.py
+++ b/test/object_expirer_functional/test_object_expirer_gluster_swift.py
diff --git a/test/object_expirer_functional/test_object_expirer_swift.py b/test/object_expirer_functional/test_object_expirer_swift.py
new file mode 100644
index 0000000..c59cc67
--- /dev/null
+++ b/test/object_expirer_functional/test_object_expirer_swift.py
@@ -0,0 +1,335 @@
+# Copyright (c) 2014 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import time
+import logging
+
+from swift.common.manager import Manager
+from swift.common.internal_client import InternalClient
+
+
+from test import get_config
+from test.functional.tests import Base, Utils
+from test.functional.swift_test_client import Account, Connection, \
+ ResponseError
+
+config = get_config('func_test')
+
+
+class TestObjectExpirerEnv:
+ @classmethod
+ def setUp(cls):
+ cls.conn = Connection(config)
+ cls.conn.authenticate()
+ cls.account = Account(cls.conn,
+ config.get('account',
+ config['username']))
+ cls.account.delete_containers()
+ cls.container = cls.account.container(Utils.create_name())
+ if not cls.container.create():
+ raise ResponseError(cls.conn.response)
+ cls.file_size = 8
+ cls.root_dir = os.path.join('/mnt/gluster-object',
+ cls.account.conn.storage_url.split('/')[2].split('_')[1])
+ devices = config.get('devices', '/mnt/gluster-object')
+ cls.client = InternalClient('/etc/swift/object-expirer.conf',
+ 'Test Object Expirer', 1)
+ cls.expirer = Manager(['object-expirer'])
+
+
+class TestObjectExpirer(Base):
+ env = TestObjectExpirerEnv
+ set_up = False
+
+ def test_object_expiry_X_Delete_At_PUT(self):
+ obj = self.env.container.file(Utils.create_name())
+ x_delete_at = str(int(time.time()) + 2)
+ obj.write_random(self.env.file_size,
+ hdrs={'X-Delete-At': x_delete_at})
+
+ # Object is not expired. Should still be accessible.
+ obj.read()
+ self.assert_status(200)
+
+ # Ensure X-Delete-At is saved as object metadata.
+ self.assertEqual(x_delete_at, str(obj.info()['x_delete_at']))
+
+ # Wait for object to be expired.
+ time.sleep(3)
+
+ # Object has expired. Should no longer be accessible.
+ self.assertRaises(ResponseError, obj.read)
+ self.assert_status(404)
+
+ # Object should still be present on filesystem.
+ self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir,
+ self.env.container.name,
+ obj.name)))
+
+ # But, GET on container should list the expired object.
+ result = self.env.container.files()
+ self.assertTrue(obj.name in self.env.container.files())
+
+ # Check existence of corresponding tracker object in gsexpiring
+ # account.
+ enteredLoop = False
+ for c in self.env.client.iter_containers("gsexpiring"):
+ for o in self.env.client.iter_objects("gsexpiring", c['name']):
+ enteredLoop = True
+ l = o['name'].split('/')
+ self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name))
+ self.assertEqual(l[1], self.env.container.name)
+ self.assertEqual(l[2], obj.name)
+ if not enteredLoop:
+ self.fail("Tracker object not found.")
+
+ # Run expirer daemon once.
+ self.env.expirer.once()
+
+ # Ensure object is physically deleted from filesystem.
+ self.assertFalse(os.path.exists(os.path.join(self.env.root_dir,
+ self.env.container.name,
+ obj.name)))
+
+ # Ensure tracker object is consumed.
+ try:
+ self.env.client.iter_containers("gsexpiring").next()
+ except StopIteration:
+ pass
+ else:
+ self.fail("Tracker object persists!")
+
+ # GET on container should no longer list the object.
+ self.assertFalse(obj.name in self.env.container.files())
+
+ def test_object_expiry_X_Delete_After_PUT(self):
+ obj = self.env.container.file(Utils.create_name())
+ obj.write_random(self.env.file_size,
+ hdrs={'X-Delete-After': 2})
+
+ # Object is not expired. Should still be accessible.
+ obj.read()
+ self.assert_status(200)
+
+ # Ensure X-Delete-At is saved as object metadata.
+ self.assertTrue(str(obj.info()['x_delete_at']))
+
+ # Wait for object to be expired.
+ time.sleep(3)
+
+ # Object has expired. Should no longer be accessible.
+ self.assertRaises(ResponseError, obj.read)
+ self.assert_status(404)
+
+ # Object should still be present on filesystem.
+ self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir,
+ self.env.container.name,
+ obj.name)))
+
+ # But, GET on container should list the expired object.
+ result = self.env.container.files()
+ self.assertTrue(obj.name in self.env.container.files())
+
+ # Check existence of corresponding tracker object in gsexpiring
+ # account.
+ enteredLoop = False
+ for c in self.env.client.iter_containers("gsexpiring"):
+ for o in self.env.client.iter_objects("gsexpiring", c['name']):
+ enteredLoop = True
+ l = o['name'].split('/')
+ self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name))
+ self.assertEqual(l[1], self.env.container.name)
+ self.assertEqual(l[2], obj.name)
+ if not enteredLoop:
+ self.fail("Tracker object not found.")
+
+ # Run expirer daemon once.
+ self.env.expirer.once()
+
+ # Ensure object is physically deleted from filesystem.
+ self.assertFalse(os.path.exists(os.path.join(self.env.root_dir,
+ self.env.container.name,
+ obj.name)))
+
+ # Ensure tracker object is consumed.
+ try:
+ self.env.client.iter_containers("gsexpiring").next()
+ except StopIteration:
+ pass
+ else:
+ self.fail("Tracker object persists!")
+
+ # GET on container should no longer list the object.
+ self.assertFalse(obj.name in self.env.container.files())
+
+ def test_object_expiry_X_Delete_At_POST(self):
+
+ # Create normal object
+ obj = self.env.container.file(Utils.create_name())
+ obj.write_random(self.env.file_size)
+ obj.read()
+ self.assert_status(200)
+
+ # Send POST on that object and set it to be expired.
+ x_delete_at = str(int(time.time()) + 2)
+ obj.sync_metadata(metadata={'X-Delete-At': x_delete_at},
+ cfg={'x_delete_at': x_delete_at})
+
+ # Ensure X-Delete-At is saved as object metadata.
+ self.assertEqual(x_delete_at, str(obj.info()['x_delete_at']))
+
+ # Object is not expired. Should still be accessible.
+ obj.read()
+ self.assert_status(200)
+
+ # Wait for object to be expired.
+ time.sleep(3)
+
+ # Object has expired. Should no longer be accessible.
+ self.assertRaises(ResponseError, obj.read)
+ self.assert_status(404)
+
+ # Object should still be present on filesystem.
+ self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir,
+ self.env.container.name,
+ obj.name)))
+
+ # But, GET on container should list the expired object.
+ result = self.env.container.files()
+ self.assertTrue(obj.name in self.env.container.files())
+
+ # Check existence of corresponding tracker object in gsexpiring
+ # account.
+
+ enteredLoop = False
+ for c in self.env.client.iter_containers("gsexpiring"):
+ for o in self.env.client.iter_objects("gsexpiring", c['name']):
+ enteredLoop = True
+ l = o['name'].split('/')
+ self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name))
+ self.assertEqual(l[1], self.env.container.name)
+ self.assertEqual(l[2], obj.name)
+ if not enteredLoop:
+ self.fail("Tracker object not found.")
+
+ # Run expirer daemon once.
+ self.env.expirer.once()
+
+ # Ensure object is physically deleted from filesystem.
+ self.assertFalse(os.path.exists(os.path.join(self.env.root_dir,
+ self.env.container.name,
+ obj.name)))
+
+ # Ensure tracker object is consumed.
+ try:
+ self.env.client.iter_containers("gsexpiring").next()
+ except StopIteration:
+ pass
+ else:
+ self.fail("Tracker object persists!")
+
+ # GET on container should no longer list the object.
+ self.assertFalse(obj.name in self.env.container.files())
+
+
+ def test_object_expiry_X_Delete_After_POST(self):
+
+ # Create normal object
+ obj = self.env.container.file(Utils.create_name())
+ obj.write_random(self.env.file_size)
+ obj.read()
+ self.assert_status(200)
+
+ # Send POST on that object and set it to be expired.
+ obj.sync_metadata(metadata={'X-Delete-After': 2},
+ cfg={'x_delete_after': 2})
+
+ # Ensure X-Delete-At is saved as object metadata.
+ self.assertTrue(str(obj.info()['x_delete_at']))
+
+ # Object is not expired. Should still be accessible.
+ obj.read()
+ self.assert_status(200)
+
+ # Wait for object to be expired.
+ time.sleep(3)
+
+ # Object has expired. Should no longer be accessible.
+ self.assertRaises(ResponseError, obj.read)
+ self.assert_status(404)
+
+ # Object should still be present on filesystem.
+ self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir,
+ self.env.container.name,
+ obj.name)))
+
+ # But, GET on container should list the expired object.
+ result = self.env.container.files()
+ self.assertTrue(obj.name in self.env.container.files())
+
+ # Check existence of corresponding tracker object in gsexpiring
+ # account.
+
+ enteredLoop = False
+ for c in self.env.client.iter_containers("gsexpiring"):
+ for o in self.env.client.iter_objects("gsexpiring", c['name']):
+ enteredLoop = True
+ l = o['name'].split('/')
+ self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name))
+ self.assertEqual(l[1], self.env.container.name)
+ self.assertEqual(l[2], obj.name)
+ if not enteredLoop:
+ self.fail("Tracker object not found.")
+
+ # Run expirer daemon once.
+ self.env.expirer.once()
+
+ # Ensure object is physically deleted from filesystem.
+ self.assertFalse(os.path.exists(os.path.join(self.env.root_dir,
+ self.env.container.name,
+ obj.name)))
+
+ # Ensure tracker object is consumed.
+ try:
+ self.env.client.iter_containers("gsexpiring").next()
+ except StopIteration:
+ pass
+ else:
+ self.fail("Tracker object persists!")
+
+ # GET on container should no longer list the object.
+ self.assertFalse(obj.name in self.env.container.files())
+
+
+ def test_object_expiry_err(self):
+ obj = self.env.container.file(Utils.create_name())
+
+ # X-Delete-At is invalid or is in the past
+ for i in (-2, 'abc', str(int(time.time()) - 2), 5.8):
+ self.assertRaises(ResponseError,
+ obj.write_random,
+ self.env.file_size,
+ hdrs={'X-Delete-At': i})
+ self.assert_status(400)
+
+ # X-Delete-After is invalid.
+ for i in (-2, 'abc', 3.7):
+ self.assertRaises(ResponseError,
+ obj.write_random,
+ self.env.file_size,
+ hdrs={'X-Delete-After': i})
+ self.assert_status(400)
+
diff --git a/test/unit/common/test_diskdir.py b/test/unit/common/test_diskdir.py
index 623164c..964bc2f 100644
--- a/test/unit/common/test_diskdir.py
+++ b/test/unit/common/test_diskdir.py
@@ -402,6 +402,37 @@ class TestContainerBroker(unittest.TestCase):
fp.write("file path: %s\n" % fullname)
return fullname
+ def test_gsexpiring_fake_md(self):
+ # Create account
+ account_path = os.path.join(self.path, "gsexpiring")
+ os.makedirs(account_path)
+
+ # Create container
+ cpath = os.path.join(account_path, "container")
+ os.mkdir(cpath)
+
+ # Create 10 objects in container. These should not reflect in
+ # X-Object-Count.
+ for o in xrange(10):
+ os.mkdir(os.path.join(cpath, str(o)))
+
+ orig_stat = os.stat(cpath)
+ expected_metadata = {
+ 'X-Object-Count': (0, 0),
+ 'X-Timestamp': ((normalize_timestamp(orig_stat.st_ctime)), 0),
+ 'X-Type': ('container', 0),
+ 'X-PUT-Timestamp': ((normalize_timestamp(orig_stat.st_mtime)), 0),
+ 'X-Bytes-Used': (0, 0)
+ }
+
+ # Create DiskDir instance
+ disk_dir = dd.DiskDir(self.path, 'gsexpiring', account='gsexpiring',
+ container='container', logger=FakeLogger())
+ self.assertEqual(expected_metadata, disk_dir.metadata)
+ info = disk_dir.get_info()
+ self.assertEqual(info['object_count'], 0)
+ self.assertEqual(info['bytes_used'], 0)
+
def test_creation(self):
# Test swift.common.db.ContainerBroker.__init__
broker = self._get_broker(account='a', container='c')
@@ -814,10 +845,10 @@ class TestContainerBroker(unittest.TestCase):
out_content_type="text/plain")
self.assertEquals(len(listing), 100)
for (name, ts, clen, ctype, etag) in listing:
- self.assertEqual(ts, 0)
+ self.assertEqual(ts, '0')
self.assertEqual(clen, 0)
- self.assertEqual(ctype, 0)
- self.assertEqual(etag, 0)
+ self.assertEqual(ctype, 'text/plain')
+ self.assertEqual(etag, '')
# Check that limit is still honored.
listing = broker.list_objects_iter(25, '', None, None, '',
@@ -1343,6 +1374,39 @@ class TestDiskAccount(unittest.TestCase):
_destroyxattr()
shutil.rmtree(self.td)
+ def test_gsexpiring_fake_md(self):
+ # Create account
+ account_path = os.path.join(self.td, "gsexpiring")
+ os.makedirs(account_path)
+
+ # Create 10 containers - these should not reflect in
+ # X-Container-Count. Also, create 10 objects in each
+ # container. These should not reflect in X-Object-Count.
+ for c in xrange(10):
+ cpath = os.path.join(account_path, str(c))
+ os.mkdir(cpath)
+ for o in xrange(10):
+ os.mkdir(os.path.join(cpath, str(o)))
+
+ orig_stat = os.stat(account_path)
+ expected_metadata = {
+ 'X-Object-Count': (0, 0),
+ 'X-Container-Count': (0, 0),
+ 'X-Timestamp': ((normalize_timestamp(orig_stat.st_ctime)), 0),
+ 'X-Type': ('Account', 0),
+ 'X-PUT-Timestamp': ((normalize_timestamp(orig_stat.st_mtime)), 0),
+ 'X-Bytes-Used': (0, 0)
+ }
+
+ # Create DiskAccount instance
+ da = dd.DiskAccount(self.td, 'gsexpiring', 'gsexpiring',
+ self.fake_logger)
+ self.assertEqual(expected_metadata, da.metadata)
+ info = da.get_info()
+ self.assertEqual(info['container_count'], 0)
+ self.assertEqual(info['object_count'], 0)
+ self.assertEqual(info['bytes_used'], 0)
+
def test_constructor_no_metadata(self):
da = dd.DiskAccount(self.td, self.fake_drives[0],
self.fake_accounts[0], self.fake_logger)