From 5a04cede1f5bb44d6c64b186335146dd4e70a6ea Mon Sep 17 00:00:00 2001 From: Prashanth Pai Date: Wed, 20 Apr 2016 15:10:43 +0530 Subject: Make swift's expirer compatible with gluster-swift Swift's object expirer in kilo series was incompatible with gluster-swift. This change does the following: * Optimizes crawl in account and container server for listing requests for containers and tracker objects in gsexpiring volume. * Enables container server to delete tracker objects from gsexpiring volume. Swift's expirer sends request directly to container server to remove tracker object entry. * delete_tracker_object() is now a common utility function that is invoked from container server and gluster-swift's object expirer. * Run functional test to be run against both swift's object expirer and gluster-swift's object expirer Change-Id: Ib5b7f7f08fe7dda574f6dd80be2f38bdfaee32bc Signed-off-by: Prashanth Pai Reviewed-on: http://review.gluster.org/14038 Reviewed-by: Thiago da Silva Tested-by: Thiago da Silva --- .../test_object_expirer.py | 337 --------------------- .../test_object_expirer_gluster_swift.py | 337 +++++++++++++++++++++ .../test_object_expirer_swift.py | 335 ++++++++++++++++++++ test/unit/common/test_diskdir.py | 70 ++++- 4 files changed, 739 insertions(+), 340 deletions(-) delete mode 100644 test/object_expirer_functional/test_object_expirer.py create mode 100644 test/object_expirer_functional/test_object_expirer_gluster_swift.py create mode 100644 test/object_expirer_functional/test_object_expirer_swift.py (limited to 'test') diff --git a/test/object_expirer_functional/test_object_expirer.py b/test/object_expirer_functional/test_object_expirer.py deleted file mode 100644 index 279994f..0000000 --- a/test/object_expirer_functional/test_object_expirer.py +++ /dev/null @@ -1,337 +0,0 @@ -# Copyright (c) 2014 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import time -import logging - -from gluster.swift.obj.expirer import ObjectExpirer, GlusterSwiftInternalClient - -from swift.common.utils import readconf - -from test import get_config -from test.functional.tests import Base, Utils -from test.functional.swift_test_client import Account, Connection, \ - ResponseError - -config = get_config('func_test') - - -class TestObjectExpirerEnv: - @classmethod - def setUp(cls): - cls.conn = Connection(config) - cls.conn.authenticate() - cls.account = Account(cls.conn, - config.get('account', - config['username'])) - cls.account.delete_containers() - cls.container = cls.account.container(Utils.create_name()) - if not cls.container.create(): - raise ResponseError(cls.conn.response) - cls.file_size = 8 - cls.root_dir = os.path.join('/mnt/gluster-object', - cls.account.conn.storage_url.split('/')[2].split('_')[1]) - devices = config.get('devices', '/mnt/gluster-object') - cls.client = GlusterSwiftInternalClient('/etc/swift/object-expirer.conf', - 'Test Object Expirer', 1, - devices=devices) - conf = readconf('/etc/swift/object-expirer.conf', 'object-expirer') - cls.expirer = ObjectExpirer(conf) - - -class TestObjectExpirer(Base): - env = TestObjectExpirerEnv - set_up = False - - def test_object_expiry_X_Delete_At_PUT(self): - obj = self.env.container.file(Utils.create_name()) - x_delete_at = str(int(time.time()) + 2) - obj.write_random(self.env.file_size, - hdrs={'X-Delete-At': x_delete_at}) - - # Object is not expired. Should still be accessible. - obj.read() - self.assert_status(200) - - # Ensure X-Delete-At is saved as object metadata. - self.assertEqual(x_delete_at, str(obj.info()['x_delete_at'])) - - # Wait for object to be expired. - time.sleep(3) - - # Object has expired. Should no longer be accessible. - self.assertRaises(ResponseError, obj.read) - self.assert_status(404) - - # Object should still be present on filesystem. - self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir, - self.env.container.name, - obj.name))) - - # But, GET on container should list the expired object. - result = self.env.container.files() - self.assertTrue(obj.name in self.env.container.files()) - - # Check existence of corresponding tracker object in gsexpiring - # account. - enteredLoop = False - for c in self.env.client.iter_containers("gsexpiring"): - for o in self.env.client.iter_objects("gsexpiring", c['name']): - enteredLoop = True - l = o['name'].split('/') - self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name)) - self.assertEqual(l[1], self.env.container.name) - self.assertEqual(l[2], obj.name) - if not enteredLoop: - self.fail("Tracker object not found.") - - # Run expirer daemon once. - self.env.expirer.run_once() - - # Ensure object is physically deleted from filesystem. - self.assertFalse(os.path.exists(os.path.join(self.env.root_dir, - self.env.container.name, - obj.name))) - - # Ensure tracker object is consumed. - try: - self.env.client.iter_containers("gsexpiring").next() - except StopIteration: - pass - else: - self.fail("Tracker object persists!") - - # GET on container should no longer list the object. - self.assertFalse(obj.name in self.env.container.files()) - - def test_object_expiry_X_Delete_After_PUT(self): - obj = self.env.container.file(Utils.create_name()) - obj.write_random(self.env.file_size, - hdrs={'X-Delete-After': 2}) - - # Object is not expired. Should still be accessible. - obj.read() - self.assert_status(200) - - # Ensure X-Delete-At is saved as object metadata. - self.assertTrue(str(obj.info()['x_delete_at'])) - - # Wait for object to be expired. - time.sleep(3) - - # Object has expired. Should no longer be accessible. - self.assertRaises(ResponseError, obj.read) - self.assert_status(404) - - # Object should still be present on filesystem. - self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir, - self.env.container.name, - obj.name))) - - # But, GET on container should list the expired object. - result = self.env.container.files() - self.assertTrue(obj.name in self.env.container.files()) - - # Check existence of corresponding tracker object in gsexpiring - # account. - enteredLoop = False - for c in self.env.client.iter_containers("gsexpiring"): - for o in self.env.client.iter_objects("gsexpiring", c['name']): - enteredLoop = True - l = o['name'].split('/') - self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name)) - self.assertEqual(l[1], self.env.container.name) - self.assertEqual(l[2], obj.name) - if not enteredLoop: - self.fail("Tracker object not found.") - - # Run expirer daemon once. - self.env.expirer.run_once() - - # Ensure object is physically deleted from filesystem. - self.assertFalse(os.path.exists(os.path.join(self.env.root_dir, - self.env.container.name, - obj.name))) - - # Ensure tracker object is consumed. - try: - self.env.client.iter_containers("gsexpiring").next() - except StopIteration: - pass - else: - self.fail("Tracker object persists!") - - # GET on container should no longer list the object. - self.assertFalse(obj.name in self.env.container.files()) - - def test_object_expiry_X_Delete_At_POST(self): - - # Create normal object - obj = self.env.container.file(Utils.create_name()) - obj.write_random(self.env.file_size) - obj.read() - self.assert_status(200) - - # Send POST on that object and set it to be expired. - x_delete_at = str(int(time.time()) + 2) - obj.sync_metadata(metadata={'X-Delete-At': x_delete_at}, - cfg={'x_delete_at': x_delete_at}) - - # Ensure X-Delete-At is saved as object metadata. - self.assertEqual(x_delete_at, str(obj.info()['x_delete_at'])) - - # Object is not expired. Should still be accessible. - obj.read() - self.assert_status(200) - - # Wait for object to be expired. - time.sleep(3) - - # Object has expired. Should no longer be accessible. - self.assertRaises(ResponseError, obj.read) - self.assert_status(404) - - # Object should still be present on filesystem. - self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir, - self.env.container.name, - obj.name))) - - # But, GET on container should list the expired object. - result = self.env.container.files() - self.assertTrue(obj.name in self.env.container.files()) - - # Check existence of corresponding tracker object in gsexpiring - # account. - - enteredLoop = False - for c in self.env.client.iter_containers("gsexpiring"): - for o in self.env.client.iter_objects("gsexpiring", c['name']): - enteredLoop = True - l = o['name'].split('/') - self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name)) - self.assertEqual(l[1], self.env.container.name) - self.assertEqual(l[2], obj.name) - if not enteredLoop: - self.fail("Tracker object not found.") - - # Run expirer daemon once. - self.env.expirer.run_once() - - # Ensure object is physically deleted from filesystem. - self.assertFalse(os.path.exists(os.path.join(self.env.root_dir, - self.env.container.name, - obj.name))) - - # Ensure tracker object is consumed. - try: - self.env.client.iter_containers("gsexpiring").next() - except StopIteration: - pass - else: - self.fail("Tracker object persists!") - - # GET on container should no longer list the object. - self.assertFalse(obj.name in self.env.container.files()) - - - def test_object_expiry_X_Delete_After_POST(self): - - # Create normal object - obj = self.env.container.file(Utils.create_name()) - obj.write_random(self.env.file_size) - obj.read() - self.assert_status(200) - - # Send POST on that object and set it to be expired. - obj.sync_metadata(metadata={'X-Delete-After': 2}, - cfg={'x_delete_after': 2}) - - # Ensure X-Delete-At is saved as object metadata. - self.assertTrue(str(obj.info()['x_delete_at'])) - - # Object is not expired. Should still be accessible. - obj.read() - self.assert_status(200) - - # Wait for object to be expired. - time.sleep(3) - - # Object has expired. Should no longer be accessible. - self.assertRaises(ResponseError, obj.read) - self.assert_status(404) - - # Object should still be present on filesystem. - self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir, - self.env.container.name, - obj.name))) - - # But, GET on container should list the expired object. - result = self.env.container.files() - self.assertTrue(obj.name in self.env.container.files()) - - # Check existence of corresponding tracker object in gsexpiring - # account. - - enteredLoop = False - for c in self.env.client.iter_containers("gsexpiring"): - for o in self.env.client.iter_objects("gsexpiring", c['name']): - enteredLoop = True - l = o['name'].split('/') - self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name)) - self.assertEqual(l[1], self.env.container.name) - self.assertEqual(l[2], obj.name) - if not enteredLoop: - self.fail("Tracker object not found.") - - # Run expirer daemon once. - self.env.expirer.run_once() - - # Ensure object is physically deleted from filesystem. - self.assertFalse(os.path.exists(os.path.join(self.env.root_dir, - self.env.container.name, - obj.name))) - - # Ensure tracker object is consumed. - try: - self.env.client.iter_containers("gsexpiring").next() - except StopIteration: - pass - else: - self.fail("Tracker object persists!") - - # GET on container should no longer list the object. - self.assertFalse(obj.name in self.env.container.files()) - - - def test_object_expiry_err(self): - obj = self.env.container.file(Utils.create_name()) - - # X-Delete-At is invalid or is in the past - for i in (-2, 'abc', str(int(time.time()) - 2), 5.8): - self.assertRaises(ResponseError, - obj.write_random, - self.env.file_size, - hdrs={'X-Delete-At': i}) - self.assert_status(400) - - # X-Delete-After is invalid. - for i in (-2, 'abc', 3.7): - self.assertRaises(ResponseError, - obj.write_random, - self.env.file_size, - hdrs={'X-Delete-After': i}) - self.assert_status(400) - diff --git a/test/object_expirer_functional/test_object_expirer_gluster_swift.py b/test/object_expirer_functional/test_object_expirer_gluster_swift.py new file mode 100644 index 0000000..279994f --- /dev/null +++ b/test/object_expirer_functional/test_object_expirer_gluster_swift.py @@ -0,0 +1,337 @@ +# Copyright (c) 2014 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time +import logging + +from gluster.swift.obj.expirer import ObjectExpirer, GlusterSwiftInternalClient + +from swift.common.utils import readconf + +from test import get_config +from test.functional.tests import Base, Utils +from test.functional.swift_test_client import Account, Connection, \ + ResponseError + +config = get_config('func_test') + + +class TestObjectExpirerEnv: + @classmethod + def setUp(cls): + cls.conn = Connection(config) + cls.conn.authenticate() + cls.account = Account(cls.conn, + config.get('account', + config['username'])) + cls.account.delete_containers() + cls.container = cls.account.container(Utils.create_name()) + if not cls.container.create(): + raise ResponseError(cls.conn.response) + cls.file_size = 8 + cls.root_dir = os.path.join('/mnt/gluster-object', + cls.account.conn.storage_url.split('/')[2].split('_')[1]) + devices = config.get('devices', '/mnt/gluster-object') + cls.client = GlusterSwiftInternalClient('/etc/swift/object-expirer.conf', + 'Test Object Expirer', 1, + devices=devices) + conf = readconf('/etc/swift/object-expirer.conf', 'object-expirer') + cls.expirer = ObjectExpirer(conf) + + +class TestObjectExpirer(Base): + env = TestObjectExpirerEnv + set_up = False + + def test_object_expiry_X_Delete_At_PUT(self): + obj = self.env.container.file(Utils.create_name()) + x_delete_at = str(int(time.time()) + 2) + obj.write_random(self.env.file_size, + hdrs={'X-Delete-At': x_delete_at}) + + # Object is not expired. Should still be accessible. + obj.read() + self.assert_status(200) + + # Ensure X-Delete-At is saved as object metadata. + self.assertEqual(x_delete_at, str(obj.info()['x_delete_at'])) + + # Wait for object to be expired. + time.sleep(3) + + # Object has expired. Should no longer be accessible. + self.assertRaises(ResponseError, obj.read) + self.assert_status(404) + + # Object should still be present on filesystem. + self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # But, GET on container should list the expired object. + result = self.env.container.files() + self.assertTrue(obj.name in self.env.container.files()) + + # Check existence of corresponding tracker object in gsexpiring + # account. + enteredLoop = False + for c in self.env.client.iter_containers("gsexpiring"): + for o in self.env.client.iter_objects("gsexpiring", c['name']): + enteredLoop = True + l = o['name'].split('/') + self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name)) + self.assertEqual(l[1], self.env.container.name) + self.assertEqual(l[2], obj.name) + if not enteredLoop: + self.fail("Tracker object not found.") + + # Run expirer daemon once. + self.env.expirer.run_once() + + # Ensure object is physically deleted from filesystem. + self.assertFalse(os.path.exists(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # Ensure tracker object is consumed. + try: + self.env.client.iter_containers("gsexpiring").next() + except StopIteration: + pass + else: + self.fail("Tracker object persists!") + + # GET on container should no longer list the object. + self.assertFalse(obj.name in self.env.container.files()) + + def test_object_expiry_X_Delete_After_PUT(self): + obj = self.env.container.file(Utils.create_name()) + obj.write_random(self.env.file_size, + hdrs={'X-Delete-After': 2}) + + # Object is not expired. Should still be accessible. + obj.read() + self.assert_status(200) + + # Ensure X-Delete-At is saved as object metadata. + self.assertTrue(str(obj.info()['x_delete_at'])) + + # Wait for object to be expired. + time.sleep(3) + + # Object has expired. Should no longer be accessible. + self.assertRaises(ResponseError, obj.read) + self.assert_status(404) + + # Object should still be present on filesystem. + self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # But, GET on container should list the expired object. + result = self.env.container.files() + self.assertTrue(obj.name in self.env.container.files()) + + # Check existence of corresponding tracker object in gsexpiring + # account. + enteredLoop = False + for c in self.env.client.iter_containers("gsexpiring"): + for o in self.env.client.iter_objects("gsexpiring", c['name']): + enteredLoop = True + l = o['name'].split('/') + self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name)) + self.assertEqual(l[1], self.env.container.name) + self.assertEqual(l[2], obj.name) + if not enteredLoop: + self.fail("Tracker object not found.") + + # Run expirer daemon once. + self.env.expirer.run_once() + + # Ensure object is physically deleted from filesystem. + self.assertFalse(os.path.exists(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # Ensure tracker object is consumed. + try: + self.env.client.iter_containers("gsexpiring").next() + except StopIteration: + pass + else: + self.fail("Tracker object persists!") + + # GET on container should no longer list the object. + self.assertFalse(obj.name in self.env.container.files()) + + def test_object_expiry_X_Delete_At_POST(self): + + # Create normal object + obj = self.env.container.file(Utils.create_name()) + obj.write_random(self.env.file_size) + obj.read() + self.assert_status(200) + + # Send POST on that object and set it to be expired. + x_delete_at = str(int(time.time()) + 2) + obj.sync_metadata(metadata={'X-Delete-At': x_delete_at}, + cfg={'x_delete_at': x_delete_at}) + + # Ensure X-Delete-At is saved as object metadata. + self.assertEqual(x_delete_at, str(obj.info()['x_delete_at'])) + + # Object is not expired. Should still be accessible. + obj.read() + self.assert_status(200) + + # Wait for object to be expired. + time.sleep(3) + + # Object has expired. Should no longer be accessible. + self.assertRaises(ResponseError, obj.read) + self.assert_status(404) + + # Object should still be present on filesystem. + self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # But, GET on container should list the expired object. + result = self.env.container.files() + self.assertTrue(obj.name in self.env.container.files()) + + # Check existence of corresponding tracker object in gsexpiring + # account. + + enteredLoop = False + for c in self.env.client.iter_containers("gsexpiring"): + for o in self.env.client.iter_objects("gsexpiring", c['name']): + enteredLoop = True + l = o['name'].split('/') + self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name)) + self.assertEqual(l[1], self.env.container.name) + self.assertEqual(l[2], obj.name) + if not enteredLoop: + self.fail("Tracker object not found.") + + # Run expirer daemon once. + self.env.expirer.run_once() + + # Ensure object is physically deleted from filesystem. + self.assertFalse(os.path.exists(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # Ensure tracker object is consumed. + try: + self.env.client.iter_containers("gsexpiring").next() + except StopIteration: + pass + else: + self.fail("Tracker object persists!") + + # GET on container should no longer list the object. + self.assertFalse(obj.name in self.env.container.files()) + + + def test_object_expiry_X_Delete_After_POST(self): + + # Create normal object + obj = self.env.container.file(Utils.create_name()) + obj.write_random(self.env.file_size) + obj.read() + self.assert_status(200) + + # Send POST on that object and set it to be expired. + obj.sync_metadata(metadata={'X-Delete-After': 2}, + cfg={'x_delete_after': 2}) + + # Ensure X-Delete-At is saved as object metadata. + self.assertTrue(str(obj.info()['x_delete_at'])) + + # Object is not expired. Should still be accessible. + obj.read() + self.assert_status(200) + + # Wait for object to be expired. + time.sleep(3) + + # Object has expired. Should no longer be accessible. + self.assertRaises(ResponseError, obj.read) + self.assert_status(404) + + # Object should still be present on filesystem. + self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # But, GET on container should list the expired object. + result = self.env.container.files() + self.assertTrue(obj.name in self.env.container.files()) + + # Check existence of corresponding tracker object in gsexpiring + # account. + + enteredLoop = False + for c in self.env.client.iter_containers("gsexpiring"): + for o in self.env.client.iter_objects("gsexpiring", c['name']): + enteredLoop = True + l = o['name'].split('/') + self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name)) + self.assertEqual(l[1], self.env.container.name) + self.assertEqual(l[2], obj.name) + if not enteredLoop: + self.fail("Tracker object not found.") + + # Run expirer daemon once. + self.env.expirer.run_once() + + # Ensure object is physically deleted from filesystem. + self.assertFalse(os.path.exists(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # Ensure tracker object is consumed. + try: + self.env.client.iter_containers("gsexpiring").next() + except StopIteration: + pass + else: + self.fail("Tracker object persists!") + + # GET on container should no longer list the object. + self.assertFalse(obj.name in self.env.container.files()) + + + def test_object_expiry_err(self): + obj = self.env.container.file(Utils.create_name()) + + # X-Delete-At is invalid or is in the past + for i in (-2, 'abc', str(int(time.time()) - 2), 5.8): + self.assertRaises(ResponseError, + obj.write_random, + self.env.file_size, + hdrs={'X-Delete-At': i}) + self.assert_status(400) + + # X-Delete-After is invalid. + for i in (-2, 'abc', 3.7): + self.assertRaises(ResponseError, + obj.write_random, + self.env.file_size, + hdrs={'X-Delete-After': i}) + self.assert_status(400) + diff --git a/test/object_expirer_functional/test_object_expirer_swift.py b/test/object_expirer_functional/test_object_expirer_swift.py new file mode 100644 index 0000000..c59cc67 --- /dev/null +++ b/test/object_expirer_functional/test_object_expirer_swift.py @@ -0,0 +1,335 @@ +# Copyright (c) 2014 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time +import logging + +from swift.common.manager import Manager +from swift.common.internal_client import InternalClient + + +from test import get_config +from test.functional.tests import Base, Utils +from test.functional.swift_test_client import Account, Connection, \ + ResponseError + +config = get_config('func_test') + + +class TestObjectExpirerEnv: + @classmethod + def setUp(cls): + cls.conn = Connection(config) + cls.conn.authenticate() + cls.account = Account(cls.conn, + config.get('account', + config['username'])) + cls.account.delete_containers() + cls.container = cls.account.container(Utils.create_name()) + if not cls.container.create(): + raise ResponseError(cls.conn.response) + cls.file_size = 8 + cls.root_dir = os.path.join('/mnt/gluster-object', + cls.account.conn.storage_url.split('/')[2].split('_')[1]) + devices = config.get('devices', '/mnt/gluster-object') + cls.client = InternalClient('/etc/swift/object-expirer.conf', + 'Test Object Expirer', 1) + cls.expirer = Manager(['object-expirer']) + + +class TestObjectExpirer(Base): + env = TestObjectExpirerEnv + set_up = False + + def test_object_expiry_X_Delete_At_PUT(self): + obj = self.env.container.file(Utils.create_name()) + x_delete_at = str(int(time.time()) + 2) + obj.write_random(self.env.file_size, + hdrs={'X-Delete-At': x_delete_at}) + + # Object is not expired. Should still be accessible. + obj.read() + self.assert_status(200) + + # Ensure X-Delete-At is saved as object metadata. + self.assertEqual(x_delete_at, str(obj.info()['x_delete_at'])) + + # Wait for object to be expired. + time.sleep(3) + + # Object has expired. Should no longer be accessible. + self.assertRaises(ResponseError, obj.read) + self.assert_status(404) + + # Object should still be present on filesystem. + self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # But, GET on container should list the expired object. + result = self.env.container.files() + self.assertTrue(obj.name in self.env.container.files()) + + # Check existence of corresponding tracker object in gsexpiring + # account. + enteredLoop = False + for c in self.env.client.iter_containers("gsexpiring"): + for o in self.env.client.iter_objects("gsexpiring", c['name']): + enteredLoop = True + l = o['name'].split('/') + self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name)) + self.assertEqual(l[1], self.env.container.name) + self.assertEqual(l[2], obj.name) + if not enteredLoop: + self.fail("Tracker object not found.") + + # Run expirer daemon once. + self.env.expirer.once() + + # Ensure object is physically deleted from filesystem. + self.assertFalse(os.path.exists(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # Ensure tracker object is consumed. + try: + self.env.client.iter_containers("gsexpiring").next() + except StopIteration: + pass + else: + self.fail("Tracker object persists!") + + # GET on container should no longer list the object. + self.assertFalse(obj.name in self.env.container.files()) + + def test_object_expiry_X_Delete_After_PUT(self): + obj = self.env.container.file(Utils.create_name()) + obj.write_random(self.env.file_size, + hdrs={'X-Delete-After': 2}) + + # Object is not expired. Should still be accessible. + obj.read() + self.assert_status(200) + + # Ensure X-Delete-At is saved as object metadata. + self.assertTrue(str(obj.info()['x_delete_at'])) + + # Wait for object to be expired. + time.sleep(3) + + # Object has expired. Should no longer be accessible. + self.assertRaises(ResponseError, obj.read) + self.assert_status(404) + + # Object should still be present on filesystem. + self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # But, GET on container should list the expired object. + result = self.env.container.files() + self.assertTrue(obj.name in self.env.container.files()) + + # Check existence of corresponding tracker object in gsexpiring + # account. + enteredLoop = False + for c in self.env.client.iter_containers("gsexpiring"): + for o in self.env.client.iter_objects("gsexpiring", c['name']): + enteredLoop = True + l = o['name'].split('/') + self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name)) + self.assertEqual(l[1], self.env.container.name) + self.assertEqual(l[2], obj.name) + if not enteredLoop: + self.fail("Tracker object not found.") + + # Run expirer daemon once. + self.env.expirer.once() + + # Ensure object is physically deleted from filesystem. + self.assertFalse(os.path.exists(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # Ensure tracker object is consumed. + try: + self.env.client.iter_containers("gsexpiring").next() + except StopIteration: + pass + else: + self.fail("Tracker object persists!") + + # GET on container should no longer list the object. + self.assertFalse(obj.name in self.env.container.files()) + + def test_object_expiry_X_Delete_At_POST(self): + + # Create normal object + obj = self.env.container.file(Utils.create_name()) + obj.write_random(self.env.file_size) + obj.read() + self.assert_status(200) + + # Send POST on that object and set it to be expired. + x_delete_at = str(int(time.time()) + 2) + obj.sync_metadata(metadata={'X-Delete-At': x_delete_at}, + cfg={'x_delete_at': x_delete_at}) + + # Ensure X-Delete-At is saved as object metadata. + self.assertEqual(x_delete_at, str(obj.info()['x_delete_at'])) + + # Object is not expired. Should still be accessible. + obj.read() + self.assert_status(200) + + # Wait for object to be expired. + time.sleep(3) + + # Object has expired. Should no longer be accessible. + self.assertRaises(ResponseError, obj.read) + self.assert_status(404) + + # Object should still be present on filesystem. + self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # But, GET on container should list the expired object. + result = self.env.container.files() + self.assertTrue(obj.name in self.env.container.files()) + + # Check existence of corresponding tracker object in gsexpiring + # account. + + enteredLoop = False + for c in self.env.client.iter_containers("gsexpiring"): + for o in self.env.client.iter_objects("gsexpiring", c['name']): + enteredLoop = True + l = o['name'].split('/') + self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name)) + self.assertEqual(l[1], self.env.container.name) + self.assertEqual(l[2], obj.name) + if not enteredLoop: + self.fail("Tracker object not found.") + + # Run expirer daemon once. + self.env.expirer.once() + + # Ensure object is physically deleted from filesystem. + self.assertFalse(os.path.exists(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # Ensure tracker object is consumed. + try: + self.env.client.iter_containers("gsexpiring").next() + except StopIteration: + pass + else: + self.fail("Tracker object persists!") + + # GET on container should no longer list the object. + self.assertFalse(obj.name in self.env.container.files()) + + + def test_object_expiry_X_Delete_After_POST(self): + + # Create normal object + obj = self.env.container.file(Utils.create_name()) + obj.write_random(self.env.file_size) + obj.read() + self.assert_status(200) + + # Send POST on that object and set it to be expired. + obj.sync_metadata(metadata={'X-Delete-After': 2}, + cfg={'x_delete_after': 2}) + + # Ensure X-Delete-At is saved as object metadata. + self.assertTrue(str(obj.info()['x_delete_at'])) + + # Object is not expired. Should still be accessible. + obj.read() + self.assert_status(200) + + # Wait for object to be expired. + time.sleep(3) + + # Object has expired. Should no longer be accessible. + self.assertRaises(ResponseError, obj.read) + self.assert_status(404) + + # Object should still be present on filesystem. + self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # But, GET on container should list the expired object. + result = self.env.container.files() + self.assertTrue(obj.name in self.env.container.files()) + + # Check existence of corresponding tracker object in gsexpiring + # account. + + enteredLoop = False + for c in self.env.client.iter_containers("gsexpiring"): + for o in self.env.client.iter_objects("gsexpiring", c['name']): + enteredLoop = True + l = o['name'].split('/') + self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name)) + self.assertEqual(l[1], self.env.container.name) + self.assertEqual(l[2], obj.name) + if not enteredLoop: + self.fail("Tracker object not found.") + + # Run expirer daemon once. + self.env.expirer.once() + + # Ensure object is physically deleted from filesystem. + self.assertFalse(os.path.exists(os.path.join(self.env.root_dir, + self.env.container.name, + obj.name))) + + # Ensure tracker object is consumed. + try: + self.env.client.iter_containers("gsexpiring").next() + except StopIteration: + pass + else: + self.fail("Tracker object persists!") + + # GET on container should no longer list the object. + self.assertFalse(obj.name in self.env.container.files()) + + + def test_object_expiry_err(self): + obj = self.env.container.file(Utils.create_name()) + + # X-Delete-At is invalid or is in the past + for i in (-2, 'abc', str(int(time.time()) - 2), 5.8): + self.assertRaises(ResponseError, + obj.write_random, + self.env.file_size, + hdrs={'X-Delete-At': i}) + self.assert_status(400) + + # X-Delete-After is invalid. + for i in (-2, 'abc', 3.7): + self.assertRaises(ResponseError, + obj.write_random, + self.env.file_size, + hdrs={'X-Delete-After': i}) + self.assert_status(400) + diff --git a/test/unit/common/test_diskdir.py b/test/unit/common/test_diskdir.py index 623164c..964bc2f 100644 --- a/test/unit/common/test_diskdir.py +++ b/test/unit/common/test_diskdir.py @@ -402,6 +402,37 @@ class TestContainerBroker(unittest.TestCase): fp.write("file path: %s\n" % fullname) return fullname + def test_gsexpiring_fake_md(self): + # Create account + account_path = os.path.join(self.path, "gsexpiring") + os.makedirs(account_path) + + # Create container + cpath = os.path.join(account_path, "container") + os.mkdir(cpath) + + # Create 10 objects in container. These should not reflect in + # X-Object-Count. + for o in xrange(10): + os.mkdir(os.path.join(cpath, str(o))) + + orig_stat = os.stat(cpath) + expected_metadata = { + 'X-Object-Count': (0, 0), + 'X-Timestamp': ((normalize_timestamp(orig_stat.st_ctime)), 0), + 'X-Type': ('container', 0), + 'X-PUT-Timestamp': ((normalize_timestamp(orig_stat.st_mtime)), 0), + 'X-Bytes-Used': (0, 0) + } + + # Create DiskDir instance + disk_dir = dd.DiskDir(self.path, 'gsexpiring', account='gsexpiring', + container='container', logger=FakeLogger()) + self.assertEqual(expected_metadata, disk_dir.metadata) + info = disk_dir.get_info() + self.assertEqual(info['object_count'], 0) + self.assertEqual(info['bytes_used'], 0) + def test_creation(self): # Test swift.common.db.ContainerBroker.__init__ broker = self._get_broker(account='a', container='c') @@ -814,10 +845,10 @@ class TestContainerBroker(unittest.TestCase): out_content_type="text/plain") self.assertEquals(len(listing), 100) for (name, ts, clen, ctype, etag) in listing: - self.assertEqual(ts, 0) + self.assertEqual(ts, '0') self.assertEqual(clen, 0) - self.assertEqual(ctype, 0) - self.assertEqual(etag, 0) + self.assertEqual(ctype, 'text/plain') + self.assertEqual(etag, '') # Check that limit is still honored. listing = broker.list_objects_iter(25, '', None, None, '', @@ -1343,6 +1374,39 @@ class TestDiskAccount(unittest.TestCase): _destroyxattr() shutil.rmtree(self.td) + def test_gsexpiring_fake_md(self): + # Create account + account_path = os.path.join(self.td, "gsexpiring") + os.makedirs(account_path) + + # Create 10 containers - these should not reflect in + # X-Container-Count. Also, create 10 objects in each + # container. These should not reflect in X-Object-Count. + for c in xrange(10): + cpath = os.path.join(account_path, str(c)) + os.mkdir(cpath) + for o in xrange(10): + os.mkdir(os.path.join(cpath, str(o))) + + orig_stat = os.stat(account_path) + expected_metadata = { + 'X-Object-Count': (0, 0), + 'X-Container-Count': (0, 0), + 'X-Timestamp': ((normalize_timestamp(orig_stat.st_ctime)), 0), + 'X-Type': ('Account', 0), + 'X-PUT-Timestamp': ((normalize_timestamp(orig_stat.st_mtime)), 0), + 'X-Bytes-Used': (0, 0) + } + + # Create DiskAccount instance + da = dd.DiskAccount(self.td, 'gsexpiring', 'gsexpiring', + self.fake_logger) + self.assertEqual(expected_metadata, da.metadata) + info = da.get_info() + self.assertEqual(info['container_count'], 0) + self.assertEqual(info['object_count'], 0) + self.assertEqual(info['bytes_used'], 0) + def test_constructor_no_metadata(self): da = dd.DiskAccount(self.td, self.fake_drives[0], self.fake_accounts[0], self.fake_logger) -- cgit