summaryrefslogtreecommitdiffstats
path: root/gluster/swift/obj/expirer.py
diff options
context:
space:
mode:
Diffstat (limited to 'gluster/swift/obj/expirer.py')
-rw-r--r--gluster/swift/obj/expirer.py160
1 files changed, 160 insertions, 0 deletions
diff --git a/gluster/swift/obj/expirer.py b/gluster/swift/obj/expirer.py
new file mode 100644
index 0000000..564a2c9
--- /dev/null
+++ b/gluster/swift/obj/expirer.py
@@ -0,0 +1,160 @@
+# Copyright (c) 2016 Red Hat
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This import will monkey-patch Ring and other classes.
+# Do not remove.
+import gluster.swift.common.constraints # noqa
+
+import errno
+import os
+
+from gluster.swift.common.utils import rmobjdir
+
+from swift.obj.expirer import ObjectExpirer as SwiftObjectExpirer
+from swift.common.http import HTTP_NOT_FOUND
+from swift.common.internal_client import InternalClient, UnexpectedResponse
+from swift.common.utils import ThreadPool
+
+EXCLUDE_DIRS = ('.trashcan', '.glusterfs')
+
+
+class GlusterSwiftInternalClient(InternalClient):
+
+ def __init__(self, conf_path, user_agent, request_tries,
+ allow_modify_pipeline=False, devices=None):
+ super(GlusterSwiftInternalClient, self).__init__(
+ conf_path, user_agent, request_tries, allow_modify_pipeline)
+ self.devices = devices
+
+ def get_account_info(self, account):
+ # Supposed to return container count and object count in gsexpiring
+ # account. This is used by object expirer only for logging.
+ return (0, 0)
+
+ def delete_container(self, account, container, acceptable_statuses=None):
+ container_path = os.path.join(self.devices, account, container)
+ try:
+ os.rmdir(container_path)
+ except OSError as err:
+ if err.errno != errno.ENOENT:
+ raise
+
+ def iter_containers(self, account):
+ account_path = os.path.join(self.devices, account)
+ for container in os.listdir(account_path):
+ if container in EXCLUDE_DIRS:
+ continue
+ container_path = os.path.join(account_path, container)
+ if os.path.isdir(container_path):
+ yield {'name': container.encode('utf8')}
+
+ def iter_objects(self, account, container):
+ container_path = os.path.join(self.devices, account, container)
+ # TODO: Use a slightly better implementation of os.walk()
+ for (root, dirs, files) in os.walk(container_path):
+ for f in files:
+ obj_path = os.path.join(root, f)
+ obj = obj_path[(len(container_path) + 1):]
+ yield {'name': obj.encode('utf8')}
+
+
+class ObjectExpirer(SwiftObjectExpirer):
+
+ def __init__(self, conf, logger=None, swift=None):
+
+ conf_path = conf.get('__file__') or '/etc/swift/object-expirer.conf'
+ self.devices = conf.get('devices', '/mnt/gluster-object')
+ # Do not retry DELETEs on getting 404. Hence default is set to 1.
+ request_tries = int(conf.get('request_tries') or 1)
+ # Use our extended version of InternalClient
+ swift = GlusterSwiftInternalClient(
+ conf_path, 'Gluster Swift Object Expirer', request_tries,
+ devices=self.devices)
+ # Let the parent class initialize self.swift
+ super(ObjectExpirer, self).__init__(conf, logger=logger, swift=swift)
+
+ self.reseller_prefix = conf.get('reseller_prefix', 'AUTH').strip()
+ if not self.reseller_prefix.endswith('_'):
+ self.reseller_prefix = self.reseller_prefix + '_'
+
+ # nthread=0 is intentional. This ensures that no green pool is
+ # used. Call to force_run_in_thread() will ensure that the method
+ # passed as arg is run in a real external thread using eventlet.tpool
+ # which has a threadpool of 20 threads (default)
+ self.threadpool = ThreadPool(nthreads=0)
+
+ def _delete_tracker_object(self, container, obj):
+ container_path = os.path.join(self.devices,
+ self.expiring_objects_account,
+ container)
+ tracker_object_path = os.path.join(container_path, obj)
+
+ try:
+ os.unlink(tracker_object_path)
+ except OSError as err:
+ if err.errno in (errno.ENOENT, errno.ESTALE):
+ # Ignore removal from another entity.
+ return
+ elif err.errno == errno.EISDIR:
+ # Handle race: Was a file during crawl, but now it's a
+ # directory. There are no 'directory marker' objects in
+ # gsexpiring volume.
+ return
+ else:
+ raise
+
+ # This part of code is very similar to DiskFile._unlinkold()
+ dirname = os.path.dirname(tracker_object_path)
+ while dirname and dirname != container_path:
+ if not rmobjdir(dirname, marker_dir_check=False):
+ # If a directory with objects has been found, we can stop
+ # garbage collection
+ break
+ else:
+ # Traverse upwards till the root of container
+ dirname = os.path.dirname(dirname)
+
+ def pop_queue(self, container, obj):
+ """
+ In Swift, this method removes tracker object entry directly from
+ container database. In gluster-swift, this method deletes tracker
+ object directly from filesystem.
+ """
+ self.threadpool.force_run_in_thread(self._delete_tracker_object,
+ container, obj)
+
+ def delete_actual_object(self, actual_obj, timestamp):
+ """
+ Swift's expirer will re-attempt expiring if the source object is not
+ available (404 or ANY other error) up to self.reclaim_age seconds
+ before it gives up and deletes the entry in the queue.
+
+ Don't do this in gluster-swift. GlusterFS isn't eventually consistent
+ and has no concept of hand-off nodes. If actual data object doesn't
+ exist (404), remove tracker object from the queue (filesystem).
+
+ However if DELETE fails due a reason other than 404, do not remove
+ tracker object yet, follow Swift's behaviour of waiting till
+ self.reclaim_age seconds.
+
+ This method is just a wrapper around parent class's method. All this
+ wrapper does is ignore 404 failures.
+ """
+ try:
+ super(ObjectExpirer, self).delete_actual_object(
+ actual_obj, timestamp)
+ except UnexpectedResponse as err:
+ if err.resp.status_int != HTTP_NOT_FOUND:
+ raise