summaryrefslogtreecommitdiffstats
path: root/swift/1.4.8/plugins/utils.py
diff options
context:
space:
mode:
Diffstat (limited to 'swift/1.4.8/plugins/utils.py')
-rw-r--r--swift/1.4.8/plugins/utils.py692
1 files changed, 692 insertions, 0 deletions
diff --git a/swift/1.4.8/plugins/utils.py b/swift/1.4.8/plugins/utils.py
new file mode 100644
index 00000000000..57c6180fdd1
--- /dev/null
+++ b/swift/1.4.8/plugins/utils.py
@@ -0,0 +1,692 @@
+# Copyright (c) 2011 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+import errno
+from hashlib import md5
+from swift.common.utils import normalize_timestamp
+from xattr import setxattr, removexattr, getxattr, removexattr
+import cPickle as pickle
+
+X_CONTENT_TYPE = 'Content-Type'
+X_CONTENT_LENGTH = 'Content-Length'
+X_TIMESTAMP = 'X-Timestamp'
+X_PUT_TIMESTAMP = 'X-PUT-Timestamp'
+X_TYPE = 'X-Type'
+X_ETAG = 'ETag'
+X_OBJECTS_COUNT = 'X-Object-Count'
+X_BYTES_USED = 'X-Bytes-Used'
+X_CONTAINER_COUNT = 'X-Container-Count'
+X_OBJECT_TYPE = 'X-Object-Type'
+DIR_TYPE = 'application/directory'
+ACCOUNT = 'Account'
+MOUNT_PATH = '/mnt/gluster-object'
+METADATA_KEY = 'user.swift.metadata'
+CONTAINER = 'container'
+DIR = 'dir'
+MARKER_DIR = 'marker_dir'
+FILE = 'file'
+DIR_TYPE = 'application/directory'
+FILE_TYPE = 'application/octet-stream'
+OBJECT = 'Object'
+OBJECT_TYPE = 'application/octet-stream'
+DEFAULT_UID = -1
+DEFAULT_GID = -1
+PICKLE_PROTOCOL = 2
+CHUNK_SIZE = 65536
+
+
+def mkdirs(path):
+ """
+ Ensures the path is a directory or makes it if not. Errors if the path
+ exists but is a file or on permissions failure.
+
+ :param path: path to create
+ """
+ if not os.path.isdir(path):
+ try:
+ do_makedirs(path)
+ except OSError, err:
+ #TODO: check, isdir will fail if mounted and volume stopped.
+ #if err.errno != errno.EEXIST or not os.path.isdir(path)
+ if err.errno != errno.EEXIST:
+ raise
+
+def rmdirs(path):
+ if os.path.isdir(path) and dir_empty(path):
+ do_rmdir(path)
+ else:
+ logging.error("rmdirs failed dir may not be empty or not valid dir")
+ return False
+
+def strip_obj_storage_path(path, string='/mnt/gluster-object'):
+ """
+ strip /mnt/gluster-object
+ """
+ return path.replace(string, '').strip('/')
+
+def do_mkdir(path):
+ try:
+ os.mkdir(path)
+ except Exception, err:
+ logging.exception("Mkdir failed on %s err: %s", path, str(err))
+ if err.errno != errno.EEXIST:
+ raise
+ return True
+
+def do_makedirs(path):
+ try:
+ os.makedirs(path)
+ except Exception, err:
+ logging.exception("Makedirs failed on %s err: %s", path, str(err))
+ if err.errno != errno.EEXIST:
+ raise
+ return True
+
+
+def do_listdir(path):
+ try:
+ buf = os.listdir(path)
+ except Exception, err:
+ logging.exception("Listdir failed on %s err: %s", path, str(err))
+ raise
+ return buf
+
+def do_chown(path, uid, gid):
+ try:
+ os.chown(path, uid, gid)
+ except Exception, err:
+ logging.exception("Chown failed on %s err: %s", path, str(err))
+ raise
+ return True
+
+def do_stat(path):
+ try:
+ #Check for fd.
+ if isinstance(path, int):
+ buf = os.fstat(path)
+ else:
+ buf = os.stat(path)
+ except Exception, err:
+ logging.exception("Stat failed on %s err: %s", path, str(err))
+ raise
+
+ return buf
+
+def do_open(path, mode):
+ try:
+ fd = open(path, mode)
+ except Exception, err:
+ logging.exception("Open failed on %s err: %s", path, str(err))
+ raise
+ return fd
+
+def do_close(fd):
+ #fd could be file or int type.
+ try:
+ if isinstance(fd, int):
+ os.close(fd)
+ else:
+ fd.close()
+ except Exception, err:
+ logging.exception("Close failed on %s err: %s", fd, str(err))
+ raise
+ return True
+
+def do_unlink(path, log = True):
+ try:
+ os.unlink(path)
+ except Exception, err:
+ if log:
+ logging.exception("Unlink failed on %s err: %s", path, str(err))
+ if err.errno != errno.ENOENT:
+ raise
+ return True
+
+def do_rmdir(path):
+ try:
+ os.rmdir(path)
+ except Exception, err:
+ logging.exception("Rmdir failed on %s err: %s", path, str(err))
+ if err.errno != errno.ENOENT:
+ raise
+ return True
+
+def do_rename(old_path, new_path):
+ try:
+ os.rename(old_path, new_path)
+ except Exception, err:
+ logging.exception("Rename failed on %s to %s err: %s", old_path, new_path, \
+ str(err))
+ raise
+ return True
+
+def do_setxattr(path, key, value):
+ fd = None
+ if not os.path.isdir(path):
+ fd = do_open(path, 'rb')
+ else:
+ fd = path
+ if fd or os.path.isdir(path):
+ try:
+ setxattr(fd, key, value)
+ except Exception, err:
+ logging.exception("setxattr failed on %s key %s err: %s", path, key, str(err))
+ raise
+ finally:
+ if fd and not os.path.isdir(path):
+ do_close(fd)
+ else:
+ logging.error("Open failed path %s", path)
+ return False
+ return True
+
+
+
+def do_getxattr(path, key, log = True):
+ fd = None
+ if not os.path.isdir(path):
+ fd = do_open(path, 'rb')
+ else:
+ fd = path
+ if fd or os.path.isdir(path):
+ try:
+ value = getxattr(fd, key)
+ except Exception, err:
+ if log:
+ logging.exception("getxattr failed on %s key %s err: %s", path, key, str(err))
+ raise
+ finally:
+ if fd and not os.path.isdir(path):
+ do_close(fd)
+ else:
+ logging.error("Open failed path %s", path)
+ return False
+ return value
+
+def do_removexattr(path, key):
+ fd = None
+ if not os.path.isdir(path):
+ fd = do_open(path, 'rb')
+ else:
+ fd = path
+ if fd or os.path.isdir(path):
+ try:
+ removexattr(fd, key)
+ except Exception, err:
+ logging.exception("removexattr failed on %s key %s err: %s", path, key, str(err))
+ raise
+ finally:
+ if fd and not os.path.isdir(path):
+ do_close(fd)
+ else:
+ logging.error("Open failed path %s", path)
+ return False
+ return True
+
+def read_metadata(path):
+ """
+ Helper function to read the pickled metadata from a File/Directory .
+
+ :param path: File/Directory to read metadata from.
+
+ :returns: dictionary of metadata
+ """
+ metadata = ''
+ key = 0
+ while True:
+ try:
+ metadata += do_getxattr(path, '%s%s' % (METADATA_KEY, (key or '')),
+ log = False)
+ except Exception:
+ break
+ key += 1
+ if metadata:
+ return pickle.loads(metadata)
+ else:
+ return {}
+
+
+def write_metadata(path, metadata):
+ """
+ Helper function to write pickled metadata for a File/Directory.
+
+ :param path: File/Directory path to write the metadata
+ :param metadata: metadata to write
+ """
+ metastr = pickle.dumps(metadata, PICKLE_PROTOCOL)
+ key = 0
+ while metastr:
+ do_setxattr(path, '%s%s' % (METADATA_KEY, key or ''), metastr[:254])
+ metastr = metastr[254:]
+ key += 1
+
+def clean_metadata(path):
+ key = 0
+ while True:
+ value = do_getxattr(path, '%s%s' % (METADATA_KEY, (key or '')))
+ do_removexattr(path, '%s%s' % (METADATA_KEY, (key or '')))
+ key += 1
+
+
+def dir_empty(path):
+ """
+ Return true if directory/container is empty.
+ :param path: Directory path.
+ :returns: True/False.
+ """
+ if os.path.isdir(path):
+ try:
+ files = do_listdir(path)
+ except Exception, err:
+ logging.exception("listdir failed on %s err: %s", path, str(err))
+ raise
+ if not files:
+ return True
+ else:
+ return False
+ else:
+ if not os.path.exists(path):
+ return True
+
+
+def get_device_from_account(account):
+ if account.startswith(RESELLER_PREFIX):
+ device = account.replace(RESELLER_PREFIX, '', 1)
+ return device
+
+def check_user_xattr(path):
+ if not os.path.exists(path):
+ return False
+ do_setxattr(path, 'user.test.key1', 'value1')
+ try:
+ removexattr(path, 'user.test.key1')
+ except Exception, err:
+ logging.exception("removexattr failed on %s err: %s", path, str(err))
+ #Remove xattr may fail in case of concurrent remove.
+ return True
+
+
+def _check_valid_account(account, fs_object):
+ mount_path = getattr(fs_object, 'mount_path', MOUNT_PATH)
+
+ if not check_account_exists(fs_object.get_export_from_account_id(account), \
+ fs_object):
+ logging.error('Account not present %s', account)
+ return False
+
+ if not os.path.ismount(os.path.join(mount_path, account)):
+ if not os.path.isdir(os.path.join(mount_path, account)):
+ mkdirs(os.path.join(mount_path, account))
+
+ fs_object.unmount(os.path.join(mount_path, account))
+
+ if fs_object:
+ if not fs_object.mount(account):
+ return False
+
+ if not check_user_xattr(os.path.join(mount_path, account)):
+ logging.error('Error: No support for user.xattr on backend %s' % account)
+ return False
+
+ chmod_cmd = ['chmod 777 %s' % (mount_path), \
+ 'chmod 777 %s/%s' % (mount_path, account)]
+
+ for cmd in chmod_cmd:
+ if os.system(cmd):
+ logging.error('Chmod failed: %s' % (cmd))
+ return False
+
+ return True
+
+def check_valid_account(account, fs_object):
+ return _check_valid_account(account, fs_object)
+
+def validate_container(metadata):
+ if not metadata:
+ logging.error('No metadata')
+ return False
+
+ if X_TYPE not in metadata.keys() or \
+ X_TIMESTAMP not in metadata.keys() or \
+ X_PUT_TIMESTAMP not in metadata.keys() or \
+ X_OBJECTS_COUNT not in metadata.keys() or \
+ X_BYTES_USED not in metadata.keys():
+ #logging.error('Container error %s' % metadata)
+ return False
+
+ if metadata[X_TYPE] == CONTAINER:
+ return True
+
+ logging.error('Container error %s' % metadata)
+ return False
+
+def validate_account(metadata):
+ if not metadata:
+ logging.error('No metadata')
+ return False
+
+ if X_TYPE not in metadata.keys() or \
+ X_TIMESTAMP not in metadata.keys() or \
+ X_PUT_TIMESTAMP not in metadata.keys() or \
+ X_OBJECTS_COUNT not in metadata.keys() or \
+ X_BYTES_USED not in metadata.keys() or \
+ X_CONTAINER_COUNT not in metadata.keys():
+ #logging.error('Account error %s' % metadata)
+ return False
+
+ if metadata[X_TYPE] == ACCOUNT:
+ return True
+
+ logging.error('Account error %s' % metadata)
+ return False
+
+def validate_object(metadata):
+ if not metadata:
+ logging.error('No metadata')
+ return False
+
+ if X_TIMESTAMP not in metadata.keys() or \
+ X_CONTENT_TYPE not in metadata.keys() or \
+ X_ETAG not in metadata.keys() or \
+ X_CONTENT_LENGTH not in metadata.keys() or \
+ X_TYPE not in metadata.keys() or \
+ X_OBJECT_TYPE not in metadata.keys():
+ #logging.error('Object error %s' % metadata)
+ return False
+
+ if metadata[X_TYPE] == OBJECT:
+ return True
+
+ logging.error('Object error %s' % metadata)
+ return False
+
+def is_marker(metadata):
+ if not metadata:
+ logging.error('No metadata')
+ return False
+
+ if X_OBJECT_TYPE not in metadata.keys():
+ logging.error('X_OBJECT_TYPE missing %s' % metadata)
+ return False
+
+ if metadata[X_OBJECT_TYPE] == MARKER_DIR:
+ return True
+ else:
+ return False
+
+def _update_list(path, const_path, src_list, reg_file=True, object_count=0,
+ bytes_used=0, obj_list=[]):
+ obj_path = strip_obj_storage_path(path, const_path)
+
+ for i in src_list:
+ if obj_path:
+ obj_list.append(os.path.join(obj_path, i))
+ else:
+ obj_list.append(i)
+
+ object_count += 1
+
+ if reg_file:
+ bytes_used += os.path.getsize(path + '/' + i)
+
+ return object_count, bytes_used
+
+def update_list(path, const_path, dirs=[], files=[], object_count=0,
+ bytes_used=0, obj_list=[]):
+ object_count, bytes_used = _update_list (path, const_path, files, True,
+ object_count, bytes_used,
+ obj_list)
+ object_count, bytes_used = _update_list (path, const_path, dirs, False,
+ object_count, bytes_used,
+ obj_list)
+ return object_count, bytes_used
+
+def get_container_details_from_fs(cont_path, const_path,
+ memcache=None):
+ """
+ get container details by traversing the filesystem
+ """
+ bytes_used = 0
+ object_count = 0
+ obj_list=[]
+ dir_list = []
+
+ if os.path.isdir(cont_path):
+ for (path, dirs, files) in os.walk(cont_path):
+ object_count, bytes_used = update_list(path, const_path, dirs, files,
+ object_count, bytes_used,
+ obj_list)
+
+ dir_list.append(path + ':' + str(do_stat(path).st_mtime))
+
+ if memcache:
+ memcache.set(strip_obj_storage_path(cont_path), obj_list)
+ memcache.set(strip_obj_storage_path(cont_path) + '-dir_list',
+ ','.join(dir_list))
+ memcache.set(strip_obj_storage_path(cont_path) + '-cont_meta',
+ [object_count, bytes_used])
+
+ return obj_list, object_count, bytes_used
+
+def get_container_details_from_memcache(cont_path, const_path,
+ memcache):
+ """
+ get container details stored in memcache
+ """
+
+ bytes_used = 0
+ object_count = 0
+ obj_list=[]
+
+ dir_contents = memcache.get(strip_obj_storage_path(cont_path) + '-dir_list')
+ if not dir_contents:
+ return get_container_details_from_fs(cont_path, const_path,
+ memcache=memcache)
+
+ for i in dir_contents.split(','):
+ path, mtime = i.split(':')
+ if mtime != str(do_stat(path).st_mtime):
+ return get_container_details_from_fs(cont_path, const_path,
+ memcache=memcache)
+
+ obj_list = memcache.get(strip_obj_storage_path(cont_path))
+
+ object_count, bytes_used = memcache.get(strip_obj_storage_path(cont_path) + '-cont_meta')
+
+ return obj_list, object_count, bytes_used
+
+def get_container_details(cont_path, memcache=None):
+ """
+ Return object_list, object_count and bytes_used.
+ """
+ if memcache:
+ object_list, object_count, bytes_used = get_container_details_from_memcache(cont_path, cont_path,
+ memcache=memcache)
+ else:
+ object_list, object_count, bytes_used = get_container_details_from_fs(cont_path, cont_path)
+
+ return object_list, object_count, bytes_used
+
+def get_account_details_from_fs(acc_path, memcache=None):
+ container_list = []
+ container_count = 0
+
+ if os.path.isdir(acc_path):
+ for name in do_listdir(acc_path):
+ if not os.path.isdir(acc_path + '/' + name) or \
+ name.lower() == 'tmp':
+ continue
+ container_count += 1
+ container_list.append(name)
+
+ if memcache:
+ memcache.set(strip_obj_storage_path(acc_path) + '_container_list', container_list)
+ memcache.set(strip_obj_storage_path(acc_path)+'_mtime', str(do_stat(acc_path).st_mtime))
+ memcache.set(strip_obj_storage_path(acc_path)+'_container_count', container_count)
+
+ return container_list, container_count
+
+def get_account_details_from_memcache(acc_path, memcache=None):
+ if memcache:
+ mtime = memcache.get(strip_obj_storage_path(acc_path)+'_mtime')
+ if not mtime or mtime != str(do_stat(acc_path).st_mtime):
+ return get_account_details_from_fs(acc_path, memcache)
+ container_list = memcache.get(strip_obj_storage_path(acc_path) + '_container_list')
+ container_count = memcache.get(strip_obj_storage_path(acc_path)+'_container_count')
+ return container_list, container_count
+
+
+def get_account_details(acc_path, memcache=None):
+ """
+ Return container_list and container_count.
+ """
+ if memcache:
+ return get_account_details_from_memcache(acc_path, memcache)
+ else:
+ return get_account_details_from_fs(acc_path, memcache)
+
+
+
+def get_etag(path):
+ etag = None
+ if os.path.exists(path):
+ etag = md5()
+ if not os.path.isdir(path):
+ fp = open(path, 'rb')
+ if fp:
+ while True:
+ chunk = fp.read(CHUNK_SIZE)
+ if chunk:
+ etag.update(chunk)
+ else:
+ break
+ fp.close()
+
+ etag = etag.hexdigest()
+
+ return etag
+
+
+def get_object_metadata(obj_path):
+ """
+ Return metadata of object.
+ """
+ metadata = {}
+ if os.path.exists(obj_path):
+ if not os.path.isdir(obj_path):
+ metadata = {
+ X_TIMESTAMP: normalize_timestamp(os.path.getctime(obj_path)),
+ X_CONTENT_TYPE: FILE_TYPE,
+ X_ETAG: get_etag(obj_path),
+ X_CONTENT_LENGTH: os.path.getsize(obj_path),
+ X_TYPE: OBJECT,
+ X_OBJECT_TYPE: FILE,
+ }
+ else:
+ metadata = {
+ X_TIMESTAMP: normalize_timestamp(os.path.getctime(obj_path)),
+ X_CONTENT_TYPE: DIR_TYPE,
+ X_ETAG: get_etag(obj_path),
+ X_CONTENT_LENGTH: 0,
+ X_TYPE: OBJECT,
+ X_OBJECT_TYPE: DIR,
+ }
+
+ return metadata
+
+def get_container_metadata(cont_path, memcache=None):
+ objects = []
+ object_count = 0
+ bytes_used = 0
+ objects, object_count, bytes_used = get_container_details(cont_path,
+ memcache=memcache)
+ metadata = {X_TYPE: CONTAINER,
+ X_TIMESTAMP: normalize_timestamp(os.path.getctime(cont_path)),
+ X_PUT_TIMESTAMP: normalize_timestamp(os.path.getmtime(cont_path)),
+ X_OBJECTS_COUNT: object_count,
+ X_BYTES_USED: bytes_used}
+ return metadata
+
+def get_account_metadata(acc_path, memcache=None):
+ containers = []
+ container_count = 0
+ containers, container_count = get_account_details(acc_path, memcache)
+ metadata = {X_TYPE: ACCOUNT,
+ X_TIMESTAMP: normalize_timestamp(os.path.getctime(acc_path)),
+ X_PUT_TIMESTAMP: normalize_timestamp(os.path.getmtime(acc_path)),
+ X_OBJECTS_COUNT: 0,
+ X_BYTES_USED: 0,
+ X_CONTAINER_COUNT: container_count}
+ return metadata
+
+def restore_object(obj_path, metadata):
+ meta = read_metadata(obj_path)
+ if meta:
+ meta.update(metadata)
+ write_metadata(obj_path, meta)
+ else:
+ write_metadata(obj_path, metadata)
+
+def restore_container(cont_path, metadata):
+ meta = read_metadata(cont_path)
+ if meta:
+ meta.update(metadata)
+ write_metadata(cont_path, meta)
+ else:
+ write_metadata(cont_path, metadata)
+
+def restore_account(acc_path, metadata):
+ meta = read_metadata(acc_path)
+ if meta:
+ meta.update(metadata)
+ write_metadata(acc_path, meta)
+ else:
+ write_metadata(acc_path, metadata)
+
+def create_object_metadata(obj_path):
+ meta = get_object_metadata(obj_path)
+ restore_object(obj_path, meta)
+ return meta
+
+def create_container_metadata(cont_path, memcache=None):
+ meta = get_container_metadata(cont_path, memcache)
+ restore_container(cont_path, meta)
+ return meta
+
+def create_account_metadata(acc_path, memcache=None):
+ meta = get_account_metadata(acc_path, memcache)
+ restore_account(acc_path, meta)
+ return meta
+
+
+def check_account_exists(account, fs_object):
+ if account not in get_account_list(fs_object):
+ logging.error('Account not exists %s' % account)
+ return False
+ else:
+ return True
+
+def get_account_list(fs_object):
+ account_list = []
+ if fs_object:
+ account_list = fs_object.get_export_list()
+ return account_list
+
+
+def get_account_id(account):
+ return RESELLER_PREFIX + md5(account + HASH_PATH_SUFFIX).hexdigest()
+
generated by cgit (git 2.34.1) at 2024-11-18 12:54:20 +0000