diff options
Diffstat (limited to 'gluster/swift/common')
| -rw-r--r-- | gluster/swift/common/DiskDir.py | 90 | ||||
| -rw-r--r-- | gluster/swift/common/Glusterfs.py | 2 | ||||
| -rw-r--r-- | gluster/swift/common/middleware/gswauth/swauth/middleware.py | 1 | ||||
| -rw-r--r-- | gluster/swift/common/utils.py | 41 |
4 files changed, 105 insertions, 29 deletions
diff --git a/gluster/swift/common/DiskDir.py b/gluster/swift/common/DiskDir.py index 6112709..e8dba35 100644 --- a/gluster/swift/common/DiskDir.py +++ b/gluster/swift/common/DiskDir.py @@ -255,7 +255,7 @@ class DiskDir(DiskCommon): :param uid: user ID container object should assume :param gid: group ID container object should assume - Usage pattern from container/server.py (Havana, 1.8.0+): + Usage pattern from container/server.py (Kilo, 2.3.0): DELETE: if auto-create and obj and not .db_file: # Creates container @@ -287,36 +287,43 @@ class DiskDir(DiskCommon): return 404 .put_object() else: - if not .db_file: - # Creates container - .initialize() - else: - # Update container timestamp - .is_deleted() + _update_or_create(): + if not .db_file: + # Creates container + .initialize() + recreated = .is_deleted(): + if recreated: + .set_storage_policy_index() + .storage_policy_index .update_put_timestamp() if .is_deleted() return conflict - if metadata: + if recreated: + .update_status_changed_at() + + if 'X-Container-Sync-To' in metadata: if .metadata .set_x_container_sync_points() .update_metadata() account_update(): .get_info() HEAD: - .pending_timeout - .stale_reads_ok - if .is_deleted(): - return 404 - .get_info() + info, is_deleted = .get_info_is_deleted() + .get_info_is_deleted(): + if not .db_file: + return {}, True + info = .get_info() + return info, ._is_deleted_info() .metadata GET: - .pending_timeout - .stale_reads_ok - if .is_deleted(): - return 404 - .get_info() - .metadata + info, is_deleted = .get_info_is_deleted() + .get_info_is_deleted(): + if not .db_file: + return {}, True + info = .get_info() + return info, ._is_deleted_info() .list_objects_iter() + .metadata POST: if .is_deleted(): return 404 @@ -346,8 +353,22 @@ class DiskDir(DiskCommon): create_container_metadata(self.datadir) self.metadata = _read_metadata(self.datadir) + def update_status_changed_at(self, timestamp): + return + + @property + def storage_policy_index(self): + if not hasattr(self, '_storage_policy_index'): + self._storage_policy_index = \ + self.get_info()['storage_policy_index'] + return self._storage_policy_index + + def set_storage_policy_index(self, policy_index, timestamp=None): + self._storage_policy_index = policy_index + def list_objects_iter(self, limit, marker, end_marker, - prefix, delimiter, path=None): + prefix, delimiter, path=None, + storage_policy_index=0): """ Returns tuple of name, created_at, size, content_type, etag. """ @@ -451,6 +472,12 @@ class DiskDir(DiskCommon): return objects + def get_info_is_deleted(self): + if not do_exists(self.datadir): + return {}, True + info = self.get_info() + return info, False + def get_info(self): """ Get global data for the container. @@ -477,7 +504,10 @@ class DiskDir(DiskCommon): 'x_container_sync_point1', -1), 'x_container_sync_point2': self.metadata.get( 'x_container_sync_point2', -1), + 'storage_policy_index': self.metadata.get( + 'storage_policy_index', 0) } + self._storage_policy_index = data['storage_policy_index'] return data def put_object(self, name, timestamp, size, content_type, etag, deleted=0): @@ -540,13 +570,14 @@ class DiskDir(DiskCommon): class DiskAccount(DiskCommon): """ - Usage pattern from account/server.py (Havana, 1.8.0+): + Usage pattern from account/server.py (Kilo, 2.3.0): DELETE: .is_deleted() + .is_status_deleted() .delete_db() + .is_status_deleted() PUT: container: - .pending_timeout .db_file .initialize() .is_deleted() @@ -555,25 +586,27 @@ class DiskAccount(DiskCommon): .db_file .initialize() .is_status_deleted() + .is_status_deleted() .is_deleted() .update_put_timestamp() - .is_deleted() ??? + .is_deleted() .update_metadata() HEAD: - .pending_timeout - .stale_reads_ok .is_deleted() + .is_status_deleted() .get_info() + .get_policy_stats() .metadata GET: - .pending_timeout - .stale_reads_ok .is_deleted() + .is_status_deleted() .get_info() + .get_policy_stats() .metadata .list_containers_iter() POST: .is_deleted() + .is_status_deleted() .update_metadata() """ @@ -748,3 +781,6 @@ class DiskAccount(DiskCommon): 'bytes_used': self.metadata.get(X_BYTES_USED, (0, 0))[0], 'hash': '', 'id': ''} return data + + def get_policy_stats(self, do_migrations=False): + return {} diff --git a/gluster/swift/common/Glusterfs.py b/gluster/swift/common/Glusterfs.py index 6a2fdb2..910f752 100644 --- a/gluster/swift/common/Glusterfs.py +++ b/gluster/swift/common/Glusterfs.py @@ -148,7 +148,7 @@ def _get_unique_id(): # own the lock. continue raise - except: + except Exception: os.close(fd) raise else: diff --git a/gluster/swift/common/middleware/gswauth/swauth/middleware.py b/gluster/swift/common/middleware/gswauth/swauth/middleware.py index cdcc638..3cd9cf7 100644 --- a/gluster/swift/common/middleware/gswauth/swauth/middleware.py +++ b/gluster/swift/common/middleware/gswauth/swauth/middleware.py @@ -28,6 +28,7 @@ import base64 from eventlet.timeout import Timeout from eventlet import TimeoutError +from swift import gettext_ as _ from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPConflict, \ HTTPCreated, HTTPForbidden, HTTPMethodNotAllowed, HTTPMovedPermanently, \ HTTPNoContent, HTTPNotFound, HTTPUnauthorized, \ diff --git a/gluster/swift/common/utils.py b/gluster/swift/common/utils.py index b6a5a09..e6f4bcc 100644 --- a/gluster/swift/common/utils.py +++ b/gluster/swift/common/utils.py @@ -17,6 +17,7 @@ import os import stat import json import errno +import random import logging from hashlib import md5 from eventlet import sleep @@ -29,7 +30,7 @@ from swift.common.db import utf8encodekeys from gluster.swift.common.fs_utils import do_getctime, do_getmtime, do_stat, \ do_listdir, do_walk, do_rmdir, do_log_rl, get_filename_from_fd, do_open, \ do_isdir, do_getsize, do_getxattr, do_setxattr, do_removexattr, do_read, \ - do_close, do_dup, do_lseek, do_fstat + do_close, do_dup, do_lseek, do_fstat, do_fsync, do_rename from gluster.swift.common import Glusterfs X_CONTENT_TYPE = 'Content-Type' @@ -607,3 +608,41 @@ def rmobjdir(dir_path): raise else: return True + + +def write_pickle(obj, dest, tmp=None, pickle_protocol=0): + """ + Ensure that a pickle file gets written to disk. The file is first written + to a tmp file location in the destination directory path, ensured it is + synced to disk, then moved to its final destination name. + + This version takes advantage of Gluster's dot-prefix-dot-suffix naming + where the a file named ".thefile.name.9a7aasv" is hashed to the same + Gluster node as "thefile.name". This ensures the renaming of a temp file + once written does not move it to another Gluster node. + + :param obj: python object to be pickled + :param dest: path of final destination file + :param tmp: path to tmp to use, defaults to None (ignored) + :param pickle_protocol: protocol to pickle the obj with, defaults to 0 + """ + dirname = os.path.dirname(dest) + # Create destination directory + try: + os.makedirs(dirname) + except OSError as err: + if err.errno != errno.EEXIST: + raise + basename = os.path.basename(dest) + tmpname = '.' + basename + '.' + \ + md5(basename + str(random.random())).hexdigest() + tmppath = os.path.join(dirname, tmpname) + with open(tmppath, 'wb') as fo: + pickle.dump(obj, fo, pickle_protocol) + # TODO: This flush() method call turns into a flush() system call + # We'll need to wrap this as well, but we would do this by writing + # a context manager for our own open() method which returns an object + # in fo which makes the gluster API call. + fo.flush() + do_fsync(fo) + do_rename(tmppath, dest) |