diff options
Diffstat (limited to 'swift/1.4.8/plugins')
| -rw-r--r-- | swift/1.4.8/plugins/DiskDir.py | 111 | ||||
| -rw-r--r-- | swift/1.4.8/plugins/__init__.py | 16 | ||||
| -rw-r--r-- | swift/1.4.8/plugins/conf/account-server/1.conf | 6 | ||||
| -rw-r--r-- | swift/1.4.8/plugins/conf/container-server/1.conf | 6 | ||||
| -rw-r--r-- | swift/1.4.8/plugins/conf/object-server/1.conf | 6 | ||||
| -rw-r--r-- | swift/1.4.8/plugins/conf/proxy-server.conf | 5 | ||||
| -rw-r--r-- | swift/1.4.8/plugins/conf/swift.conf | 85 | ||||
| -rw-r--r-- | swift/1.4.8/plugins/constraints.py | 98 | ||||
| -rw-r--r-- | swift/1.4.8/plugins/middleware/__init__.py | 0 | ||||
| -rw-r--r-- | swift/1.4.8/plugins/middleware/gluster.py | 40 | ||||
| -rw-r--r-- | swift/1.4.8/plugins/middleware/glusterfs.py | 55 | ||||
| -rw-r--r-- | swift/1.4.8/plugins/utils.py | 36 |
12 files changed, 322 insertions, 142 deletions
diff --git a/swift/1.4.8/plugins/DiskDir.py b/swift/1.4.8/plugins/DiskDir.py index 28671be3cd9..e8024768d8b 100644 --- a/swift/1.4.8/plugins/DiskDir.py +++ b/swift/1.4.8/plugins/DiskDir.py @@ -18,7 +18,7 @@ import os from swift.plugins.utils import clean_metadata, dir_empty, rmdirs, mkdirs, \ validate_account, validate_container, check_valid_account, is_marker, \ get_container_details, get_account_details, create_container_metadata, \ - create_account_metadata, DEFAULT_GID, DEFAULT_UID, get_account_details, \ + create_account_metadata, DEFAULT_GID, DEFAULT_UID, \ validate_object, create_object_metadata, read_metadata, write_metadata from swift.common.constraints import CONTAINER_LISTING_LIMIT, \ @@ -38,6 +38,26 @@ def strip_obj_storage_path(path, string='/mnt/gluster-object'): DATADIR = 'containers' +def _read_metadata(dd): + """ Filter read metadata so that it always returns a tuple that includes + some kind of timestamp. With 1.4.8 of the Swift integration the + timestamps were not stored. Here we fabricate timestamps for volumes + where the existing data has no timestamp (that is, stored data is not + a tuple), allowing us a measure of backward compatibility. + + FIXME: At this time it does not appear that the timestamps on each + metadata are used for much, so this should not hurt anything. + """ + metadata_i = read_metadata(dd) + metadata = {} + timestamp = 0 + for key, value in metadata_i.iteritems(): + if not isinstance(value, tuple): + value = (value, timestamp) + metadata[key] = value + return metadata + + class DiskCommon(object): def is_deleted(self): return not os.path.exists(self.datadir) @@ -112,6 +132,7 @@ class DiskCommon(object): write_metadata(acc_path, metadata) self.metadata = metadata + class DiskDir(DiskCommon): """ Manage object files on disk. @@ -153,7 +174,7 @@ class DiskDir(DiskCommon): self.dir_exists = os.path.exists(self.datadir) if self.dir_exists: try: - self.metadata = read_metadata(self.datadir) + self.metadata = _read_metadata(self.datadir) except EOFError: create_container_metadata(self.datadir) else: @@ -161,19 +182,19 @@ class DiskDir(DiskCommon): if container: if not self.metadata: create_container_metadata(self.datadir) - self.metadata = read_metadata(self.datadir) + self.metadata = _read_metadata(self.datadir) else: if not validate_container(self.metadata): create_container_metadata(self.datadir) - self.metadata = read_metadata(self.datadir) + self.metadata = _read_metadata(self.datadir) else: if not self.metadata: create_account_metadata(self.datadir) - self.metadata = read_metadata(self.datadir) + self.metadata = _read_metadata(self.datadir) else: if not validate_account(self.metadata): create_account_metadata(self.datadir) - self.metadata = read_metadata(self.datadir) + self.metadata = _read_metadata(self.datadir) def empty(self): return dir_empty(self.datadir) @@ -187,7 +208,6 @@ class DiskDir(DiskCommon): rmdirs(self.datadir) self.dir_exists = False - def put_metadata(self, metadata): """ Write metadata to directory/container. @@ -209,34 +229,40 @@ class DiskDir(DiskCommon): self.dir_exists = True def put_obj(self, content_length, timestamp): - self.metadata[X_OBJECTS_COUNT] = int(self.metadata[X_OBJECTS_COUNT]) + 1 + ocnt = self.metadata[X_OBJECTS_COUNT][0] + self.metadata[X_OBJECTS_COUNT] = (int(ocnt) + 1, timestamp) self.metadata[X_PUT_TIMESTAMP] = timestamp - self.metadata[X_BYTES_USED] = int(self.metadata[X_BYTES_USED]) + int(content_length) + bused = self.metadata[X_BYTES_USED][0] + self.metadata[X_BYTES_USED] = (int(bused) + int(content_length), timestamp) #TODO: define update_metadata instad of writing whole metadata again. self.put_metadata(self.metadata) def delete_obj(self, content_length): - self.metadata[X_OBJECTS_COUNT] = int(self.metadata[X_OBJECTS_COUNT]) - 1 - self.metadata[X_BYTES_USED] = int(self.metadata[X_BYTES_USED]) - int(content_length) + ocnt, timestamp = self.metadata[X_OBJECTS_COUNT][0] + self.metadata[X_OBJECTS_COUNT] = (int(ocnt) - 1, timestamp) + bused, timestamp = self.metadata[X_BYTES_USED] + self.metadata[X_BYTES_USED] = (int(bused) - int(content_length), timestamp) self.put_metadata(self.metadata) def put_container(self, container, put_timestamp, del_timestamp, object_count, bytes_used): """ For account server. """ - self.metadata[X_OBJECTS_COUNT] = 0 - self.metadata[X_BYTES_USED] = 0 - self.metadata[X_CONTAINER_COUNT] = int(self.metadata[X_CONTAINER_COUNT]) + 1 - self.metadata[X_PUT_TIMESTAMP] = 1 + self.metadata[X_OBJECTS_COUNT] = (0, put_timestamp) + self.metadata[X_BYTES_USED] = (0, put_timestamp) + ccnt = self.metadata[X_CONTAINER_COUNT][0] + self.metadata[X_CONTAINER_COUNT] = (int(ccnt) + 1, put_timestamp) + self.metadata[X_PUT_TIMESTAMP] = (1, put_timestamp) self.put_metadata(self.metadata) def delete_container(self, object_count, bytes_used): """ For account server. """ - self.metadata[X_OBJECTS_COUNT] = 0 - self.metadata[X_BYTES_USED] = 0 - self.metadata[X_CONTAINER_COUNT] = int(self.metadata[X_CONTAINER_COUNT]) - 1 + self.metadata[X_OBJECTS_COUNT] = (0, 0) + self.metadata[X_BYTES_USED] = (0, 0) + ccnt, timestamp = self.metadata[X_CONTAINER_COUNT] + self.metadata[X_CONTAINER_COUNT] = (int(ccnt) - 1, timestamp) self.put_metadata(self.metadata) def unlink(self): @@ -264,10 +290,10 @@ class DiskDir(DiskCommon): objects, object_count, bytes_used = get_container_details(self.datadir) - if int(self.metadata[X_OBJECTS_COUNT]) != object_count or \ - int(self.metadata[X_BYTES_USED]) != bytes_used: - self.metadata[X_OBJECTS_COUNT] = object_count - self.metadata[X_BYTES_USED] = bytes_used + if int(self.metadata[X_OBJECTS_COUNT][0]) != object_count or \ + int(self.metadata[X_BYTES_USED][0]) != bytes_used: + self.metadata[X_OBJECTS_COUNT] = (object_count, 0) + self.metadata[X_BYTES_USED] = (bytes_used, 0) self.update_container(self.metadata) if objects: @@ -316,11 +342,10 @@ class DiskDir(DiskCommon): bytes_used = 0 objects, object_count, bytes_used = get_container_details(self.datadir) - - if int(self.metadata[X_OBJECTS_COUNT]) != object_count or \ - int(self.metadata[X_BYTES_USED]) != bytes_used: - self.metadata[X_OBJECTS_COUNT] = object_count - self.metadata[X_BYTES_USED] = bytes_used + if int(self.metadata[X_OBJECTS_COUNT][0]) != object_count or \ + int(self.metadata[X_BYTES_USED][0]) != bytes_used: + self.metadata[X_OBJECTS_COUNT] = (object_count, 0) + self.metadata[X_BYTES_USED] = (bytes_used, 0) self.update_container(self.metadata) def update_container_count(self): @@ -329,8 +354,8 @@ class DiskDir(DiskCommon): containers, container_count = get_account_details(self.datadir) - if int(self.metadata[X_CONTAINER_COUNT]) != container_count: - self.metadata[X_CONTAINER_COUNT] = container_count + if int(self.metadata[X_CONTAINER_COUNT][0]) != container_count: + self.metadata[X_CONTAINER_COUNT] = (container_count, 0) self.update_account(self.metadata) def get_info(self, include_metadata=False): @@ -350,13 +375,13 @@ class DiskDir(DiskCommon): metadata = {} if os.path.exists(self.datadir): - metadata = read_metadata(self.datadir) + metadata = _read_metadata(self.datadir) data = {'account' : self.account, 'container' : self.name, - 'object_count' : metadata.get(X_OBJECTS_COUNT, '0'), - 'bytes_used' : metadata.get(X_BYTES_USED, '0'), + 'object_count' : metadata.get(X_OBJECTS_COUNT, ('0', 0))[0], + 'bytes_used' : metadata.get(X_BYTES_USED, ('0',0))[0], 'hash': '', 'id' : '', 'created_at' : '1', - 'put_timestamp' : metadata.get(X_PUT_TIMESTAMP, '0'), + 'put_timestamp' : metadata.get(X_PUT_TIMESTAMP, ('0',0))[0], 'delete_timestamp' : '1', 'reported_put_timestamp' : '1', 'reported_delete_timestamp' : '1', 'reported_object_count' : '1', 'reported_bytes_used' : '1'} @@ -401,7 +426,7 @@ class DiskAccount(DiskDir): self.datadir = os.path.join(self.root, self.account) if not check_mount(root, account): check_valid_account(account, fs_object) - self.metadata = read_metadata(self.datadir) + self.metadata = _read_metadata(self.datadir) if not self.metadata or not validate_account(self.metadata): self.metadata = create_account_metadata(self.datadir) @@ -419,8 +444,8 @@ class DiskAccount(DiskDir): containers, container_count = get_account_details(self.datadir) - if int(self.metadata[X_CONTAINER_COUNT]) != container_count: - self.metadata[X_CONTAINER_COUNT] = container_count + if int(self.metadata[X_CONTAINER_COUNT][0]) != container_count: + self.metadata[X_CONTAINER_COUNT] = (container_count, 0) self.update_account(self.metadata) if containers: @@ -447,13 +472,13 @@ class DiskAccount(DiskDir): list_item = [] metadata = None list_item.append(cont) - metadata = read_metadata(self.datadir + '/' + cont) + metadata = _read_metadata(self.datadir + '/' + cont) if not metadata or not validate_container(metadata): metadata = create_container_metadata(self.datadir + '/' + cont) if metadata: - list_item.append(metadata[X_OBJECTS_COUNT]) - list_item.append(metadata[X_BYTES_USED]) + list_item.append(metadata[X_OBJECTS_COUNT][0]) + list_item.append(metadata[X_BYTES_USED][0]) list_item.append(0) account_list.append(list_item) @@ -468,15 +493,15 @@ class DiskAccount(DiskDir): """ metadata = {} if (os.path.exists(self.datadir)): - metadata = read_metadata(self.datadir) + metadata = _read_metadata(self.datadir) if not metadata: metadata = create_account_metadata(self.datadir) data = {'account' : self.account, 'created_at' : '1', 'put_timestamp' : '1', 'delete_timestamp' : '1', - 'container_count' : metadata.get(X_CONTAINER_COUNT, 0), - 'object_count' : metadata.get(X_OBJECTS_COUNT, 0), - 'bytes_used' : metadata.get(X_BYTES_USED, 0), + 'container_count' : metadata.get(X_CONTAINER_COUNT, (0,0))[0], + 'object_count' : metadata.get(X_OBJECTS_COUNT, (0,0))[0], + 'bytes_used' : metadata.get(X_BYTES_USED, (0,0))[0], 'hash' : '', 'id' : ''} if include_metadata: diff --git a/swift/1.4.8/plugins/__init__.py b/swift/1.4.8/plugins/__init__.py index 3d98c960c14..e69de29bb2d 100644 --- a/swift/1.4.8/plugins/__init__.py +++ b/swift/1.4.8/plugins/__init__.py @@ -1,16 +0,0 @@ -# Copyright (c) 2011 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from Glusterfs import Glusterfs diff --git a/swift/1.4.8/plugins/conf/account-server/1.conf b/swift/1.4.8/plugins/conf/account-server/1.conf index 54cbf654003..a82bc35eaa4 100644 --- a/swift/1.4.8/plugins/conf/account-server/1.conf +++ b/swift/1.4.8/plugins/conf/account-server/1.conf @@ -6,13 +6,13 @@ user = root log_facility = LOG_LOCAL2 [pipeline:main] -pipeline = gluster account-server +pipeline = glusterfs account-server [app:account-server] use = egg:swift#account -[filter:gluster] -use = egg:swift#gluster +[filter:glusterfs] +use = egg:swift#glusterfs [account-replicator] vm_test_mode = yes diff --git a/swift/1.4.8/plugins/conf/container-server/1.conf b/swift/1.4.8/plugins/conf/container-server/1.conf index 9e776838f58..e6d313c3872 100644 --- a/swift/1.4.8/plugins/conf/container-server/1.conf +++ b/swift/1.4.8/plugins/conf/container-server/1.conf @@ -6,13 +6,13 @@ user = root log_facility = LOG_LOCAL2 [pipeline:main] -pipeline = gluster container-server +pipeline = glusterfs container-server [app:container-server] use = egg:swift#container -[filter:gluster] -use = egg:swift#gluster +[filter:glusterfs] +use = egg:swift#glusterfs [container-replicator] vm_test_mode = yes diff --git a/swift/1.4.8/plugins/conf/object-server/1.conf b/swift/1.4.8/plugins/conf/object-server/1.conf index f191cefcf01..665aa94be8b 100644 --- a/swift/1.4.8/plugins/conf/object-server/1.conf +++ b/swift/1.4.8/plugins/conf/object-server/1.conf @@ -6,13 +6,13 @@ user = root log_facility = LOG_LOCAL2 [pipeline:main] -pipeline = gluster object-server +pipeline = glusterfs object-server [app:object-server] use = egg:swift#object -[filter:gluster] -use = egg:swift#gluster +[filter:glusterfs] +use = egg:swift#glusterfs [object-replicator] vm_test_mode = yes diff --git a/swift/1.4.8/plugins/conf/proxy-server.conf b/swift/1.4.8/plugins/conf/proxy-server.conf index 1fcde8e0d3a..9bec6425355 100644 --- a/swift/1.4.8/plugins/conf/proxy-server.conf +++ b/swift/1.4.8/plugins/conf/proxy-server.conf @@ -4,7 +4,7 @@ user = root log_facility = LOG_LOCAL1 [pipeline:main] -pipeline = healthcheck cache tempauth proxy-server +pipeline = healthcheck cache tempauth gluster proxy-server [app:proxy-server] use = egg:swift#proxy @@ -19,3 +19,6 @@ use = egg:swift#healthcheck [filter:cache] use = egg:swift#memcache + +[filter:gluster] +use = egg:swift#gluster
\ No newline at end of file diff --git a/swift/1.4.8/plugins/conf/swift.conf b/swift/1.4.8/plugins/conf/swift.conf index f9864e35231..e506b6f54a8 100644 --- a/swift/1.4.8/plugins/conf/swift.conf +++ b/swift/1.4.8/plugins/conf/swift.conf @@ -1,7 +1,92 @@ [DEFAULT] Enable_plugin = yes + [swift-hash] # random unique string that can never change (DO NOT LOSE) swift_hash_path_suffix = gluster + +# The swift-constraints section sets the basic constraints on data +# saved in the swift cluster. + +[swift-constraints] + +# max_file_size is the largest "normal" object that can be saved in +# the cluster. This is also the limit on the size of each segment of +# a "large" object when using the large object manifest support. +# This value is set in bytes. Setting it to lower than 1MiB will cause +# some tests to fail. It is STRONGLY recommended to leave this value at +# the default (5 * 2**30 + 2). + +# FIXME: Really? Gluster can handle a 2^64 sized file? And can the fronting +# web service handle such a size? I think with UFO, we need to keep with the +# default size from Swift and encourage users to research what size their web +# services infrastructure can handle. + +max_file_size = 18446744073709551616 + + +# max_meta_name_length is the max number of bytes in the utf8 encoding +# of the name portion of a metadata header. + +#max_meta_name_length = 128 + + +# max_meta_value_length is the max number of bytes in the utf8 encoding +# of a metadata value + +#max_meta_value_length = 256 + + +# max_meta_count is the max number of metadata keys that can be stored +# on a single account, container, or object + +#max_meta_count = 90 + + +# max_meta_overall_size is the max number of bytes in the utf8 encoding +# of the metadata (keys + values) + +#max_meta_overall_size = 4096 + + +# max_object_name_length is the max number of bytes in the utf8 encoding of an +# object name: Gluster FS can handle much longer file names, but the length +# between the slashes of the URL is handled below. Remember that most web +# clients can't handle anything greater than 2048, and those that do are +# rather clumsy. + +max_object_name_length = 2048 + +# max_object_name_component_length (GlusterFS) is the max number of bytes in +# the utf8 encoding of an object name component (the part between the +# slashes); this is a limit imposed by the underlying file system (for XFS it +# is 255 bytes). + +max_object_name_component_length = 255 + +# container_listing_limit is the default (and max) number of items +# returned for a container listing request + +#container_listing_limit = 10000 + + +# account_listing_limit is the default (and max) number of items returned +# for an account listing request + +#account_listing_limit = 10000 + + +# max_account_name_length is the max number of bytes in the utf8 encoding of +# an account name: Gluster FS Filename limit (XFS limit?), must be the same +# size as max_object_name_component_length above. + +max_account_name_length = 255 + + +# max_container_name_length is the max number of bytes in the utf8 encoding +# of a container name: Gluster FS Filename limit (XFS limit?), must be the same +# size as max_object_name_component_length above. + +max_container_name_length = 255 diff --git a/swift/1.4.8/plugins/constraints.py b/swift/1.4.8/plugins/constraints.py index 26640717684..d522090829f 100644 --- a/swift/1.4.8/plugins/constraints.py +++ b/swift/1.4.8/plugins/constraints.py @@ -1,4 +1,4 @@ -# Copyright (c) 2011 Red Hat, Inc. +# Copyright (c) 2012 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,46 +13,28 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os -import logging -from swift.common.constraints import check_utf8, check_metadata +import swift.common.constraints -from webob.exc import HTTPBadRequest, HTTPLengthRequired, \ - HTTPRequestEntityTooLarge +MAX_OBJECT_NAME_COMPONENT_LENGTH = swift.common.constraints.constraints_conf_int( + 'max_object_name_component_length', 255) - -#: Max file size allowed for objects -MAX_FILE_SIZE = 0xffffffffffffffff -#: Max length of the name of a key for metadata -MAX_META_NAME_LENGTH = 128 -#: Max length of the value of a key for metadata -MAX_META_VALUE_LENGTH = 256 -#: Max number of metadata items -MAX_META_COUNT = 90 -#: Max overall size of metadata -MAX_META_OVERALL_SIZE = 4096 -#: Max object name length -MAX_OBJECT_NAME_LENGTH = 255 -#: Max object list length of a get request for a container -CONTAINER_LISTING_LIMIT = 10000 -#: Max container list length of a get request for an account -ACCOUNT_LISTING_LIMIT = 10000 -MAX_ACCOUNT_NAME_LENGTH = 255 -MAX_CONTAINER_NAME_LENGTH = 255 - -def validate_obj_name(obj): - if len(obj) > MAX_OBJECT_NAME_LENGTH: - logging.error('Object name too long %s' % obj) - return False +def validate_obj_name_component(obj): + if len(obj) > MAX_OBJECT_NAME_COMPONENT_LENGTH: + return 'too long (%d)' % len(obj) if obj == '.' or obj == '..': - logging.error('Object name cannot be . or .. %s' % obj) - return False + return 'cannot be . or ..' + return '' - return True +# Save the original +__check_object_creation = swift.common.constraints.check_object_creation -def check_object_creation(req, object_name): +# Define our new one which invokes the original +def gluster_check_object_creation(req, object_name): """ Check to ensure that everything is alright about an object to be created. + Monkey patches swift.common.constraints.check_object_creation, invoking + the original, and then adding an additional check for individual object + name components. :param req: HTTP request object :param object_name: name of object to be created @@ -62,38 +44,18 @@ def check_object_creation(req, object_name): :raises HTTPBadRequest: missing or bad content-type header, or bad metadata """ - if req.content_length and req.content_length > MAX_FILE_SIZE: - return HTTPRequestEntityTooLarge(body='Your request is too large.', - request=req, - content_type='text/plain') - if req.content_length is None and \ - req.headers.get('transfer-encoding') != 'chunked': - return HTTPLengthRequired(request=req) - if 'X-Copy-From' in req.headers and req.content_length: - return HTTPBadRequest(body='Copy requests require a zero byte body', - request=req, content_type='text/plain') - for obj in object_name.split('/'): - if not validate_obj_name(obj): - return HTTPBadRequest(body='Invalid object name %s' % - (obj), request=req, - content_type='text/plain') - if 'Content-Type' not in req.headers: - return HTTPBadRequest(request=req, content_type='text/plain', - body='No content type') - if not check_utf8(req.headers['Content-Type']): - return HTTPBadRequest(request=req, body='Invalid Content-Type', - content_type='text/plain') - if 'x-object-manifest' in req.headers: - value = req.headers['x-object-manifest'] - container = prefix = None - try: - container, prefix = value.split('/', 1) - except ValueError: - pass - if not container or not prefix or '?' in value or '&' in value or \ - prefix[0] == '/': - return HTTPBadRequest( - request=req, - body='X-Object-Manifest must in the format container/prefix') - return check_metadata(req, 'object') + ret = __check_object_creation(req, object_name) + + if ret is None: + for obj in object_name.split('/'): + reason = validate_obj_name(obj) + if reason: + bdy = 'Invalid object name "%s", component "%s" %s' \ + % (object_name, obj, reason) + ret = HTTPBadRequest(body=bdy, + request=req, + content_type='text/plain') + + return ret +swift.common.constraints.check_object_creation = gluster_check_object_creation diff --git a/swift/1.4.8/plugins/middleware/__init__.py b/swift/1.4.8/plugins/middleware/__init__.py new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/swift/1.4.8/plugins/middleware/__init__.py diff --git a/swift/1.4.8/plugins/middleware/gluster.py b/swift/1.4.8/plugins/middleware/gluster.py new file mode 100644 index 00000000000..e0e65e35312 --- /dev/null +++ b/swift/1.4.8/plugins/middleware/gluster.py @@ -0,0 +1,40 @@ +# Copyright (c) 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Noop Middleware that simply allows us to monkey patch the constraints +import swift.plugins.constraints + +class Gluster(object): + """ + Noop middleware for use with the proxy server to get paste.deploy to load + this middleware such that the plugin constraints monkey patch the common + constraints ahead of their use. + """ + def __init__(self, app, conf): + self.app = app + self.conf = conf + + def __call__(self, env, start_response): + return self.app(env, start_response) + + +def filter_factory(global_conf, **local_conf): + """Returns a WSGI filter app for use with paste.deploy.""" + conf = global_conf.copy() + conf.update(local_conf) + + def gluster_filter(app): + return Gluster(app, conf) + return gluster_filter diff --git a/swift/1.4.8/plugins/middleware/glusterfs.py b/swift/1.4.8/plugins/middleware/glusterfs.py new file mode 100644 index 00000000000..7472146406c --- /dev/null +++ b/swift/1.4.8/plugins/middleware/glusterfs.py @@ -0,0 +1,55 @@ +# Copyright (c) 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Monkey patch constraints +import swift.plugins.constraints +from swift.plugins.Glusterfs import Glusterfs +from ConfigParser import ConfigParser + +fs_conf = ConfigParser() +if fs_conf.read('/etc/swift/fs.conf'): + try: + mount_path = fs_conf.get ('DEFAULT', 'mount_path') + except NoSectionError, NoOptionError: + # FIXME - How to log during module initialization + logger.exception(_('ERROR mount_path not present')) + mount_path = '' +else: + mount_path = '' + + +class Gluster(object): + """ + Update the environment with keys that reflect GlusterFS middleware enabled + """ + def __init__(self, app, conf): + self.app = app + self.conf = conf + + def __call__(self, env, start_response): + env['Gluster_enabled'] = True + env['fs_object'] = Glusterfs() + env['root'] = mount_path + return self.app(env, start_response) + + +def filter_factory(global_conf, **local_conf): + """Returns a WSGI filter app for use with paste.deploy.""" + conf = global_conf.copy() + conf.update(local_conf) + + def gluster_filter(app): + return Gluster(app, conf) + return gluster_filter diff --git a/swift/1.4.8/plugins/utils.py b/swift/1.4.8/plugins/utils.py index 1c7eeb92f54..59e704702cd 100644 --- a/swift/1.4.8/plugins/utils.py +++ b/swift/1.4.8/plugins/utils.py @@ -18,8 +18,9 @@ import os import errno import xattr from hashlib import md5 -from swift.common.utils import normalize_timestamp +from swift.common.utils import normalize_timestamp, TRUE_VALUES import cPickle as pickle +from ConfigParser import ConfigParser, NoSectionError, NoOptionError X_CONTENT_TYPE = 'Content-Type' X_CONTENT_LENGTH = 'Content-Length' @@ -174,6 +175,18 @@ def do_rename(old_path, new_path): raise return True +def _add_timestamp(metadata_i): + # At this point we have a simple key/value dictionary, turn it into + # key/(value,timestamp) pairs. + timestamp = 0 + metadata = {} + for key, value_i in metadata_i.iteritems(): + if not isinstance(value_i, tuple): + metadata[key] = (value_i, timestamp) + else: + metadata[key] = value_i + return metadata + def read_metadata(path): """ Helper function to read the pickled metadata from a File/Directory . @@ -301,7 +314,8 @@ def validate_container(metadata): #logging.warn('validate_container: Metadata missing entries: %s' % metadata) return False - if metadata[X_TYPE] == CONTAINER: + (value, timestamp) = metadata[X_TYPE] + if value == CONTAINER: return True logging.warn('validate_container: metadata type is not CONTAINER (%r)' % (value,)) @@ -321,7 +335,8 @@ def validate_account(metadata): #logging.warn('validate_account: Metadata missing entries: %s' % metadata) return False - if metadata[X_TYPE] == ACCOUNT: + (value, timestamp) = metadata[X_TYPE] + if value == ACCOUNT: return True logging.warn('validate_account: metadata type is not ACCOUNT (%r)' % (value,)) @@ -552,7 +567,7 @@ def get_container_metadata(cont_path, memcache=None): X_PUT_TIMESTAMP: normalize_timestamp(os.path.getmtime(cont_path)), X_OBJECTS_COUNT: object_count, X_BYTES_USED: bytes_used} - return metadata + return _add_timestamp(metadata) def get_account_metadata(acc_path, memcache=None): containers = [] @@ -564,7 +579,7 @@ def get_account_metadata(acc_path, memcache=None): X_OBJECTS_COUNT: 0, X_BYTES_USED: 0, X_CONTAINER_COUNT: container_count} - return metadata + return _add_timestamp(metadata) def restore_object(obj_path, metadata): meta = read_metadata(obj_path) @@ -623,3 +638,14 @@ def get_account_list(fs_object): def get_account_id(account): return RESELLER_PREFIX + md5(account + HASH_PATH_SUFFIX).hexdigest() + +__swift_conf = ConfigParser() +__swift_conf.read(os.path.join('/etc/swift', 'swift.conf')) +try: + _plugin_enabled = __swift_conf.get('DEFAULT', 'Enable_plugin', 'no') in TRUE_VALUES +except NoOptionError, NoSectionError: + _plugin_enabled = False +del __swift_conf + +def plugin_enabled(): + return _plugin_enabled |