summaryrefslogtreecommitdiffstats
path: root/swift/1.4.8/plugins
diff options
context:
space:
mode:
authorPeter Portante <peter.portante@redhat.com>2012-10-16 23:27:21 -0400
committerAnand Avati <avati@redhat.com>2012-10-17 11:04:05 -0700
commitdbe793bcde5d0ba936eceb19bcb8a6f376a0dbc4 (patch)
treeff281eec3636f11366bc89ee2b82f865d54814b5 /swift/1.4.8/plugins
parent59dfcf15578e08731f80c1f0c88cd4b7cd79d23d (diff)
object-storage: Refactor code to reduce Swift diffs carried
The upstream Swift code base contains the following commit which adds the ability to modify constraint values using the swift.conf file: https://github.com/openstack/swift/commit/a2ac5efaa64f57fbbe059066c6c4636dfd0715c2 These changes rely on the above commit being back-ported to Swift 1.4.8 (see https://github.com/portante/swift/commit/fc2421b04022ac6bbe9d5014362ec5f99f94c5e0). As a result, a good number of differences we carry can be removed, since the provided swift.conf file now contains the values we need. Along with these changes, we add middleware to get constraints loaded properly (via monkey patching) until subclassing is implemented in a future commit. We further simplify the diffs to a minimal set by storing the timestamp in file system metadata and moving new files to the plugins/middleware directory. Note that the original "gluster" middleware was in the swift.diff file, and has been renamed to "glusterfs" and moved to the new plugins/middleware directory. The "gluster" middleware is now a temporary way to get the constraints properly loaded (both of these middleware pieces should disappear in future commits when we refactor further to subclass the Swift objects instead of patching them). Change-Id: I9dc00d6b6cdd64e277896d75c2fb06431c3e69cb BUG: 862052 Signed-off-by: Peter Portante <peter.portante@redhat.com> Reviewed-on: http://review.gluster.org/4093 Tested-by: Peter Portante <pportant@redhat.com> Reviewed-by: Anand Avati <avati@redhat.com>
Diffstat (limited to 'swift/1.4.8/plugins')
-rw-r--r--swift/1.4.8/plugins/DiskDir.py111
-rw-r--r--swift/1.4.8/plugins/__init__.py16
-rw-r--r--swift/1.4.8/plugins/conf/account-server/1.conf6
-rw-r--r--swift/1.4.8/plugins/conf/container-server/1.conf6
-rw-r--r--swift/1.4.8/plugins/conf/object-server/1.conf6
-rw-r--r--swift/1.4.8/plugins/conf/proxy-server.conf5
-rw-r--r--swift/1.4.8/plugins/conf/swift.conf85
-rw-r--r--swift/1.4.8/plugins/constraints.py98
-rw-r--r--swift/1.4.8/plugins/middleware/__init__.py0
-rw-r--r--swift/1.4.8/plugins/middleware/gluster.py40
-rw-r--r--swift/1.4.8/plugins/middleware/glusterfs.py55
-rw-r--r--swift/1.4.8/plugins/utils.py36
12 files changed, 322 insertions, 142 deletions
diff --git a/swift/1.4.8/plugins/DiskDir.py b/swift/1.4.8/plugins/DiskDir.py
index 28671be3..e8024768 100644
--- a/swift/1.4.8/plugins/DiskDir.py
+++ b/swift/1.4.8/plugins/DiskDir.py
@@ -18,7 +18,7 @@ import os
from swift.plugins.utils import clean_metadata, dir_empty, rmdirs, mkdirs, \
validate_account, validate_container, check_valid_account, is_marker, \
get_container_details, get_account_details, create_container_metadata, \
- create_account_metadata, DEFAULT_GID, DEFAULT_UID, get_account_details, \
+ create_account_metadata, DEFAULT_GID, DEFAULT_UID, \
validate_object, create_object_metadata, read_metadata, write_metadata
from swift.common.constraints import CONTAINER_LISTING_LIMIT, \
@@ -38,6 +38,26 @@ def strip_obj_storage_path(path, string='/mnt/gluster-object'):
DATADIR = 'containers'
+def _read_metadata(dd):
+ """ Filter read metadata so that it always returns a tuple that includes
+ some kind of timestamp. With 1.4.8 of the Swift integration the
+ timestamps were not stored. Here we fabricate timestamps for volumes
+ where the existing data has no timestamp (that is, stored data is not
+ a tuple), allowing us a measure of backward compatibility.
+
+ FIXME: At this time it does not appear that the timestamps on each
+ metadata are used for much, so this should not hurt anything.
+ """
+ metadata_i = read_metadata(dd)
+ metadata = {}
+ timestamp = 0
+ for key, value in metadata_i.iteritems():
+ if not isinstance(value, tuple):
+ value = (value, timestamp)
+ metadata[key] = value
+ return metadata
+
+
class DiskCommon(object):
def is_deleted(self):
return not os.path.exists(self.datadir)
@@ -112,6 +132,7 @@ class DiskCommon(object):
write_metadata(acc_path, metadata)
self.metadata = metadata
+
class DiskDir(DiskCommon):
"""
Manage object files on disk.
@@ -153,7 +174,7 @@ class DiskDir(DiskCommon):
self.dir_exists = os.path.exists(self.datadir)
if self.dir_exists:
try:
- self.metadata = read_metadata(self.datadir)
+ self.metadata = _read_metadata(self.datadir)
except EOFError:
create_container_metadata(self.datadir)
else:
@@ -161,19 +182,19 @@ class DiskDir(DiskCommon):
if container:
if not self.metadata:
create_container_metadata(self.datadir)
- self.metadata = read_metadata(self.datadir)
+ self.metadata = _read_metadata(self.datadir)
else:
if not validate_container(self.metadata):
create_container_metadata(self.datadir)
- self.metadata = read_metadata(self.datadir)
+ self.metadata = _read_metadata(self.datadir)
else:
if not self.metadata:
create_account_metadata(self.datadir)
- self.metadata = read_metadata(self.datadir)
+ self.metadata = _read_metadata(self.datadir)
else:
if not validate_account(self.metadata):
create_account_metadata(self.datadir)
- self.metadata = read_metadata(self.datadir)
+ self.metadata = _read_metadata(self.datadir)
def empty(self):
return dir_empty(self.datadir)
@@ -187,7 +208,6 @@ class DiskDir(DiskCommon):
rmdirs(self.datadir)
self.dir_exists = False
-
def put_metadata(self, metadata):
"""
Write metadata to directory/container.
@@ -209,34 +229,40 @@ class DiskDir(DiskCommon):
self.dir_exists = True
def put_obj(self, content_length, timestamp):
- self.metadata[X_OBJECTS_COUNT] = int(self.metadata[X_OBJECTS_COUNT]) + 1
+ ocnt = self.metadata[X_OBJECTS_COUNT][0]
+ self.metadata[X_OBJECTS_COUNT] = (int(ocnt) + 1, timestamp)
self.metadata[X_PUT_TIMESTAMP] = timestamp
- self.metadata[X_BYTES_USED] = int(self.metadata[X_BYTES_USED]) + int(content_length)
+ bused = self.metadata[X_BYTES_USED][0]
+ self.metadata[X_BYTES_USED] = (int(bused) + int(content_length), timestamp)
#TODO: define update_metadata instad of writing whole metadata again.
self.put_metadata(self.metadata)
def delete_obj(self, content_length):
- self.metadata[X_OBJECTS_COUNT] = int(self.metadata[X_OBJECTS_COUNT]) - 1
- self.metadata[X_BYTES_USED] = int(self.metadata[X_BYTES_USED]) - int(content_length)
+ ocnt, timestamp = self.metadata[X_OBJECTS_COUNT][0]
+ self.metadata[X_OBJECTS_COUNT] = (int(ocnt) - 1, timestamp)
+ bused, timestamp = self.metadata[X_BYTES_USED]
+ self.metadata[X_BYTES_USED] = (int(bused) - int(content_length), timestamp)
self.put_metadata(self.metadata)
def put_container(self, container, put_timestamp, del_timestamp, object_count, bytes_used):
"""
For account server.
"""
- self.metadata[X_OBJECTS_COUNT] = 0
- self.metadata[X_BYTES_USED] = 0
- self.metadata[X_CONTAINER_COUNT] = int(self.metadata[X_CONTAINER_COUNT]) + 1
- self.metadata[X_PUT_TIMESTAMP] = 1
+ self.metadata[X_OBJECTS_COUNT] = (0, put_timestamp)
+ self.metadata[X_BYTES_USED] = (0, put_timestamp)
+ ccnt = self.metadata[X_CONTAINER_COUNT][0]
+ self.metadata[X_CONTAINER_COUNT] = (int(ccnt) + 1, put_timestamp)
+ self.metadata[X_PUT_TIMESTAMP] = (1, put_timestamp)
self.put_metadata(self.metadata)
def delete_container(self, object_count, bytes_used):
"""
For account server.
"""
- self.metadata[X_OBJECTS_COUNT] = 0
- self.metadata[X_BYTES_USED] = 0
- self.metadata[X_CONTAINER_COUNT] = int(self.metadata[X_CONTAINER_COUNT]) - 1
+ self.metadata[X_OBJECTS_COUNT] = (0, 0)
+ self.metadata[X_BYTES_USED] = (0, 0)
+ ccnt, timestamp = self.metadata[X_CONTAINER_COUNT]
+ self.metadata[X_CONTAINER_COUNT] = (int(ccnt) - 1, timestamp)
self.put_metadata(self.metadata)
def unlink(self):
@@ -264,10 +290,10 @@ class DiskDir(DiskCommon):
objects, object_count, bytes_used = get_container_details(self.datadir)
- if int(self.metadata[X_OBJECTS_COUNT]) != object_count or \
- int(self.metadata[X_BYTES_USED]) != bytes_used:
- self.metadata[X_OBJECTS_COUNT] = object_count
- self.metadata[X_BYTES_USED] = bytes_used
+ if int(self.metadata[X_OBJECTS_COUNT][0]) != object_count or \
+ int(self.metadata[X_BYTES_USED][0]) != bytes_used:
+ self.metadata[X_OBJECTS_COUNT] = (object_count, 0)
+ self.metadata[X_BYTES_USED] = (bytes_used, 0)
self.update_container(self.metadata)
if objects:
@@ -316,11 +342,10 @@ class DiskDir(DiskCommon):
bytes_used = 0
objects, object_count, bytes_used = get_container_details(self.datadir)
-
- if int(self.metadata[X_OBJECTS_COUNT]) != object_count or \
- int(self.metadata[X_BYTES_USED]) != bytes_used:
- self.metadata[X_OBJECTS_COUNT] = object_count
- self.metadata[X_BYTES_USED] = bytes_used
+ if int(self.metadata[X_OBJECTS_COUNT][0]) != object_count or \
+ int(self.metadata[X_BYTES_USED][0]) != bytes_used:
+ self.metadata[X_OBJECTS_COUNT] = (object_count, 0)
+ self.metadata[X_BYTES_USED] = (bytes_used, 0)
self.update_container(self.metadata)
def update_container_count(self):
@@ -329,8 +354,8 @@ class DiskDir(DiskCommon):
containers, container_count = get_account_details(self.datadir)
- if int(self.metadata[X_CONTAINER_COUNT]) != container_count:
- self.metadata[X_CONTAINER_COUNT] = container_count
+ if int(self.metadata[X_CONTAINER_COUNT][0]) != container_count:
+ self.metadata[X_CONTAINER_COUNT] = (container_count, 0)
self.update_account(self.metadata)
def get_info(self, include_metadata=False):
@@ -350,13 +375,13 @@ class DiskDir(DiskCommon):
metadata = {}
if os.path.exists(self.datadir):
- metadata = read_metadata(self.datadir)
+ metadata = _read_metadata(self.datadir)
data = {'account' : self.account, 'container' : self.name,
- 'object_count' : metadata.get(X_OBJECTS_COUNT, '0'),
- 'bytes_used' : metadata.get(X_BYTES_USED, '0'),
+ 'object_count' : metadata.get(X_OBJECTS_COUNT, ('0', 0))[0],
+ 'bytes_used' : metadata.get(X_BYTES_USED, ('0',0))[0],
'hash': '', 'id' : '', 'created_at' : '1',
- 'put_timestamp' : metadata.get(X_PUT_TIMESTAMP, '0'),
+ 'put_timestamp' : metadata.get(X_PUT_TIMESTAMP, ('0',0))[0],
'delete_timestamp' : '1',
'reported_put_timestamp' : '1', 'reported_delete_timestamp' : '1',
'reported_object_count' : '1', 'reported_bytes_used' : '1'}
@@ -401,7 +426,7 @@ class DiskAccount(DiskDir):
self.datadir = os.path.join(self.root, self.account)
if not check_mount(root, account):
check_valid_account(account, fs_object)
- self.metadata = read_metadata(self.datadir)
+ self.metadata = _read_metadata(self.datadir)
if not self.metadata or not validate_account(self.metadata):
self.metadata = create_account_metadata(self.datadir)
@@ -419,8 +444,8 @@ class DiskAccount(DiskDir):
containers, container_count = get_account_details(self.datadir)
- if int(self.metadata[X_CONTAINER_COUNT]) != container_count:
- self.metadata[X_CONTAINER_COUNT] = container_count
+ if int(self.metadata[X_CONTAINER_COUNT][0]) != container_count:
+ self.metadata[X_CONTAINER_COUNT] = (container_count, 0)
self.update_account(self.metadata)
if containers:
@@ -447,13 +472,13 @@ class DiskAccount(DiskDir):
list_item = []
metadata = None
list_item.append(cont)
- metadata = read_metadata(self.datadir + '/' + cont)
+ metadata = _read_metadata(self.datadir + '/' + cont)
if not metadata or not validate_container(metadata):
metadata = create_container_metadata(self.datadir + '/' + cont)
if metadata:
- list_item.append(metadata[X_OBJECTS_COUNT])
- list_item.append(metadata[X_BYTES_USED])
+ list_item.append(metadata[X_OBJECTS_COUNT][0])
+ list_item.append(metadata[X_BYTES_USED][0])
list_item.append(0)
account_list.append(list_item)
@@ -468,15 +493,15 @@ class DiskAccount(DiskDir):
"""
metadata = {}
if (os.path.exists(self.datadir)):
- metadata = read_metadata(self.datadir)
+ metadata = _read_metadata(self.datadir)
if not metadata:
metadata = create_account_metadata(self.datadir)
data = {'account' : self.account, 'created_at' : '1',
'put_timestamp' : '1', 'delete_timestamp' : '1',
- 'container_count' : metadata.get(X_CONTAINER_COUNT, 0),
- 'object_count' : metadata.get(X_OBJECTS_COUNT, 0),
- 'bytes_used' : metadata.get(X_BYTES_USED, 0),
+ 'container_count' : metadata.get(X_CONTAINER_COUNT, (0,0))[0],
+ 'object_count' : metadata.get(X_OBJECTS_COUNT, (0,0))[0],
+ 'bytes_used' : metadata.get(X_BYTES_USED, (0,0))[0],
'hash' : '', 'id' : ''}
if include_metadata:
diff --git a/swift/1.4.8/plugins/__init__.py b/swift/1.4.8/plugins/__init__.py
index 3d98c960..e69de29b 100644
--- a/swift/1.4.8/plugins/__init__.py
+++ b/swift/1.4.8/plugins/__init__.py
@@ -1,16 +0,0 @@
-# Copyright (c) 2011 Red Hat, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from Glusterfs import Glusterfs
diff --git a/swift/1.4.8/plugins/conf/account-server/1.conf b/swift/1.4.8/plugins/conf/account-server/1.conf
index 54cbf654..a82bc35e 100644
--- a/swift/1.4.8/plugins/conf/account-server/1.conf
+++ b/swift/1.4.8/plugins/conf/account-server/1.conf
@@ -6,13 +6,13 @@ user = root
log_facility = LOG_LOCAL2
[pipeline:main]
-pipeline = gluster account-server
+pipeline = glusterfs account-server
[app:account-server]
use = egg:swift#account
-[filter:gluster]
-use = egg:swift#gluster
+[filter:glusterfs]
+use = egg:swift#glusterfs
[account-replicator]
vm_test_mode = yes
diff --git a/swift/1.4.8/plugins/conf/container-server/1.conf b/swift/1.4.8/plugins/conf/container-server/1.conf
index 9e776838..e6d313c3 100644
--- a/swift/1.4.8/plugins/conf/container-server/1.conf
+++ b/swift/1.4.8/plugins/conf/container-server/1.conf
@@ -6,13 +6,13 @@ user = root
log_facility = LOG_LOCAL2
[pipeline:main]
-pipeline = gluster container-server
+pipeline = glusterfs container-server
[app:container-server]
use = egg:swift#container
-[filter:gluster]
-use = egg:swift#gluster
+[filter:glusterfs]
+use = egg:swift#glusterfs
[container-replicator]
vm_test_mode = yes
diff --git a/swift/1.4.8/plugins/conf/object-server/1.conf b/swift/1.4.8/plugins/conf/object-server/1.conf
index f191cefc..665aa94b 100644
--- a/swift/1.4.8/plugins/conf/object-server/1.conf
+++ b/swift/1.4.8/plugins/conf/object-server/1.conf
@@ -6,13 +6,13 @@ user = root
log_facility = LOG_LOCAL2
[pipeline:main]
-pipeline = gluster object-server
+pipeline = glusterfs object-server
[app:object-server]
use = egg:swift#object
-[filter:gluster]
-use = egg:swift#gluster
+[filter:glusterfs]
+use = egg:swift#glusterfs
[object-replicator]
vm_test_mode = yes
diff --git a/swift/1.4.8/plugins/conf/proxy-server.conf b/swift/1.4.8/plugins/conf/proxy-server.conf
index 1fcde8e0..9bec6425 100644
--- a/swift/1.4.8/plugins/conf/proxy-server.conf
+++ b/swift/1.4.8/plugins/conf/proxy-server.conf
@@ -4,7 +4,7 @@ user = root
log_facility = LOG_LOCAL1
[pipeline:main]
-pipeline = healthcheck cache tempauth proxy-server
+pipeline = healthcheck cache tempauth gluster proxy-server
[app:proxy-server]
use = egg:swift#proxy
@@ -19,3 +19,6 @@ use = egg:swift#healthcheck
[filter:cache]
use = egg:swift#memcache
+
+[filter:gluster]
+use = egg:swift#gluster \ No newline at end of file
diff --git a/swift/1.4.8/plugins/conf/swift.conf b/swift/1.4.8/plugins/conf/swift.conf
index f9864e35..e506b6f5 100644
--- a/swift/1.4.8/plugins/conf/swift.conf
+++ b/swift/1.4.8/plugins/conf/swift.conf
@@ -1,7 +1,92 @@
[DEFAULT]
Enable_plugin = yes
+
[swift-hash]
# random unique string that can never change (DO NOT LOSE)
swift_hash_path_suffix = gluster
+
+# The swift-constraints section sets the basic constraints on data
+# saved in the swift cluster.
+
+[swift-constraints]
+
+# max_file_size is the largest "normal" object that can be saved in
+# the cluster. This is also the limit on the size of each segment of
+# a "large" object when using the large object manifest support.
+# This value is set in bytes. Setting it to lower than 1MiB will cause
+# some tests to fail. It is STRONGLY recommended to leave this value at
+# the default (5 * 2**30 + 2).
+
+# FIXME: Really? Gluster can handle a 2^64 sized file? And can the fronting
+# web service handle such a size? I think with UFO, we need to keep with the
+# default size from Swift and encourage users to research what size their web
+# services infrastructure can handle.
+
+max_file_size = 18446744073709551616
+
+
+# max_meta_name_length is the max number of bytes in the utf8 encoding
+# of the name portion of a metadata header.
+
+#max_meta_name_length = 128
+
+
+# max_meta_value_length is the max number of bytes in the utf8 encoding
+# of a metadata value
+
+#max_meta_value_length = 256
+
+
+# max_meta_count is the max number of metadata keys that can be stored
+# on a single account, container, or object
+
+#max_meta_count = 90
+
+
+# max_meta_overall_size is the max number of bytes in the utf8 encoding
+# of the metadata (keys + values)
+
+#max_meta_overall_size = 4096
+
+
+# max_object_name_length is the max number of bytes in the utf8 encoding of an
+# object name: Gluster FS can handle much longer file names, but the length
+# between the slashes of the URL is handled below. Remember that most web
+# clients can't handle anything greater than 2048, and those that do are
+# rather clumsy.
+
+max_object_name_length = 2048
+
+# max_object_name_component_length (GlusterFS) is the max number of bytes in
+# the utf8 encoding of an object name component (the part between the
+# slashes); this is a limit imposed by the underlying file system (for XFS it
+# is 255 bytes).
+
+max_object_name_component_length = 255
+
+# container_listing_limit is the default (and max) number of items
+# returned for a container listing request
+
+#container_listing_limit = 10000
+
+
+# account_listing_limit is the default (and max) number of items returned
+# for an account listing request
+
+#account_listing_limit = 10000
+
+
+# max_account_name_length is the max number of bytes in the utf8 encoding of
+# an account name: Gluster FS Filename limit (XFS limit?), must be the same
+# size as max_object_name_component_length above.
+
+max_account_name_length = 255
+
+
+# max_container_name_length is the max number of bytes in the utf8 encoding
+# of a container name: Gluster FS Filename limit (XFS limit?), must be the same
+# size as max_object_name_component_length above.
+
+max_container_name_length = 255
diff --git a/swift/1.4.8/plugins/constraints.py b/swift/1.4.8/plugins/constraints.py
index 26640717..d5220908 100644
--- a/swift/1.4.8/plugins/constraints.py
+++ b/swift/1.4.8/plugins/constraints.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2011 Red Hat, Inc.
+# Copyright (c) 2012 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -13,46 +13,28 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import os
-import logging
-from swift.common.constraints import check_utf8, check_metadata
+import swift.common.constraints
-from webob.exc import HTTPBadRequest, HTTPLengthRequired, \
- HTTPRequestEntityTooLarge
+MAX_OBJECT_NAME_COMPONENT_LENGTH = swift.common.constraints.constraints_conf_int(
+ 'max_object_name_component_length', 255)
-
-#: Max file size allowed for objects
-MAX_FILE_SIZE = 0xffffffffffffffff
-#: Max length of the name of a key for metadata
-MAX_META_NAME_LENGTH = 128
-#: Max length of the value of a key for metadata
-MAX_META_VALUE_LENGTH = 256
-#: Max number of metadata items
-MAX_META_COUNT = 90
-#: Max overall size of metadata
-MAX_META_OVERALL_SIZE = 4096
-#: Max object name length
-MAX_OBJECT_NAME_LENGTH = 255
-#: Max object list length of a get request for a container
-CONTAINER_LISTING_LIMIT = 10000
-#: Max container list length of a get request for an account
-ACCOUNT_LISTING_LIMIT = 10000
-MAX_ACCOUNT_NAME_LENGTH = 255
-MAX_CONTAINER_NAME_LENGTH = 255
-
-def validate_obj_name(obj):
- if len(obj) > MAX_OBJECT_NAME_LENGTH:
- logging.error('Object name too long %s' % obj)
- return False
+def validate_obj_name_component(obj):
+ if len(obj) > MAX_OBJECT_NAME_COMPONENT_LENGTH:
+ return 'too long (%d)' % len(obj)
if obj == '.' or obj == '..':
- logging.error('Object name cannot be . or .. %s' % obj)
- return False
+ return 'cannot be . or ..'
+ return ''
- return True
+# Save the original
+__check_object_creation = swift.common.constraints.check_object_creation
-def check_object_creation(req, object_name):
+# Define our new one which invokes the original
+def gluster_check_object_creation(req, object_name):
"""
Check to ensure that everything is alright about an object to be created.
+ Monkey patches swift.common.constraints.check_object_creation, invoking
+ the original, and then adding an additional check for individual object
+ name components.
:param req: HTTP request object
:param object_name: name of object to be created
@@ -62,38 +44,18 @@ def check_object_creation(req, object_name):
:raises HTTPBadRequest: missing or bad content-type header, or
bad metadata
"""
- if req.content_length and req.content_length > MAX_FILE_SIZE:
- return HTTPRequestEntityTooLarge(body='Your request is too large.',
- request=req,
- content_type='text/plain')
- if req.content_length is None and \
- req.headers.get('transfer-encoding') != 'chunked':
- return HTTPLengthRequired(request=req)
- if 'X-Copy-From' in req.headers and req.content_length:
- return HTTPBadRequest(body='Copy requests require a zero byte body',
- request=req, content_type='text/plain')
- for obj in object_name.split('/'):
- if not validate_obj_name(obj):
- return HTTPBadRequest(body='Invalid object name %s' %
- (obj), request=req,
- content_type='text/plain')
- if 'Content-Type' not in req.headers:
- return HTTPBadRequest(request=req, content_type='text/plain',
- body='No content type')
- if not check_utf8(req.headers['Content-Type']):
- return HTTPBadRequest(request=req, body='Invalid Content-Type',
- content_type='text/plain')
- if 'x-object-manifest' in req.headers:
- value = req.headers['x-object-manifest']
- container = prefix = None
- try:
- container, prefix = value.split('/', 1)
- except ValueError:
- pass
- if not container or not prefix or '?' in value or '&' in value or \
- prefix[0] == '/':
- return HTTPBadRequest(
- request=req,
- body='X-Object-Manifest must in the format container/prefix')
- return check_metadata(req, 'object')
+ ret = __check_object_creation(req, object_name)
+
+ if ret is None:
+ for obj in object_name.split('/'):
+ reason = validate_obj_name(obj)
+ if reason:
+ bdy = 'Invalid object name "%s", component "%s" %s' \
+ % (object_name, obj, reason)
+ ret = HTTPBadRequest(body=bdy,
+ request=req,
+ content_type='text/plain')
+
+ return ret
+swift.common.constraints.check_object_creation = gluster_check_object_creation
diff --git a/swift/1.4.8/plugins/middleware/__init__.py b/swift/1.4.8/plugins/middleware/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/swift/1.4.8/plugins/middleware/__init__.py
diff --git a/swift/1.4.8/plugins/middleware/gluster.py b/swift/1.4.8/plugins/middleware/gluster.py
new file mode 100644
index 00000000..e0e65e35
--- /dev/null
+++ b/swift/1.4.8/plugins/middleware/gluster.py
@@ -0,0 +1,40 @@
+# Copyright (c) 2012 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Noop Middleware that simply allows us to monkey patch the constraints
+import swift.plugins.constraints
+
+class Gluster(object):
+ """
+ Noop middleware for use with the proxy server to get paste.deploy to load
+ this middleware such that the plugin constraints monkey patch the common
+ constraints ahead of their use.
+ """
+ def __init__(self, app, conf):
+ self.app = app
+ self.conf = conf
+
+ def __call__(self, env, start_response):
+ return self.app(env, start_response)
+
+
+def filter_factory(global_conf, **local_conf):
+ """Returns a WSGI filter app for use with paste.deploy."""
+ conf = global_conf.copy()
+ conf.update(local_conf)
+
+ def gluster_filter(app):
+ return Gluster(app, conf)
+ return gluster_filter
diff --git a/swift/1.4.8/plugins/middleware/glusterfs.py b/swift/1.4.8/plugins/middleware/glusterfs.py
new file mode 100644
index 00000000..74721464
--- /dev/null
+++ b/swift/1.4.8/plugins/middleware/glusterfs.py
@@ -0,0 +1,55 @@
+# Copyright (c) 2012 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Monkey patch constraints
+import swift.plugins.constraints
+from swift.plugins.Glusterfs import Glusterfs
+from ConfigParser import ConfigParser
+
+fs_conf = ConfigParser()
+if fs_conf.read('/etc/swift/fs.conf'):
+ try:
+ mount_path = fs_conf.get ('DEFAULT', 'mount_path')
+ except NoSectionError, NoOptionError:
+ # FIXME - How to log during module initialization
+ logger.exception(_('ERROR mount_path not present'))
+ mount_path = ''
+else:
+ mount_path = ''
+
+
+class Gluster(object):
+ """
+ Update the environment with keys that reflect GlusterFS middleware enabled
+ """
+ def __init__(self, app, conf):
+ self.app = app
+ self.conf = conf
+
+ def __call__(self, env, start_response):
+ env['Gluster_enabled'] = True
+ env['fs_object'] = Glusterfs()
+ env['root'] = mount_path
+ return self.app(env, start_response)
+
+
+def filter_factory(global_conf, **local_conf):
+ """Returns a WSGI filter app for use with paste.deploy."""
+ conf = global_conf.copy()
+ conf.update(local_conf)
+
+ def gluster_filter(app):
+ return Gluster(app, conf)
+ return gluster_filter
diff --git a/swift/1.4.8/plugins/utils.py b/swift/1.4.8/plugins/utils.py
index 1c7eeb92..59e70470 100644
--- a/swift/1.4.8/plugins/utils.py
+++ b/swift/1.4.8/plugins/utils.py
@@ -18,8 +18,9 @@ import os
import errno
import xattr
from hashlib import md5
-from swift.common.utils import normalize_timestamp
+from swift.common.utils import normalize_timestamp, TRUE_VALUES
import cPickle as pickle
+from ConfigParser import ConfigParser, NoSectionError, NoOptionError
X_CONTENT_TYPE = 'Content-Type'
X_CONTENT_LENGTH = 'Content-Length'
@@ -174,6 +175,18 @@ def do_rename(old_path, new_path):
raise
return True
+def _add_timestamp(metadata_i):
+ # At this point we have a simple key/value dictionary, turn it into
+ # key/(value,timestamp) pairs.
+ timestamp = 0
+ metadata = {}
+ for key, value_i in metadata_i.iteritems():
+ if not isinstance(value_i, tuple):
+ metadata[key] = (value_i, timestamp)
+ else:
+ metadata[key] = value_i
+ return metadata
+
def read_metadata(path):
"""
Helper function to read the pickled metadata from a File/Directory .
@@ -301,7 +314,8 @@ def validate_container(metadata):
#logging.warn('validate_container: Metadata missing entries: %s' % metadata)
return False
- if metadata[X_TYPE] == CONTAINER:
+ (value, timestamp) = metadata[X_TYPE]
+ if value == CONTAINER:
return True
logging.warn('validate_container: metadata type is not CONTAINER (%r)' % (value,))
@@ -321,7 +335,8 @@ def validate_account(metadata):
#logging.warn('validate_account: Metadata missing entries: %s' % metadata)
return False
- if metadata[X_TYPE] == ACCOUNT:
+ (value, timestamp) = metadata[X_TYPE]
+ if value == ACCOUNT:
return True
logging.warn('validate_account: metadata type is not ACCOUNT (%r)' % (value,))
@@ -552,7 +567,7 @@ def get_container_metadata(cont_path, memcache=None):
X_PUT_TIMESTAMP: normalize_timestamp(os.path.getmtime(cont_path)),
X_OBJECTS_COUNT: object_count,
X_BYTES_USED: bytes_used}
- return metadata
+ return _add_timestamp(metadata)
def get_account_metadata(acc_path, memcache=None):
containers = []
@@ -564,7 +579,7 @@ def get_account_metadata(acc_path, memcache=None):
X_OBJECTS_COUNT: 0,
X_BYTES_USED: 0,
X_CONTAINER_COUNT: container_count}
- return metadata
+ return _add_timestamp(metadata)
def restore_object(obj_path, metadata):
meta = read_metadata(obj_path)
@@ -623,3 +638,14 @@ def get_account_list(fs_object):
def get_account_id(account):
return RESELLER_PREFIX + md5(account + HASH_PATH_SUFFIX).hexdigest()
+
+__swift_conf = ConfigParser()
+__swift_conf.read(os.path.join('/etc/swift', 'swift.conf'))
+try:
+ _plugin_enabled = __swift_conf.get('DEFAULT', 'Enable_plugin', 'no') in TRUE_VALUES
+except NoOptionError, NoSectionError:
+ _plugin_enabled = False
+del __swift_conf
+
+def plugin_enabled():
+ return _plugin_enabled