diff options
| author | Peter Portante <peter.portante@redhat.com> | 2013-05-08 21:51:58 -0400 |
|---|---|---|
| committer | Anand Avati <avati@redhat.com> | 2013-05-10 11:03:55 -0700 |
| commit | 40d026e10013f533c4fee33b87dabc4ca11c94b3 (patch) | |
| tree | aa6315c23a49288b72424f58a71c3f87d8aca7ff /ufo | |
| parent | fef1270fc2d665c67084c4ce06f60fc8faed4d52 (diff) | |
object-storage: final removal of ufo code
See https://git.gluster.org/gluster-swift.git for the new location of
the Gluster-Swift code.
With this patch, no OpenStack Swift related RPMs are constructed.
This patch also removes the unused code that references the
user.ufo-test xattr key in the DHT translator.
Change-Id: I2da32642cbd777737a41c5f9f6d33f059c85a2c1
BUG: 961902 (https://bugzilla.redhat.com/show_bug.cgi?id=961902)
Signed-off-by: Peter Portante <peter.portante@redhat.com>
Reviewed-on: http://review.gluster.org/4970
Reviewed-by: Kaleb KEITHLEY <kkeithle@redhat.com>
Reviewed-by: Luis Pabon <lpabon@redhat.com>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Anand Avati <avati@redhat.com>
Diffstat (limited to 'ufo')
55 files changed, 0 insertions, 5108 deletions
diff --git a/ufo/README b/ufo/README deleted file mode 100644 index 9efd918b59a..00000000000 --- a/ufo/README +++ /dev/null @@ -1,16 +0,0 @@ -Gluster Unified File and Object Storage allows files and directories created -via gluster-native/nfs mount to be accessed as containers and objects. It is -a plugin for OpenStack Swift project. - -Install - * TBD - -Once this is done, you can access the GlusterFS volumes as Swift accounts. -Add the Volume names with the user-name and its corresponding password to the -/etc/swift/proxy-server.conf (follow the syntax used in the sample conf file). - -Command to start the servers (TBD) - swift-init main start - -Command to stop the servers (TBD) - swift-init main stop diff --git a/ufo/bin/gluster-swift-gen-builders b/ufo/bin/gluster-swift-gen-builders deleted file mode 100755 index bef5d1493de..00000000000 --- a/ufo/bin/gluster-swift-gen-builders +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -# Note that these port numbers must match the configured values for the -# various servers in their configuration files. -declare -A port=(["account.builder"]=6012 ["container.builder"]=6011 \ - ["object.builder"]=6010) - -builder_files="account.builder container.builder object.builder" - -function create { - swift-ring-builder $1 create 1 1 1 >> /tmp/out -} - -function add { - swift-ring-builder $1 add z$2-127.0.0.1:$3/$4_ 100.0 -} - -function rebalance { - swift-ring-builder $1 rebalance -} - -function build { - swift-ring-builder $1 -} - -if [ "$1x" = "x" ]; then - echo "Please specify the gluster volume name to use." - exit 1 -fi - -for builder_file in $builder_files -do - create $builder_file - - zone=1 - for volname in $@ - do - add $builder_file $zone ${port[$builder_file]} $volname - zone=$(expr $zone + 1) - done - - rebalance $builder_file - build $builder_file -done diff --git a/ufo/etc/account-server/1.conf-gluster b/ufo/etc/account-server/1.conf-gluster deleted file mode 100644 index da8f3172634..00000000000 --- a/ufo/etc/account-server/1.conf-gluster +++ /dev/null @@ -1,19 +0,0 @@ -[DEFAULT] -devices = /mnt/gluster-object -mount_check = true -bind_port = 6012 -user = root -log_facility = LOG_LOCAL2 - -[pipeline:main] -pipeline = account-server - -[app:account-server] -use = egg:gluster_swift_ufo#account - -[account-replicator] -vm_test_mode = yes - -[account-auditor] - -[account-reaper] diff --git a/ufo/etc/container-server/1.conf-gluster b/ufo/etc/container-server/1.conf-gluster deleted file mode 100644 index acad6213543..00000000000 --- a/ufo/etc/container-server/1.conf-gluster +++ /dev/null @@ -1,21 +0,0 @@ -[DEFAULT] -devices = /mnt/gluster-object -mount_check = true -bind_port = 6011 -user = root -log_facility = LOG_LOCAL2 - -[pipeline:main] -pipeline = container-server - -[app:container-server] -use = egg:gluster_swift_ufo#container - -[container-replicator] -vm_test_mode = yes - -[container-updater] - -[container-auditor] - -[container-sync] diff --git a/ufo/etc/fs.conf-gluster b/ufo/etc/fs.conf-gluster deleted file mode 100644 index 71a9b03130b..00000000000 --- a/ufo/etc/fs.conf-gluster +++ /dev/null @@ -1,17 +0,0 @@ -[DEFAULT] -# IP address of a GlusterFS volume server member. By default, we assume the -# local host. -mount_ip = localhost - -# By default it is assumed the Gluster volumes can be accessed using other -# methods besides UFO (not object only), which disables a caching -# optimizations in order to keep in sync with file system changes. -object_only = no - -# Performance optimization parameter. When turned off, the filesystem will -# see a reduced number of stat calls, resulting in substantially faster -# response time for GET and HEAD container requests on containers with large -# numbers of objects, at the expense of an accurate count of combined bytes -# used by all objects in the container. For most installations "off" works -# fine. -accurate_size_in_listing = off
\ No newline at end of file diff --git a/ufo/etc/object-server/1.conf-gluster b/ufo/etc/object-server/1.conf-gluster deleted file mode 100644 index 0d85546cdc0..00000000000 --- a/ufo/etc/object-server/1.conf-gluster +++ /dev/null @@ -1,36 +0,0 @@ -[DEFAULT] -devices = /mnt/gluster-object -mount_check = true -bind_port = 6010 -# If not doing the above, setting this value initially to match the number of -# CPUs is a good starting point for determining the right value. -workers = 1 - -[pipeline:main] -pipeline = object-server - -[app:object-server] -use = egg:gluster_swift_ufo#object -user = root -log_facility = LOG_LOCAL2 -# Timeout clients that don't read or write to the proxy server after 5 -# seconds. -conn_timeout = 5 -# For high load situations, once connected to a container server, allow for -# delays communicating with it. -node_timeout = 60 -# Adjust this value to match the stripe width of the underlying storage array -# (not the stripe element size). This will provide a reasonable starting point -# for tuning this value. -disk_chunk_size = 65536 -# Adjust this value match whatever is set for the disk_chunk_size -# initially. This will provide a reasonable starting point for tuning this -# value. -network_chunk_size = 65556 - -[object-replicator] -vm_test_mode = yes - -[object-updater] - -[object-auditor] diff --git a/ufo/etc/proxy-server.conf-gluster b/ufo/etc/proxy-server.conf-gluster deleted file mode 100644 index e04efecea52..00000000000 --- a/ufo/etc/proxy-server.conf-gluster +++ /dev/null @@ -1,69 +0,0 @@ -[DEFAULT] -bind_port = 8080 -user = root -log_facility = LOG_LOCAL1 -# Consider using 1 worker per CPU -workers = 1 - -[pipeline:main] -pipeline = healthcheck cache tempauth proxy-server - -[app:proxy-server] -use = egg:gluster_swift_ufo#proxy -log_facility = LOG_LOCAL1 -# The API allows for account creation and deletion, but since Gluster/Swift -# automounts a Gluster volume for a given account, there is no way to create -# or delete an account. So leave this off. -allow_account_management = false -account_autocreate = true -# Only need to recheck the account exists once a day -recheck_account_existence = 86400 -# May want to consider bumping this up if containers are created and destroyed -# infrequently. -recheck_container_existence = 60 -# Timeout clients that don't read or write to the proxy server after 5 -# seconds. -client_timeout = 5 -# Give more time to connect to the object, container or account servers in -# cases of high load. -conn_timeout = 5 -# For high load situations, once connected to an object, container or account -# server, allow for delays communicating with them. -node_timeout = 60 -# May want to consider bumping up this value to 1 - 4 MB depending on how much -# traffic is for multi-megabyte or gigabyte requests; perhaps matching the -# stripe width (not stripe element size) of your storage volume is a good -# starting point. See below for sizing information. -object_chunk_size = 65536 -# If you do decide to increase the object_chunk_size, then consider lowering -# this value to one. Up to "put_queue_length" object_chunk_size'd buffers can -# be queued to the object server for processing. Given one proxy server worker -# can handle up to 1,024 connections, by default, it will consume 10 * 65,536 -# * 1,024 bytes of memory in the worse case (default values). Be sure the -# amount of memory available on the system can accommodate increased values -# for object_chunk_size. -put_queue_depth = 10 - -[filter:tempauth] -use = egg:swift#tempauth -# Here you need to add users explicitly. See the OpenStack Swift Deployment -# Guide for more information. The user and user64 directives take the -# following form: -# user_<account>_<username> = <key> [group] [group] [...] [storage_url] -# user64_<account_b64>_<username_b64> = <key> [group] [group] [...] [storage_url] -# Where you use user64 for accounts and/or usernames that include underscores. -# -# NOTE (and WARNING): The account name must match the device name specified -# when generating the account, container, and object build rings. -# -# E.g. -# user_ufo0_admin = abc123 .admin - -[filter:healthcheck] -use = egg:swift#healthcheck - -[filter:cache] -use = egg:swift#memcache -# Update this line to contain a comma separated list of memcache servers -# shared by all nodes running the proxy-server service. -memcache_servers = localhost:11211 diff --git a/ufo/etc/swift.conf-gluster b/ufo/etc/swift.conf-gluster deleted file mode 100644 index 25c3ca1571f..00000000000 --- a/ufo/etc/swift.conf-gluster +++ /dev/null @@ -1,91 +0,0 @@ -[DEFAULT] - - -[swift-hash] -# random unique string that can never change (DO NOT LOSE) -swift_hash_path_suffix = gluster - - -# The swift-constraints section sets the basic constraints on data -# saved in the swift cluster. - -[swift-constraints] - -# max_file_size is the largest "normal" object that can be saved in -# the cluster. This is also the limit on the size of each segment of -# a "large" object when using the large object manifest support. -# This value is set in bytes. Setting it to lower than 1MiB will cause -# some tests to fail. It is STRONGLY recommended to leave this value at -# the default (5 * 2**30 + 2). - -# FIXME: Really? Gluster can handle a 2^64 sized file? And can the fronting -# web service handle such a size? I think with UFO, we need to keep with the -# default size from Swift and encourage users to research what size their web -# services infrastructure can handle. - -max_file_size = 18446744073709551616 - - -# max_meta_name_length is the max number of bytes in the utf8 encoding -# of the name portion of a metadata header. - -#max_meta_name_length = 128 - - -# max_meta_value_length is the max number of bytes in the utf8 encoding -# of a metadata value - -#max_meta_value_length = 256 - - -# max_meta_count is the max number of metadata keys that can be stored -# on a single account, container, or object - -#max_meta_count = 90 - - -# max_meta_overall_size is the max number of bytes in the utf8 encoding -# of the metadata (keys + values) - -#max_meta_overall_size = 4096 - - -# max_object_name_length is the max number of bytes in the utf8 encoding of an -# object name: Gluster FS can handle much longer file names, but the length -# between the slashes of the URL is handled below. Remember that most web -# clients can't handle anything greater than 2048, and those that do are -# rather clumsy. - -max_object_name_length = 2048 - -# max_object_name_component_length (GlusterFS) is the max number of bytes in -# the utf8 encoding of an object name component (the part between the -# slashes); this is a limit imposed by the underlying file system (for XFS it -# is 255 bytes). - -max_object_name_component_length = 255 - -# container_listing_limit is the default (and max) number of items -# returned for a container listing request - -#container_listing_limit = 10000 - - -# account_listing_limit is the default (and max) number of items returned -# for an account listing request - -#account_listing_limit = 10000 - - -# max_account_name_length is the max number of bytes in the utf8 encoding of -# an account name: Gluster FS Filename limit (XFS limit?), must be the same -# size as max_object_name_component_length above. - -max_account_name_length = 255 - - -# max_container_name_length is the max number of bytes in the utf8 encoding -# of a container name: Gluster FS Filename limit (XFS limit?), must be the same -# size as max_object_name_component_length above. - -max_container_name_length = 255 diff --git a/ufo/gluster-swift-ufo.spec b/ufo/gluster-swift-ufo.spec deleted file mode 100644 index 8c0167cb73f..00000000000 --- a/ufo/gluster-swift-ufo.spec +++ /dev/null @@ -1,79 +0,0 @@ -############################################################################################################ -# Command to build rpms.# -# $ rpmbuild -ta %{name}-%{version}-%{release}.tar.gz # -############################################################################################################ -# Setting up the environment. # -# * Create a directory %{name}-%{version} under $HOME/rpmbuild/SOURCES # -# * Copy the contents of gluster directory into $HOME/rpmbuild/SOURCES/%{name}-%{version} # -# * tar zcvf %{name}-%{version}-%{release}.tar.gz $HOME/rpmbuild/SOURCES/%{name}-%{version} %{name}.spec # -# For more information refer # -# http://fedoraproject.org/wiki/How_to_create_an_RPM_package # -############################################################################################################ - -%if ! (0%{?fedora} > 12 || 0%{?rhel} > 5) -%{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")} -%endif - -%define _confdir %{_sysconfdir}/swift -%define _ufo_version 1.1 -%define _ufo_release 4 - -Summary : GlusterFS Unified File and Object Storage. -Name : gluster-swift-ufo -Version : %{_ufo_version} -Release : %{_ufo_release} -Group : Application/File -Vendor : Red Hat Inc. -Source0 : %{name}-%{version}-%{release}.tar.gz -Packager : gluster-users@gluster.org -License : Apache -BuildArch: noarch -Requires : memcached -Requires : openssl -Requires : python -Requires : openstack-swift >= 1.4.8 -Requires : openstack-swift-account >= 1.4.8 -Requires : openstack-swift-container >= 1.4.8 -Requires : openstack-swift-object >= 1.4.8 -Requires : openstack-swift-proxy >= 1.4.8 -Obsoletes: gluster-swift -Obsoletes: gluster-swift-plugin - -%description -Gluster Unified File and Object Storage unifies NAS and object storage -technology. This provides a system for data storage that enables users to access -the same data as an object and as a file, simplifying management and controlling -storage costs. - -%prep -%setup -q - -%build -%{__python} setup.py build - -%install -rm -rf %{buildroot} - -%{__python} setup.py install -O1 --skip-build --root %{buildroot} - -mkdir -p %{buildroot}/%{_confdir}/ -cp -r etc/* %{buildroot}/%{_confdir}/ - -mkdir -p %{buildroot}/%{_bindir}/ -cp bin/gluster-swift-gen-builders %{buildroot}/%{_bindir}/ - -%clean -rm -rf %{buildroot} - -%files -%defattr(-,root,root) -%{python_sitelib}/gluster -%{python_sitelib}/gluster_swift_ufo-%{version}-*.egg-info -%{_bindir}/gluster-swift-gen-builders -%dir %{_confdir} -%config %{_confdir}/account-server/1.conf-gluster -%config %{_confdir}/container-server/1.conf-gluster -%config %{_confdir}/object-server/1.conf-gluster -%config %{_confdir}/swift.conf-gluster -%config %{_confdir}/proxy-server.conf-gluster -%config %{_confdir}/fs.conf-gluster diff --git a/ufo/gluster/__init__.py b/ufo/gluster/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/ufo/gluster/__init__.py +++ /dev/null diff --git a/ufo/gluster/swift/__init__.py b/ufo/gluster/swift/__init__.py deleted file mode 100644 index ef350fd5111..00000000000 --- a/ufo/gluster/swift/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -""" Gluster Swift UFO """ - -class Version(object): - def __init__(self, canonical_version, final): - self.canonical_version = canonical_version - self.final = final - - @property - def pretty_version(self): - if self.final: - return self.canonical_version - else: - return '%s-dev' % (self.canonical_version,) - - -_version = Version('1.1', False) -__version__ = _version.pretty_version -__canonical_version__ = _version.canonical_version diff --git a/ufo/gluster/swift/account/__init__.py b/ufo/gluster/swift/account/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/ufo/gluster/swift/account/__init__.py +++ /dev/null diff --git a/ufo/gluster/swift/account/server.py b/ufo/gluster/swift/account/server.py deleted file mode 100644 index aeaabc94238..00000000000 --- a/ufo/gluster/swift/account/server.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" Account Server for Gluster Swift UFO """ - -# Simply importing this monkey patches the constraint handling to fit our -# needs -import gluster.swift.common.constraints - -from swift.account import server -from gluster.swift.common.DiskDir import DiskAccount - - -class AccountController(server.AccountController): - def _get_account_broker(self, drive, part, account): - """ - Overriden to provide the GlusterFS specific broker that talks to - Gluster for the information related to servicing a given request - instead of talking to a database. - - :param drive: drive that holds the container - :param part: partition the container is in - :param account: account name - :returns: DiskDir object - """ - return DiskAccount(self.root, drive, account, self.logger) - - -def app_factory(global_conf, **local_conf): - """paste.deploy app factory for creating WSGI account server apps.""" - conf = global_conf.copy() - conf.update(local_conf) - return AccountController(conf) diff --git a/ufo/gluster/swift/common/DiskDir.py b/ufo/gluster/swift/common/DiskDir.py deleted file mode 100644 index 18d08cc0f16..00000000000 --- a/ufo/gluster/swift/common/DiskDir.py +++ /dev/null @@ -1,496 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os, errno - -from gluster.swift.common.utils import clean_metadata, dir_empty, rmdirs, \ - mkdirs, validate_account, validate_container, is_marker, \ - get_container_details, get_account_details, get_container_metadata, \ - create_container_metadata, create_account_metadata, DEFAULT_GID, \ - DEFAULT_UID, validate_object, create_object_metadata, read_metadata, \ - write_metadata, X_CONTENT_TYPE, X_CONTENT_LENGTH, X_TIMESTAMP, \ - X_PUT_TIMESTAMP, X_TYPE, X_ETAG, X_OBJECTS_COUNT, X_BYTES_USED, \ - X_CONTAINER_COUNT, CONTAINER, os_path -from gluster.swift.common import Glusterfs - -from swift.common.constraints import CONTAINER_LISTING_LIMIT -from swift.common.utils import normalize_timestamp, TRUE_VALUES - - -DATADIR = 'containers' - -# Create a dummy db_file in /etc/swift -_unittests_enabled = os.getenv('GLUSTER_UNIT_TEST_ENABLED', 'no') -if _unittests_enabled in TRUE_VALUES: - _tmp_dir = '/tmp/gluster_unit_tests' - try: - os.mkdir(_tmp_dir) - except OSError as e: - if e.errno != errno.EEXIST: - raise - _db_file = os.path.join(_tmp_dir, 'db_file.db') -else: - _db_file = '/etc/swift/db_file.db' -if not os.path.exists(_db_file): - file(_db_file, 'w+') - - -def _read_metadata(dd): - """ Filter read metadata so that it always returns a tuple that includes - some kind of timestamp. With 1.4.8 of the Swift integration the - timestamps were not stored. Here we fabricate timestamps for volumes - where the existing data has no timestamp (that is, stored data is not - a tuple), allowing us a measure of backward compatibility. - - FIXME: At this time it does not appear that the timestamps on each - metadata are used for much, so this should not hurt anything. - """ - metadata_i = read_metadata(dd) - metadata = {} - timestamp = 0 - for key, value in metadata_i.iteritems(): - if not isinstance(value, tuple): - value = (value, timestamp) - metadata[key] = value - return metadata - - -class DiskCommon(object): - def is_deleted(self): - return not os_path.exists(self.datadir) - - def filter_prefix(self, objects, prefix): - """ - Accept sorted list. - """ - found = 0 - filtered_objs = [] - for object_name in objects: - if object_name.startswith(prefix): - filtered_objs.append(object_name) - found = 1 - else: - if found: - break - return filtered_objs - - def filter_delimiter(self, objects, delimiter, prefix): - """ - Accept sorted list. - Objects should start with prefix. - """ - filtered_objs=[] - for object_name in objects: - tmp_obj = object_name.replace(prefix, '', 1) - sufix = tmp_obj.split(delimiter, 1) - new_obj = prefix + sufix[0] - if new_obj and new_obj not in filtered_objs: - filtered_objs.append(new_obj) - - return filtered_objs - - def filter_marker(self, objects, marker): - """ - TODO: We can traverse in reverse order to optimize. - Accept sorted list. - """ - filtered_objs=[] - found = 0 - if objects[-1] < marker: - return filtered_objs - for object_name in objects: - if object_name > marker: - filtered_objs.append(object_name) - - return filtered_objs - - def filter_end_marker(self, objects, end_marker): - """ - Accept sorted list. - """ - filtered_objs=[] - for object_name in objects: - if object_name < end_marker: - filtered_objs.append(object_name) - else: - break - - return filtered_objs - - def filter_limit(self, objects, limit): - filtered_objs=[] - for i in range(0, limit): - filtered_objs.append(objects[i]) - - return filtered_objs - - -class DiskDir(DiskCommon): - """ - Manage object files on disk. - - :param path: path to devices on the node - :param drive: gluster volume drive name - :param account: account name for the object - :param container: container name for the object - :param logger: account or container server logging object - :param uid: user ID container object should assume - :param gid: group ID container object should assume - """ - - def __init__(self, path, drive, account, container, logger, - uid=DEFAULT_UID, gid=DEFAULT_GID): - self.root = path - if container: - self.container = container - else: - self.container = None - if self.container: - self.datadir = os.path.join(path, drive, self.container) - else: - self.datadir = os.path.join(path, drive) - self.account = account - assert logger is not None - self.logger = logger - self.metadata = {} - self.container_info = None - self.object_info = None - self.uid = int(uid) - self.gid = int(gid) - self.db_file = _db_file - self.dir_exists = os_path.exists(self.datadir) - if self.dir_exists: - try: - self.metadata = _read_metadata(self.datadir) - except EOFError: - create_container_metadata(self.datadir) - else: - return - if self.container: - if not self.metadata: - create_container_metadata(self.datadir) - self.metadata = _read_metadata(self.datadir) - else: - if not validate_container(self.metadata): - create_container_metadata(self.datadir) - self.metadata = _read_metadata(self.datadir) - else: - if not self.metadata: - create_account_metadata(self.datadir) - self.metadata = _read_metadata(self.datadir) - else: - if not validate_account(self.metadata): - create_account_metadata(self.datadir) - self.metadata = _read_metadata(self.datadir) - - def empty(self): - return dir_empty(self.datadir) - - def delete(self): - if self.empty(): - #For delete account. - if os_path.ismount(self.datadir): - clean_metadata(self.datadir) - else: - rmdirs(self.datadir) - self.dir_exists = False - - def put_metadata(self, metadata): - """ - Write metadata to directory/container. - """ - write_metadata(self.datadir, metadata) - self.metadata = metadata - - def put(self, metadata): - """ - Create and write metatdata to directory/container. - :param metadata: Metadata to write. - """ - if not self.dir_exists: - mkdirs(self.datadir) - - os.chown(self.datadir, self.uid, self.gid) - write_metadata(self.datadir, metadata) - self.metadata = metadata - self.dir_exists = True - - def put_obj(self, content_length, timestamp): - ocnt = self.metadata[X_OBJECTS_COUNT][0] - self.metadata[X_OBJECTS_COUNT] = (int(ocnt) + 1, timestamp) - self.metadata[X_PUT_TIMESTAMP] = timestamp - bused = self.metadata[X_BYTES_USED][0] - self.metadata[X_BYTES_USED] = (int(bused) + int(content_length), timestamp) - #TODO: define update_metadata instad of writing whole metadata again. - self.put_metadata(self.metadata) - - def delete_obj(self, content_length): - ocnt, timestamp = self.metadata[X_OBJECTS_COUNT][0] - self.metadata[X_OBJECTS_COUNT] = (int(ocnt) - 1, timestamp) - bused, timestamp = self.metadata[X_BYTES_USED] - self.metadata[X_BYTES_USED] = (int(bused) - int(content_length), timestamp) - self.put_metadata(self.metadata) - - def put_container(self, container, put_timestamp, del_timestamp, object_count, bytes_used): - """ - For account server. - """ - self.metadata[X_OBJECTS_COUNT] = (0, put_timestamp) - self.metadata[X_BYTES_USED] = (0, put_timestamp) - ccnt = self.metadata[X_CONTAINER_COUNT][0] - self.metadata[X_CONTAINER_COUNT] = (int(ccnt) + 1, put_timestamp) - self.metadata[X_PUT_TIMESTAMP] = (1, put_timestamp) - self.put_metadata(self.metadata) - - def delete_container(self, object_count, bytes_used): - """ - For account server. - """ - self.metadata[X_OBJECTS_COUNT] = (0, 0) - self.metadata[X_BYTES_USED] = (0, 0) - ccnt, timestamp = self.metadata[X_CONTAINER_COUNT] - self.metadata[X_CONTAINER_COUNT] = (int(ccnt) - 1, timestamp) - self.put_metadata(self.metadata) - - def unlink(self): - """ - Remove directory/container if empty. - """ - if dir_empty(self.datadir): - rmdirs(self.datadir) - - def list_objects_iter(self, limit, marker, end_marker, - prefix, delimiter, path): - """ - Returns tuple of name, created_at, size, content_type, etag. - """ - if path: - prefix = path = path.rstrip('/') + '/' - delimiter = '/' - if delimiter and not prefix: - prefix = '' - - self.update_object_count() - - objects, object_count, bytes_used = self.object_info - - if objects and prefix: - objects = self.filter_prefix(objects, prefix) - - if objects and delimiter: - objects = self.filter_delimiter(objects, delimiter, prefix) - - if objects and marker: - objects = self.filter_marker(objects, marker) - - if objects and end_marker: - objects = self.filter_end_marker(objects, end_marker) - - if objects and limit: - if len(objects) > limit: - objects = self.filter_limit(objects, limit) - - container_list = [] - if objects: - for obj in objects: - list_item = [] - list_item.append(obj) - obj_path = os.path.join(self.datadir, obj) - metadata = read_metadata(obj_path) - if not metadata or not validate_object(metadata): - metadata = create_object_metadata(obj_path) - if metadata: - list_item.append(metadata[X_TIMESTAMP]) - list_item.append(int(metadata[X_CONTENT_LENGTH])) - list_item.append(metadata[X_CONTENT_TYPE]) - list_item.append(metadata[X_ETAG]) - container_list.append(list_item) - - return container_list - - def update_object_count(self): - if not self.object_info: - self.object_info = get_container_details(self.datadir) - - objects, object_count, bytes_used = self.object_info - - if X_OBJECTS_COUNT not in self.metadata \ - or int(self.metadata[X_OBJECTS_COUNT][0]) != object_count \ - or X_BYTES_USED not in self.metadata \ - or int(self.metadata[X_BYTES_USED][0]) != bytes_used: - self.metadata[X_OBJECTS_COUNT] = (object_count, 0) - self.metadata[X_BYTES_USED] = (bytes_used, 0) - write_metadata(self.datadir, self.metadata) - - def update_container_count(self): - if not self.container_info: - self.container_info = get_account_details(self.datadir) - - containers, container_count = self.container_info - - if X_CONTAINER_COUNT not in self.metadata \ - or int(self.metadata[X_CONTAINER_COUNT][0]) != container_count: - self.metadata[X_CONTAINER_COUNT] = (container_count, 0) - write_metadata(self.datadir, self.metadata) - - def get_info(self, include_metadata=False): - """ - Get global data for the container. - :returns: dict with keys: account, container, object_count, bytes_used, - hash, id, created_at, put_timestamp, delete_timestamp, - reported_put_timestamp, reported_delete_timestamp, - reported_object_count, and reported_bytes_used. - If include_metadata is set, metadata is included as a key - pointing to a dict of tuples of the metadata - """ - # TODO: delete_timestamp, reported_put_timestamp - # reported_delete_timestamp, reported_object_count, - # reported_bytes_used, created_at - if not Glusterfs.OBJECT_ONLY: - # If we are not configured for object only environments, we should - # update the object counts in case they changed behind our back. - self.update_object_count() - - data = {'account' : self.account, 'container' : self.container, - 'object_count' : self.metadata.get(X_OBJECTS_COUNT, ('0', 0))[0], - 'bytes_used' : self.metadata.get(X_BYTES_USED, ('0',0))[0], - 'hash': '', 'id' : '', 'created_at' : '1', - 'put_timestamp' : self.metadata.get(X_PUT_TIMESTAMP, ('0',0))[0], - 'delete_timestamp' : '1', - 'reported_put_timestamp' : '1', 'reported_delete_timestamp' : '1', - 'reported_object_count' : '1', 'reported_bytes_used' : '1'} - if include_metadata: - data['metadata'] = self.metadata - return data - - def put_object(self, name, timestamp, size, content_type, - etag, deleted=0): - # TODO: Implement the specifics of this func. - pass - - def initialize(self, timestamp): - pass - - def update_put_timestamp(self, timestamp): - """ - Create the container if it doesn't exist and update the timestamp - """ - if not os_path.exists(self.datadir): - self.put(self.metadata) - - def delete_object(self, name, timestamp): - # TODO: Implement the delete object - pass - - def delete_db(self, timestamp): - """ - Delete the container - """ - self.unlink() - - def update_metadata(self, metadata): - assert self.metadata, "Valid container/account metadata should have been created by now" - if metadata: - new_metadata = self.metadata.copy() - new_metadata.update(metadata) - if new_metadata != self.metadata: - write_metadata(self.datadir, new_metadata) - self.metadata = new_metadata - - -class DiskAccount(DiskDir): - def __init__(self, root, drive, account, logger): - super(DiskAccount, self).__init__(root, drive, account, None, logger) - assert self.dir_exists - - def list_containers_iter(self, limit, marker, end_marker, - prefix, delimiter): - """ - Return tuple of name, object_count, bytes_used, 0(is_subdir). - Used by account server. - """ - if delimiter and not prefix: - prefix = '' - - self.update_container_count() - - containers, container_count = self.container_info - - if containers: - containers.sort() - - if containers and prefix: - containers = self.filter_prefix(containers, prefix) - - if containers and delimiter: - containers = self.filter_delimiter(containers, delimiter, prefix) - - if containers and marker: - containers = self.filter_marker(containers, marker) - - if containers and end_marker: - containers = self.filter_end_marker(containers, end_marker) - - if containers and limit: - if len(containers) > limit: - containers = self.filter_limit(containers, limit) - - account_list = [] - if containers: - for cont in containers: - list_item = [] - metadata = None - list_item.append(cont) - cont_path = os.path.join(self.datadir, cont) - metadata = _read_metadata(cont_path) - if not metadata or not validate_container(metadata): - metadata = create_container_metadata(cont_path) - - if metadata: - list_item.append(metadata[X_OBJECTS_COUNT][0]) - list_item.append(metadata[X_BYTES_USED][0]) - list_item.append(0) - account_list.append(list_item) - - return account_list - - def get_info(self, include_metadata=False): - """ - Get global data for the account. - :returns: dict with keys: account, created_at, put_timestamp, - delete_timestamp, container_count, object_count, - bytes_used, hash, id - """ - if not Glusterfs.OBJECT_ONLY: - # If we are not configured for object only environments, we should - # update the container counts in case they changed behind our back. - self.update_container_count() - - data = {'account' : self.account, 'created_at' : '1', - 'put_timestamp' : '1', 'delete_timestamp' : '1', - 'container_count' : self.metadata.get(X_CONTAINER_COUNT, (0,0))[0], - 'object_count' : self.metadata.get(X_OBJECTS_COUNT, (0,0))[0], - 'bytes_used' : self.metadata.get(X_BYTES_USED, (0,0))[0], - 'hash' : '', 'id' : ''} - - if include_metadata: - data['metadata'] = self.metadata - return data - - def get_container_timestamp(self, container): - cont_path = os.path.join(self.datadir, container) - metadata = read_metadata(cont_path) - - return int(metadata.get(X_PUT_TIMESTAMP, ('0',0))[0]) or None diff --git a/ufo/gluster/swift/common/DiskFile.py b/ufo/gluster/swift/common/DiskFile.py deleted file mode 100644 index 900bd498fb9..00000000000 --- a/ufo/gluster/swift/common/DiskFile.py +++ /dev/null @@ -1,338 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import errno -import random -from hashlib import md5 -from contextlib import contextmanager -from swift.common.utils import normalize_timestamp, renamer -from swift.common.exceptions import DiskFileNotExist -from gluster.swift.common.exceptions import AlreadyExistsAsDir -from gluster.swift.common.utils import mkdirs, rmdirs, validate_object, \ - create_object_metadata, do_open, do_close, do_unlink, do_chown, \ - do_listdir, read_metadata, write_metadata, os_path, do_fsync -from gluster.swift.common.utils import X_CONTENT_TYPE, X_CONTENT_LENGTH, \ - X_TIMESTAMP, X_PUT_TIMESTAMP, X_TYPE, X_ETAG, X_OBJECTS_COUNT, \ - X_BYTES_USED, X_OBJECT_TYPE, FILE, DIR, MARKER_DIR, OBJECT, DIR_TYPE, \ - FILE_TYPE, DEFAULT_UID, DEFAULT_GID - -import logging -from swift.obj.server import DiskFile - - -DEFAULT_DISK_CHUNK_SIZE = 65536 -# keep these lower-case -DISALLOWED_HEADERS = set('content-length content-type deleted etag'.split()) - - -def _adjust_metadata(metadata): - # Fix up the metadata to ensure it has a proper value for the - # Content-Type metadata, as well as an X_TYPE and X_OBJECT_TYPE - # metadata values. - content_type = metadata['Content-Type'] - if not content_type: - # FIXME: How can this be that our caller supplied us with metadata - # that has a content type that evaluates to False? - # - # FIXME: If the file exists, we would already know it is a - # directory. So why are we assuming it is a file object? - metadata['Content-Type'] = FILE_TYPE - x_object_type = FILE - else: - x_object_type = MARKER_DIR if content_type.lower() == DIR_TYPE else FILE - metadata[X_TYPE] = OBJECT - metadata[X_OBJECT_TYPE] = x_object_type - return metadata - - -class Gluster_DiskFile(DiskFile): - """ - Manage object files on disk. - - :param path: path to devices on the node/mount path for UFO. - :param device: device name/account_name for UFO. - :param partition: partition on the device the object lives in - :param account: account name for the object - :param container: container name for the object - :param obj: object name for the object - :param logger: logger object for writing out log file messages - :param keep_data_fp: if True, don't close the fp, otherwise close it - :param disk_chunk_Size: size of chunks on file reads - :param uid: user ID disk object should assume (file or directory) - :param gid: group ID disk object should assume (file or directory) - """ - - def __init__(self, path, device, partition, account, container, obj, - logger, keep_data_fp=False, - disk_chunk_size=DEFAULT_DISK_CHUNK_SIZE, - uid=DEFAULT_UID, gid=DEFAULT_GID, iter_hook=None): - self.disk_chunk_size = disk_chunk_size - self.iter_hook = iter_hook - # Don't support obj_name ending/begining with '/', like /a, a/, /a/b/, - # etc. - obj = obj.strip(os.path.sep) - if os.path.sep in obj: - self._obj_path, self._obj = os.path.split(obj) - else: - self._obj_path = '' - self._obj = obj - - if self._obj_path: - self.name = os.path.join(container, self._obj_path) - else: - self.name = container - # Absolute path for object directory. - self.datadir = os.path.join(path, device, self.name) - self.device_path = os.path.join(path, device) - self._container_path = os.path.join(path, device, container) - self._is_dir = False - self.tmppath = None - self.logger = logger - self.metadata = {} - self.meta_file = None - self.fp = None - self.iter_etag = None - self.started_at_0 = False - self.read_to_eof = False - self.quarantined_dir = None - self.keep_cache = False - self.uid = int(uid) - self.gid = int(gid) - - # Don't store a value for data_file until we know it exists. - self.data_file = None - data_file = os.path.join(self.datadir, self._obj) - if not os_path.exists(data_file): - return - - self.data_file = os.path.join(data_file) - self.metadata = read_metadata(data_file) - if not self.metadata: - create_object_metadata(data_file) - self.metadata = read_metadata(data_file) - - if not validate_object(self.metadata): - create_object_metadata(data_file) - self.metadata = read_metadata(data_file) - - self.filter_metadata() - - if os_path.isdir(data_file): - self._is_dir = True - else: - if keep_data_fp: - # The caller has an assumption that the "fp" field of this - # object is an file object if keep_data_fp is set. However, - # this implementation of the DiskFile object does not need to - # open the file for internal operations. So if the caller - # requests it, we'll just open the file for them. - self.fp = do_open(data_file, 'rb') - - def close(self, verify_file=True): - """ - Close the file. Will handle quarantining file if necessary. - - :param verify_file: Defaults to True. If false, will not check - file to see if it needs quarantining. - """ - #Marker directory - if self._is_dir: - return - if self.fp: - do_close(self.fp) - self.fp = None - - def is_deleted(self): - """ - Check if the file is deleted. - - :returns: True if the file doesn't exist or has been flagged as - deleted. - """ - return not self.data_file - - def _create_dir_object(self, dir_path): - #TODO: if object already exists??? - if os_path.exists(dir_path) and not os_path.isdir(dir_path): - self.logger.error("Deleting file %s", dir_path) - do_unlink(dir_path) - #If dir aleady exist just override metadata. - mkdirs(dir_path) - do_chown(dir_path, self.uid, self.gid) - create_object_metadata(dir_path) - - def put_metadata(self, metadata, tombstone=False): - """ - Short hand for putting metadata to .meta and .ts files. - - :param metadata: dictionary of metadata to be written - :param tombstone: whether or not we are writing a tombstone - """ - if tombstone: - # We don't write tombstone files. So do nothing. - return - assert self.data_file is not None, "put_metadata: no file to put metadata into" - metadata = _adjust_metadata(metadata) - write_metadata(self.data_file, metadata) - self.metadata = metadata - self.filter_metadata() - - def put(self, fd, metadata, extension='.data'): - """ - Finalize writing the file on disk, and renames it from the temp file to - the real location. This should be called after the data has been - written to the temp file. - - :param fd: file descriptor of the temp file - :param metadata: dictionary of metadata to be written - :param extension: extension to be used when making the file - """ - # Our caller will use '.data' here; we just ignore it since we map the - # URL directly to the file system. - extension = '' - - metadata = _adjust_metadata(metadata) - - if metadata[X_OBJECT_TYPE] == MARKER_DIR: - if not self.data_file: - self.data_file = os.path.join(self.datadir, self._obj) - self._create_dir_object(self.data_file) - self.put_metadata(metadata) - return - - # Check if directory already exists. - if self._is_dir: - # FIXME: How can we have a directory and it not be marked as a - # MARKER_DIR (see above)? - msg = 'File object exists as a directory: %s' % self.data_file - raise AlreadyExistsAsDir(msg) - - timestamp = normalize_timestamp(metadata[X_TIMESTAMP]) - write_metadata(self.tmppath, metadata) - if X_CONTENT_LENGTH in metadata: - self.drop_cache(fd, 0, int(metadata[X_CONTENT_LENGTH])) - do_fsync(fd) - if self._obj_path: - dir_objs = self._obj_path.split('/') - assert len(dir_objs) >= 1 - tmp_path = self._container_path - for dir_name in dir_objs: - tmp_path = os.path.join(tmp_path, dir_name) - self._create_dir_object(tmp_path) - - newpath = os.path.join(self.datadir, self._obj) - renamer(self.tmppath, newpath) - do_chown(newpath, self.uid, self.gid) - self.metadata = metadata - self.data_file = newpath - self.filter_metadata() - return - - def unlinkold(self, timestamp): - """ - Remove any older versions of the object file. Any file that has an - older timestamp than timestamp will be deleted. - - :param timestamp: timestamp to compare with each file - """ - if not self.metadata or self.metadata['X-Timestamp'] >= timestamp: - return - - assert self.data_file, \ - "Have metadata, %r, but no data_file" % self.metadata - - if self._is_dir: - # Marker directory object - if not rmdirs(self.data_file): - logging.error('Unable to delete dir object: %s', self.data_file) - return - else: - # File object - do_unlink(self.data_file) - - self.metadata = {} - self.data_file = None - - def get_data_file_size(self): - """ - Returns the os_path.getsize for the file. Raises an exception if this - file does not match the Content-Length stored in the metadata. Or if - self.data_file does not exist. - - :returns: file size as an int - :raises DiskFileError: on file size mismatch. - :raises DiskFileNotExist: on file not existing (including deleted) - """ - #Marker directory. - if self._is_dir: - return 0 - try: - file_size = 0 - if self.data_file: - file_size = os_path.getsize(self.data_file) - if X_CONTENT_LENGTH in self.metadata: - metadata_size = int(self.metadata[X_CONTENT_LENGTH]) - if file_size != metadata_size: - self.metadata[X_CONTENT_LENGTH] = file_size - write_metadata(self.data_file, self.metadata) - - return file_size - except OSError as err: - if err.errno != errno.ENOENT: - raise - raise DiskFileNotExist('Data File does not exist.') - - def filter_metadata(self): - if X_TYPE in self.metadata: - self.metadata.pop(X_TYPE) - if X_OBJECT_TYPE in self.metadata: - self.metadata.pop(X_OBJECT_TYPE) - - @contextmanager - def mkstemp(self): - """Contextmanager to make a temporary file.""" - - # Creating intermidiate directories and corresponding metadata. - # For optimization, check if the subdirectory already exists, - # if exists, then it means that it also has its metadata. - # Not checking for container, since the container should already - # exist for the call to come here. - if not os_path.exists(self.datadir): - path = self._container_path - subdir_list = self._obj_path.split(os.path.sep) - for i in range(len(subdir_list)): - path = os.path.join(path, subdir_list[i]); - if not os_path.exists(path): - self._create_dir_object(path) - - tmpfile = '.' + self._obj + '.' + md5(self._obj + \ - str(random.random())).hexdigest() - - self.tmppath = os.path.join(self.datadir, tmpfile) - fd = do_open(self.tmppath, os.O_RDWR | os.O_CREAT | os.O_EXCL) - try: - yield fd - finally: - try: - do_close(fd) - except OSError: - pass - tmppath, self.tmppath = self.tmppath, None - try: - do_unlink(tmppath) - except OSError as err: - if err.errno != errno.ENOENT: - raise diff --git a/ufo/gluster/swift/common/Glusterfs.py b/ufo/gluster/swift/common/Glusterfs.py deleted file mode 100644 index a0c81269cdc..00000000000 --- a/ufo/gluster/swift/common/Glusterfs.py +++ /dev/null @@ -1,144 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import os, sys, fcntl, time, errno -from ConfigParser import ConfigParser, NoSectionError, NoOptionError -from swift.common.utils import TRUE_VALUES, search_tree -from gluster.swift.common.fs_utils import mkdirs - -# -# Read the fs.conf file once at startup (module load) -# -_fs_conf = ConfigParser() -MOUNT_IP = 'localhost' -OBJECT_ONLY = False -RUN_DIR='/var/run/swift' -SWIFT_DIR = '/etc/swift' -_do_getsize = False -if _fs_conf.read(os.path.join('/etc/swift', 'fs.conf')): - try: - MOUNT_IP = _fs_conf.get('DEFAULT', 'mount_ip', 'localhost') - except (NoSectionError, NoOptionError): - pass - try: - OBJECT_ONLY = _fs_conf.get('DEFAULT', 'object_only', "no") in TRUE_VALUES - except (NoSectionError, NoOptionError): - pass - try: - RUN_DIR = _fs_conf.get('DEFAULT', 'run_dir', '/var/run/swift') - except (NoSectionError, NoOptionError): - pass - - try: - _do_getsize = _fs_conf.get('DEFAULT', 'accurate_size_in_listing', \ - "no") in TRUE_VALUES - except (NoSectionError, NoOptionError): - pass - -NAME = 'glusterfs' - - -def _busy_wait(full_mount_path): - # Iterate for definite number of time over a given - # interval for successful mount - for i in range(0, 5): - if os.path.ismount(os.path.join(full_mount_path)): - return True - time.sleep(2) - logging.error('Busy wait for mount timed out for mount %s', full_mount_path) - return False - -def mount(root, drive): - # FIXME: Possible thundering herd problem here - - el = _get_export_list() - for export in el: - if drive == export: - break - else: - logging.error('No export found in %r matching drive, %s', el, drive) - return False - - # NOTE: root is typically the default value of /mnt/gluster-object - full_mount_path = os.path.join(root, drive) - if not os.path.isdir(full_mount_path): - mkdirs(full_mount_path) - - lck_file = os.path.join(RUN_DIR, '%s.lock' %drive); - - if not os.path.exists(RUN_DIR): - mkdirs(RUN_DIR) - - fd = os.open(lck_file, os.O_CREAT|os.O_RDWR) - with os.fdopen(fd, 'r+b') as f: - try: - fcntl.lockf(f, fcntl.LOCK_EX|fcntl.LOCK_NB) - except IOError as ex: - if ex.errno in (errno.EACCES, errno.EAGAIN): - # This means that some other process is mounting the - # filesystem, so wait for the mount process to complete - return _busy_wait(full_mount_path) - else: - raise ex - mnt_cmd = 'mount -t glusterfs %s:%s %s' % (MOUNT_IP, export, \ - full_mount_path) - if os.system(mnt_cmd) or not _busy_wait(full_mount_path): - logging.error('Mount failed %s: %s', NAME, mnt_cmd) - return False - return True - -def unmount(full_mount_path): - # FIXME: Possible thundering herd problem here - - umnt_cmd = 'umount %s 2>> /dev/null' % full_mount_path - if os.system(umnt_cmd): - logging.error('Unable to unmount %s %s' % (full_mount_path, NAME)) - -def _get_export_list(): - cmnd = 'gluster --remote-host=%s volume info' % MOUNT_IP - - export_list = [] - - if os.system(cmnd + ' >> /dev/null'): - logging.error('Getting volume info failed for %s', NAME) - else: - fp = os.popen(cmnd) - while True: - item = fp.readline() - if not item: - break - item = item.strip('\n').strip(' ') - if item.lower().startswith('volume name:'): - export_list.append(item.split(':')[1].strip(' ')) - - return export_list - -def get_mnt_point(vol_name, conf_dir=SWIFT_DIR, conf_file="object-server*"): - """Read the object-server's configuration file and return - the device value""" - - mnt_dir = '' - conf_files = search_tree(conf_dir, conf_file, '.conf') - if not conf_files: - raise Exception("Config file not found") - - _conf = ConfigParser() - if _conf.read(conf_files[0]): - try: - mnt_dir = _conf.get('DEFAULT', 'devices', '') - except (NoSectionError, NoOptionError): - raise - return os.path.join(mnt_dir, vol_name) diff --git a/ufo/gluster/swift/common/__init__.py b/ufo/gluster/swift/common/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/ufo/gluster/swift/common/__init__.py +++ /dev/null diff --git a/ufo/gluster/swift/common/constraints.py b/ufo/gluster/swift/common/constraints.py deleted file mode 100644 index 11f626bb98e..00000000000 --- a/ufo/gluster/swift/common/constraints.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -try: - from webob.exc import HTTPBadRequest -except ImportError: - from swift.common.swob import HTTPBadRequest -import swift.common.constraints -import swift.common.ring as _ring -from gluster.swift.common import Glusterfs, ring - -if hasattr(swift.common.constraints, 'constraints_conf_int'): - MAX_OBJECT_NAME_COMPONENT_LENGTH = \ - swift.common.constraints.constraints_conf_int( - 'max_object_name_component_length', 255) -else: - MAX_OBJECT_NAME_COMPONENT_LENGTH = 255 - -def validate_obj_name_component(obj): - if len(obj) > MAX_OBJECT_NAME_COMPONENT_LENGTH: - return 'too long (%d)' % len(obj) - if obj == '.' or obj == '..': - return 'cannot be . or ..' - return '' - -# Save the original check object creation -__check_object_creation = swift.common.constraints.check_object_creation - -# Define our new one which invokes the original -def gluster_check_object_creation(req, object_name): - """ - Check to ensure that everything is alright about an object to be created. - Monkey patches swift.common.constraints.check_object_creation, invoking - the original, and then adding an additional check for individual object - name components. - - :param req: HTTP request object - :param object_name: name of object to be created - :raises HTTPRequestEntityTooLarge: the object is too large - :raises HTTPLengthRequered: missing content-length header and not - a chunked request - :raises HTTPBadRequest: missing or bad content-type header, or - bad metadata - """ - ret = __check_object_creation(req, object_name) - - if ret is None: - for obj in object_name.split('/'): - reason = validate_obj_name_component(obj) - if reason: - bdy = 'Invalid object name "%s", component "%s" %s' \ - % (object_name, obj, reason) - ret = HTTPBadRequest(body=bdy, - request=req, - content_type='text/plain') - - return ret - -# Replace the original check object creation with ours -swift.common.constraints.check_object_creation = gluster_check_object_creation - -# Save the original check mount -__check_mount = swift.common.constraints.check_mount - -# Define our new one which invokes the original -def gluster_check_mount(root, drive): - # FIXME: Potential performance optimization here to not call the original - # check mount which makes two stat calls. We could do what they do with - # just one. - if __check_mount(root, drive): - return True - - return Glusterfs.mount(root, drive) - -# Replace the original check mount with ours -swift.common.constraints.check_mount = gluster_check_mount - -# Save the original Ring class -__Ring = _ring.Ring - -# Replace the original Ring class -_ring.Ring = ring.Ring diff --git a/ufo/gluster/swift/common/exceptions.py b/ufo/gluster/swift/common/exceptions.py deleted file mode 100644 index d9357dbb42f..00000000000 --- a/ufo/gluster/swift/common/exceptions.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -class GlusterfsException(Exception): - pass - -class FileOrDirNotFoundError(GlusterfsException): - pass - -class NotDirectoryError(GlusterfsException): - pass - -class AlreadyExistsAsDir(GlusterfsException): - pass - diff --git a/ufo/gluster/swift/common/fs_utils.py b/ufo/gluster/swift/common/fs_utils.py deleted file mode 100644 index 0613a26dce8..00000000000 --- a/ufo/gluster/swift/common/fs_utils.py +++ /dev/null @@ -1,179 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import os -import errno -import os.path as os_path -from eventlet import tpool -from gluster.swift.common.exceptions import FileOrDirNotFoundError, \ - NotDirectoryError - -def do_walk(*args, **kwargs): - return os.walk(*args, **kwargs) - -def do_write(fd, msg): - try: - cnt = os.write(fd, msg) - except OSError as err: - logging.exception("Write failed, err: %s", str(err)) - raise - return cnt - -def do_mkdir(path): - try: - os.mkdir(path) - except OSError as err: - if err.errno != errno.EEXIST: - logging.exception("Mkdir failed on %s err: %s", path, err.strerror) - raise - return True - -def do_makedirs(path): - try: - os.makedirs(path) - except OSError as err: - if err.errno != errno.EEXIST: - logging.exception("Makedirs failed on %s err: %s", path, err.strerror) - raise - return True - -def do_listdir(path): - try: - buf = os.listdir(path) - except OSError as err: - logging.exception("Listdir failed on %s err: %s", path, err.strerror) - raise - return buf - -def do_chown(path, uid, gid): - try: - os.chown(path, uid, gid) - except OSError as err: - logging.exception("Chown failed on %s err: %s", path, err.strerror) - raise - return True - -def do_stat(path): - try: - #Check for fd. - if isinstance(path, int): - buf = os.fstat(path) - else: - buf = os.stat(path) - except OSError as err: - logging.exception("Stat failed on %s err: %s", path, err.strerror) - raise - return buf - -def do_open(path, mode): - if isinstance(mode, int): - try: - fd = os.open(path, mode) - except OSError as err: - logging.exception("Open failed on %s err: %s", path, str(err)) - raise - else: - try: - fd = open(path, mode) - except IOError as err: - logging.exception("Open failed on %s err: %s", path, str(err)) - raise - return fd - -def do_close(fd): - #fd could be file or int type. - try: - if isinstance(fd, int): - os.close(fd) - else: - fd.close() - except OSError as err: - logging.exception("Close failed on %s err: %s", fd, err.strerror) - raise - return True - -def do_unlink(path, log = True): - try: - os.unlink(path) - except OSError as err: - if err.errno != errno.ENOENT: - if log: - logging.exception("Unlink failed on %s err: %s", path, err.strerror) - raise - return True - -def do_rmdir(path): - try: - os.rmdir(path) - except OSError as err: - if err.errno != errno.ENOENT: - logging.exception("Rmdir failed on %s err: %s", path, err.strerror) - raise - res = False - else: - res = True - return res - -def do_rename(old_path, new_path): - try: - os.rename(old_path, new_path) - except OSError as err: - logging.exception("Rename failed on %s to %s err: %s", old_path, new_path, \ - err.strerror) - raise - return True - -def mkdirs(path): - """ - Ensures the path is a directory or makes it if not. Errors if the path - exists but is a file or on permissions failure. - - :param path: path to create - """ - if not os.path.isdir(path): - do_makedirs(path) - -def dir_empty(path): - """ - Return true if directory/container is empty. - :param path: Directory path. - :returns: True/False. - """ - if os.path.isdir(path): - files = do_listdir(path) - return not files - elif not os.path.exists(path): - raise FileOrDirNotFoundError() - raise NotDirectoryError() - -def rmdirs(path): - if not os.path.isdir(path): - return False - try: - os.rmdir(path) - except OSError as err: - if err.errno != errno.ENOENT: - logging.error("rmdirs failed on %s, err: %s", path, err.strerror) - return False - return True - -def do_fsync(fd): - try: - tpool.execute(os.fsync, fd) - except OSError as err: - logging.exception("fsync failed with err: %s", err.strerror) - raise - return True diff --git a/ufo/gluster/swift/common/middleware/__init__.py b/ufo/gluster/swift/common/middleware/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/ufo/gluster/swift/common/middleware/__init__.py +++ /dev/null diff --git a/ufo/gluster/swift/common/middleware/gluster.py b/ufo/gluster/swift/common/middleware/gluster.py deleted file mode 100644 index ab63c51e12f..00000000000 --- a/ufo/gluster/swift/common/middleware/gluster.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Noop Middleware that simply allows us to monkey patch the constraints -import gluster.swift.common.constraints - -class Gluster(object): - """ - Noop middleware for use with the proxy server to get paste.deploy to load - this middleware such that the plugin constraints monkey patch the common - constraints ahead of their use. - """ - def __init__(self, app, conf): - self.app = app - self.conf = conf - - def __call__(self, env, start_response): - return self.app(env, start_response) - - -def filter_factory(global_conf, **local_conf): - """Returns a WSGI filter app for use with paste.deploy.""" - conf = global_conf.copy() - conf.update(local_conf) - - def gluster_filter(app): - return Gluster(app, conf) - return gluster_filter diff --git a/ufo/gluster/swift/common/ring.py b/ufo/gluster/swift/common/ring.py deleted file mode 100644 index 06aab8d3763..00000000000 --- a/ufo/gluster/swift/common/ring.py +++ /dev/null @@ -1,111 +0,0 @@ -# Copyright (c) 2013 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from ConfigParser import ConfigParser -from swift.common.ring import ring -from swift.common.utils import search_tree -from gluster.swift.common.Glusterfs import SWIFT_DIR - -reseller_prefix = "AUTH_" -conf_files = search_tree(SWIFT_DIR, "proxy-server*", 'conf') -if conf_files: - conf_file = conf_files[0] - -_conf = ConfigParser() -if conf_files and _conf.read(conf_file): - if _conf.defaults().get("reseller_prefix", None): - reseller_prefix = _conf.defaults().get("reseller_prefix") - else: - for key, value in _conf._sections.items(): - if value.get("reseller_prefix", None): - reseller_prefix = value["reseller_prefix"] - break - -if not reseller_prefix.endswith('_'): - reseller_prefix = reseller_prefix + '_' - -class Ring(ring.Ring): - def _get_part_nodes(self, part): - seen_ids = set() - nodes = [dev for dev in self._devs \ - if dev['device'] == self.acc_name \ - and not (dev['id'] in seen_ids \ - or seen_ids.add(dev['id']))] - if not nodes: - nodes = [self.false_node] - return nodes - - def get_part_nodes(self, part): - """ - Get the nodes that are responsible for the partition. If one - node is responsible for more than one replica of the same - partition, it will only appear in the output once. - - :param part: partition to get nodes for - :returns: list of node dicts - - See :func:`get_nodes` for a description of the node dicts. - """ - return self._get_part_nodes(part) - - def get_nodes(self, account, container=None, obj=None): - """ - Get the partition and nodes for an account/container/object. - If a node is responsible for more than one replica, it will - only appear in the output once. - :param account: account name - :param container: container name - :param obj: object name - :returns: a tuple of (partition, list of node dicts) - - Each node dict will have at least the following keys: - ====== =============================================================== - id unique integer identifier amongst devices - weight a float of the relative weight of this device as compared to - others; this indicates how many partitions the builder will try - to assign to this device - zone integer indicating which zone the device is in; a given - partition will not be assigned to multiple devices within the - same zone - ip the ip address of the device - port the tcp port of the device - device the device's name on disk (sdb1, for example) - meta general use 'extra' field; for example: the online date, the - hardware description - ====== =============================================================== - """ - self.false_node = {'zone': 1, 'weight': 100.0, 'ip': '127.0.0.1', 'id': 0, \ - 'meta': '', 'device': 'volume_not_in_ring', \ - 'port': 6012} - if account.startswith(reseller_prefix): - self.acc_name = account.replace(reseller_prefix, '', 1) - else: - self.acc_name = account - - part = 0 - return part, self._get_part_nodes(part) - - - def get_more_nodes(self, part): - """ - Generator to get extra nodes for a partition for hinted handoff. - - :param part: partition to get handoff nodes for - :returns: generator of node dicts - - See :func:`get_nodes` for a description of the node dicts. - Should never be called in the swift UFO environment, so yield nothing - """ - yield self.false_node diff --git a/ufo/gluster/swift/common/utils.py b/ufo/gluster/swift/common/utils.py deleted file mode 100644 index f2cd8dea10c..00000000000 --- a/ufo/gluster/swift/common/utils.py +++ /dev/null @@ -1,496 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import os -import errno -import xattr -import random -from hashlib import md5 -from eventlet import sleep -import cPickle as pickle -from ConfigParser import ConfigParser, NoSectionError, NoOptionError -from swift.common.utils import normalize_timestamp, TRUE_VALUES -from gluster.swift.common.fs_utils import * -from gluster.swift.common import Glusterfs - -X_CONTENT_TYPE = 'Content-Type' -X_CONTENT_LENGTH = 'Content-Length' -X_TIMESTAMP = 'X-Timestamp' -X_PUT_TIMESTAMP = 'X-PUT-Timestamp' -X_TYPE = 'X-Type' -X_ETAG = 'ETag' -X_OBJECTS_COUNT = 'X-Object-Count' -X_BYTES_USED = 'X-Bytes-Used' -X_CONTAINER_COUNT = 'X-Container-Count' -X_OBJECT_TYPE = 'X-Object-Type' -DIR_TYPE = 'application/directory' -ACCOUNT = 'Account' -METADATA_KEY = 'user.swift.metadata' -MAX_XATTR_SIZE = 65536 -CONTAINER = 'container' -DIR = 'dir' -MARKER_DIR = 'marker_dir' -TEMP_DIR = 'tmp' -ASYNCDIR = 'async_pending' # Keep in sync with swift.obj.server.ASYNCDIR -FILE = 'file' -FILE_TYPE = 'application/octet-stream' -OBJECT = 'Object' -OBJECT_TYPE = 'application/octet-stream' -DEFAULT_UID = -1 -DEFAULT_GID = -1 -PICKLE_PROTOCOL = 2 -CHUNK_SIZE = 65536 -MEMCACHE_KEY_PREFIX = 'gluster.swift.' -MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX = MEMCACHE_KEY_PREFIX + 'account.details.' -MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX = MEMCACHE_KEY_PREFIX + 'container.details.' - -def read_metadata(path): - """ - Helper function to read the pickled metadata from a File/Directory. - - :param path: File/Directory to read metadata from. - - :returns: dictionary of metadata - """ - metadata = None - metadata_s = '' - key = 0 - while metadata is None: - try: - metadata_s += xattr.getxattr(path, '%s%s' % (METADATA_KEY, (key or ''))) - except IOError as err: - if err.errno == errno.ENODATA: - if key > 0: - # No errors reading the xattr keys, but since we have not - # been able to find enough chunks to get a successful - # unpickle operation, we consider the metadata lost, and - # drop the existing data so that the internal state can be - # recreated. - clean_metadata(path) - # We either could not find any metadata key, or we could find - # some keys, but were not successful in performing the - # unpickling (missing keys perhaps)? Either way, just report - # to the caller we have no metadata. - metadata = {} - else: - logging.exception("xattr.getxattr failed on %s key %s err: %s", - path, key, str(err)) - # Note that we don't touch the keys on errors fetching the - # data since it could be a transient state. - raise - else: - try: - # If this key provides all or the remaining part of the pickle - # data, we don't need to keep searching for more keys. This - # means if we only need to store data in N xattr key/value - # pair, we only need to invoke xattr get N times. With large - # keys sizes we are shooting for N = 1. - metadata = pickle.loads(metadata_s) - assert isinstance(metadata, dict) - except EOFError, pickle.UnpicklingError: - # We still are not able recognize this existing data collected - # as a pickled object. Make sure we loop around to try to get - # more from another xattr key. - metadata = None - key += 1 - return metadata - -def write_metadata(path, metadata): - """ - Helper function to write pickled metadata for a File/Directory. - - :param path: File/Directory path to write the metadata - :param metadata: dictionary to metadata write - """ - assert isinstance(metadata, dict) - metastr = pickle.dumps(metadata, PICKLE_PROTOCOL) - key = 0 - while metastr: - try: - xattr.setxattr(path, '%s%s' % (METADATA_KEY, key or ''), metastr[:MAX_XATTR_SIZE]) - except IOError as err: - logging.exception("setxattr failed on %s key %s err: %s", path, key, str(err)) - raise - metastr = metastr[MAX_XATTR_SIZE:] - key += 1 - -def clean_metadata(path): - key = 0 - while True: - try: - xattr.removexattr(path, '%s%s' % (METADATA_KEY, (key or ''))) - except IOError as err: - if err.errno == errno.ENODATA: - break - raise - key += 1 - -def check_user_xattr(path): - if not os_path.exists(path): - return False - try: - xattr.setxattr(path, 'user.test.key1', 'value1') - except IOError as err: - logging.exception("check_user_xattr: set failed on %s err: %s", path, str(err)) - raise - try: - xattr.removexattr(path, 'user.test.key1') - except IOError as err: - logging.exception("check_user_xattr: remove failed on %s err: %s", path, str(err)) - #Remove xattr may fail in case of concurrent remove. - return True - -def validate_container(metadata): - if not metadata: - logging.warn('validate_container: No metadata') - return False - - if X_TYPE not in metadata.keys() or \ - X_TIMESTAMP not in metadata.keys() or \ - X_PUT_TIMESTAMP not in metadata.keys() or \ - X_OBJECTS_COUNT not in metadata.keys() or \ - X_BYTES_USED not in metadata.keys(): - #logging.warn('validate_container: Metadata missing entries: %s' % metadata) - return False - - (value, timestamp) = metadata[X_TYPE] - if value == CONTAINER: - return True - - logging.warn('validate_container: metadata type is not CONTAINER (%r)' % (value,)) - return False - -def validate_account(metadata): - if not metadata: - logging.warn('validate_account: No metadata') - return False - - if X_TYPE not in metadata.keys() or \ - X_TIMESTAMP not in metadata.keys() or \ - X_PUT_TIMESTAMP not in metadata.keys() or \ - X_OBJECTS_COUNT not in metadata.keys() or \ - X_BYTES_USED not in metadata.keys() or \ - X_CONTAINER_COUNT not in metadata.keys(): - #logging.warn('validate_account: Metadata missing entries: %s' % metadata) - return False - - (value, timestamp) = metadata[X_TYPE] - if value == ACCOUNT: - return True - - logging.warn('validate_account: metadata type is not ACCOUNT (%r)' % (value,)) - return False - -def validate_object(metadata): - if not metadata: - logging.warn('validate_object: No metadata') - return False - - if X_TIMESTAMP not in metadata.keys() or \ - X_CONTENT_TYPE not in metadata.keys() or \ - X_ETAG not in metadata.keys() or \ - X_CONTENT_LENGTH not in metadata.keys() or \ - X_TYPE not in metadata.keys() or \ - X_OBJECT_TYPE not in metadata.keys(): - #logging.warn('validate_object: Metadata missing entries: %s' % metadata) - return False - - if metadata[X_TYPE] == OBJECT: - return True - - logging.warn('validate_object: metadata type is not OBJECT (%r)' % (metadata[X_TYPE],)) - return False - -def is_marker(metadata): - if not metadata: - logging.warn('is_marker: No metadata') - return False - - if X_OBJECT_TYPE not in metadata.keys(): - logging.warn('is_marker: X_OBJECT_TYPE missing from metadata: %s' % metadata) - return False - - if metadata[X_OBJECT_TYPE] == MARKER_DIR: - return True - else: - return False - -def _update_list(path, cont_path, src_list, reg_file=True, object_count=0, - bytes_used=0, obj_list=[]): - # strip the prefix off, also stripping the leading and trailing slashes - obj_path = path.replace(cont_path, '').strip(os.path.sep) - - for obj_name in src_list: - if obj_path: - obj_list.append(os.path.join(obj_path, obj_name)) - else: - obj_list.append(obj_name) - - object_count += 1 - - if Glusterfs._do_getsize and reg_file: - bytes_used += os_path.getsize(os.path.join(path, obj_name)) - sleep() - - return object_count, bytes_used - -def update_list(path, cont_path, dirs=[], files=[], object_count=0, - bytes_used=0, obj_list=[]): - if files: - object_count, bytes_used = _update_list(path, cont_path, files, True, - object_count, bytes_used, - obj_list) - if dirs: - object_count, bytes_used = _update_list(path, cont_path, dirs, False, - object_count, bytes_used, - obj_list) - return object_count, bytes_used - - -class ContainerDetails(object): - def __init__(self, bytes_used, object_count, obj_list, dir_list): - self.bytes_used = bytes_used - self.object_count = object_count - self.obj_list = obj_list - self.dir_list = dir_list - - -def _get_container_details_from_fs(cont_path): - """ - get container details by traversing the filesystem - """ - bytes_used = 0 - object_count = 0 - obj_list = [] - dir_list = [] - - if os_path.isdir(cont_path): - for (path, dirs, files) in do_walk(cont_path): - object_count, bytes_used = update_list(path, cont_path, dirs, files, - object_count, bytes_used, - obj_list) - - dir_list.append((path, do_stat(path).st_mtime)) - sleep() - - return ContainerDetails(bytes_used, object_count, obj_list, dir_list) - -def get_container_details(cont_path, memcache=None): - """ - Return object_list, object_count and bytes_used. - """ - mkey = '' - if memcache: - mkey = MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + cont_path - cd = memcache.get(mkey) - if cd: - if not cd.dir_list: - cd = None - else: - for (path, mtime) in cd.dir_list: - if mtime != do_stat(path).st_mtime: - cd = None - else: - cd = None - if not cd: - cd = _get_container_details_from_fs(cont_path) - if memcache: - memcache.set(mkey, cd) - return cd.obj_list, cd.object_count, cd.bytes_used - - -class AccountDetails(object): - """ A simple class to store the three pieces of information associated - with an account: - - 1. The last known modification time - 2. The count of containers in the following list - 3. The list of containers - """ - def __init__(self, mtime, container_count, container_list): - self.mtime = mtime - self.container_count = container_count - self.container_list = container_list - - -def _get_account_details_from_fs(acc_path, acc_stats): - container_list = [] - container_count = 0 - - if not acc_stats: - acc_stats = do_stat(acc_path) - is_dir = (acc_stats.st_mode & 0040000) != 0 - if is_dir: - for name in do_listdir(acc_path): - if name.lower() == TEMP_DIR \ - or name.lower() == ASYNCDIR \ - or not os_path.isdir(os.path.join(acc_path, name)): - continue - container_count += 1 - container_list.append(name) - - return AccountDetails(acc_stats.st_mtime, container_count, container_list) - -def get_account_details(acc_path, memcache=None): - """ - Return container_list and container_count. - """ - acc_stats = None - mkey = '' - if memcache: - mkey = MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX + acc_path - ad = memcache.get(mkey) - if ad: - # FIXME: Do we really need to stat the file? If we are object - # only, then we can track the other Swift HTTP APIs that would - # modify the account and invalidate the cached entry there. If we - # are not object only, are we even called on this path? - acc_stats = do_stat(acc_path) - if ad.mtime != acc_stats.st_mtime: - ad = None - else: - ad = None - if not ad: - ad = _get_account_details_from_fs(acc_path, acc_stats) - if memcache: - memcache.set(mkey, ad) - return ad.container_list, ad.container_count - -def _get_etag(path): - etag = md5() - with open(path, 'rb') as fp: - while True: - chunk = fp.read(CHUNK_SIZE) - if chunk: - etag.update(chunk) - else: - break - return etag.hexdigest() - -def get_object_metadata(obj_path): - """ - Return metadata of object. - """ - try: - stats = do_stat(obj_path) - except OSError as e: - if e.errno != errno.ENOENT: - raise - metadata = {} - else: - is_dir = (stats.st_mode & 0040000) != 0 - metadata = { - X_TYPE: OBJECT, - X_TIMESTAMP: normalize_timestamp(stats.st_ctime), - X_CONTENT_TYPE: DIR_TYPE if is_dir else FILE_TYPE, - X_OBJECT_TYPE: DIR if is_dir else FILE, - X_CONTENT_LENGTH: 0 if is_dir else stats.st_size, - X_ETAG: md5().hexdigest() if is_dir else _get_etag(obj_path), - } - return metadata - -def _add_timestamp(metadata_i): - # At this point we have a simple key/value dictionary, turn it into - # key/(value,timestamp) pairs. - timestamp = 0 - metadata = {} - for key, value_i in metadata_i.iteritems(): - if not isinstance(value_i, tuple): - metadata[key] = (value_i, timestamp) - else: - metadata[key] = value_i - return metadata - -def get_container_metadata(cont_path, memcache=None): - objects = [] - object_count = 0 - bytes_used = 0 - objects, object_count, bytes_used = get_container_details(cont_path, memcache) - metadata = {X_TYPE: CONTAINER, - X_TIMESTAMP: normalize_timestamp(os_path.getctime(cont_path)), - X_PUT_TIMESTAMP: normalize_timestamp(os_path.getmtime(cont_path)), - X_OBJECTS_COUNT: object_count, - X_BYTES_USED: bytes_used} - return _add_timestamp(metadata) - -def get_account_metadata(acc_path, memcache=None): - containers = [] - container_count = 0 - containers, container_count = get_account_details(acc_path, memcache) - metadata = {X_TYPE: ACCOUNT, - X_TIMESTAMP: normalize_timestamp(os_path.getctime(acc_path)), - X_PUT_TIMESTAMP: normalize_timestamp(os_path.getmtime(acc_path)), - X_OBJECTS_COUNT: 0, - X_BYTES_USED: 0, - X_CONTAINER_COUNT: container_count} - return _add_timestamp(metadata) - -def restore_metadata(path, metadata): - meta_orig = read_metadata(path) - if meta_orig: - meta_new = meta_orig.copy() - meta_new.update(metadata) - else: - meta_new = metadata - if meta_orig != meta_new: - write_metadata(path, meta_new) - return meta_new - -def create_object_metadata(obj_path): - metadata = get_object_metadata(obj_path) - return restore_metadata(obj_path, metadata) - -def create_container_metadata(cont_path, memcache=None): - metadata = get_container_metadata(cont_path, memcache) - return restore_metadata(cont_path, metadata) - -def create_account_metadata(acc_path, memcache=None): - metadata = get_account_metadata(acc_path, memcache) - return restore_metadata(acc_path, metadata) - -def write_pickle(obj, dest, tmp=None, pickle_protocol=0): - """ - Ensure that a pickle file gets written to disk. The file is first written - to a tmp file location in the destination directory path, ensured it is - synced to disk, then moved to its final destination name. - - This version takes advantage of Gluster's dot-prefix-dot-suffix naming - where the a file named ".thefile.name.9a7aasv" is hashed to the same - Gluster node as "thefile.name". This ensures the renaming of a temp file - once written does not move it to another Gluster node. - - :param obj: python object to be pickled - :param dest: path of final destination file - :param tmp: path to tmp to use, defaults to None (ignored) - :param pickle_protocol: protocol to pickle the obj with, defaults to 0 - """ - dirname = os.path.dirname(dest) - basename = os.path.basename(dest) - tmpname = '.' + basename + '.' + md5(basename + str(random.random())).hexdigest() - tmppath = os.path.join(dirname, tmpname) - with open(tmppath, 'wb') as fo: - pickle.dump(obj, fo, pickle_protocol) - # TODO: This flush() method call turns into a flush() system call - # We'll need to wrap this as well, but we would do this by writing - #a context manager for our own open() method which returns an object - # in fo which makes the gluster API call. - fo.flush() - do_fsync(fo) - do_rename(tmppath, dest) - -# Over-ride Swift's utils.write_pickle with ours -import swift.common.utils -swift.common.utils.write_pickle = write_pickle diff --git a/ufo/gluster/swift/container/__init__.py b/ufo/gluster/swift/container/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/ufo/gluster/swift/container/__init__.py +++ /dev/null diff --git a/ufo/gluster/swift/container/server.py b/ufo/gluster/swift/container/server.py deleted file mode 100644 index c5792aaa289..00000000000 --- a/ufo/gluster/swift/container/server.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" Container Server for Gluster Swift UFO """ - -# Simply importing this monkey patches the constraint handling to fit our -# needs -import gluster.swift.common.constraints - -from swift.container import server -from gluster.swift.common.DiskDir import DiskDir - - -class ContainerController(server.ContainerController): - def _get_container_broker(self, drive, part, account, container): - """ - Overriden to provide the GlusterFS specific broker that talks to - Gluster for the information related to servicing a given request - instead of talking to a database. - - :param drive: drive that holds the container - :param part: partition the container is in - :param account: account name - :param container: container name - :returns: DiskDir object - """ - return DiskDir(self.root, drive, account, container, self.logger) - - -def app_factory(global_conf, **local_conf): - """paste.deploy app factory for creating WSGI container server apps.""" - conf = global_conf.copy() - conf.update(local_conf) - return ContainerController(conf) diff --git a/ufo/gluster/swift/obj/__init__.py b/ufo/gluster/swift/obj/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/ufo/gluster/swift/obj/__init__.py +++ /dev/null diff --git a/ufo/gluster/swift/obj/server.py b/ufo/gluster/swift/obj/server.py deleted file mode 100644 index 1c2b6cb1d7f..00000000000 --- a/ufo/gluster/swift/obj/server.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" Object Server for Gluster Swift UFO """ - -# Simply importing this monkey patches the constraint handling to fit our -# needs -import gluster.swift.common.constraints -import gluster.swift.common.utils - -from swift.obj import server -from gluster.swift.common.DiskFile import Gluster_DiskFile - -# Monkey patch the object server module to use Gluster's DiskFile definition -server.DiskFile = Gluster_DiskFile - - -def app_factory(global_conf, **local_conf): - """paste.deploy app factory for creating WSGI object server apps""" - conf = global_conf.copy() - conf.update(local_conf) - return server.ObjectController(conf) diff --git a/ufo/gluster/swift/proxy/__init__.py b/ufo/gluster/swift/proxy/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/ufo/gluster/swift/proxy/__init__.py +++ /dev/null diff --git a/ufo/gluster/swift/proxy/server.py b/ufo/gluster/swift/proxy/server.py deleted file mode 100644 index 792a97df9a2..00000000000 --- a/ufo/gluster/swift/proxy/server.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# Simply importing this monkey patches the constraint handling to fit our -# needs -import gluster.swift.common.constraints - -from swift.proxy import server - -def app_factory(global_conf, **local_conf): - """paste.deploy app factory for creating WSGI proxy apps.""" - conf = global_conf.copy() - conf.update(local_conf) - return server.Application(conf) diff --git a/ufo/setup.py b/ufo/setup.py deleted file mode 100644 index a48310282d9..00000000000 --- a/ufo/setup.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/python -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from setuptools import setup, find_packages - -from gluster.swift import __canonical_version__ as version - - -name = 'gluster_swift_ufo' - - -setup( - name=name, - version=version, - description='Gluster Swift/UFO', - license='Apache License (2.0)', - author='Red Hat, Inc.', - author_email='gluster-users@gluster.org', - url='https://gluster.org/', - packages=find_packages(exclude=['test', 'bin']), - test_suite='nose.collector', - classifiers=[ - 'Development Status :: 4 - Beta', - 'License :: OSI Approved :: Apache Software License', - 'Operating System :: POSIX :: Linux', - 'Programming Language :: Python :: 2.6', - 'Environment :: No Input/Output (Daemon)', - ], - install_requires=[], # removed for better compat - scripts=[ - 'bin/gluster-swift-gen-builders', - ], - entry_points={ - 'paste.app_factory': [ - 'proxy=gluster.swift.proxy.server:app_factory', - 'object=gluster.swift.obj.server:app_factory', - 'container=gluster.swift.container.server:app_factory', - 'account=gluster.swift.account.server:app_factory', - ], - 'paste.filter_factory': [ - 'gluster=gluster.swift.common.middleware.gluster:filter_factory', - ], - }, - ) diff --git a/ufo/test/__init__.py b/ufo/test/__init__.py deleted file mode 100644 index 50b24ed1a92..00000000000 --- a/ufo/test/__init__.py +++ /dev/null @@ -1,49 +0,0 @@ -# See http://code.google.com/p/python-nose/issues/detail?id=373 -# The code below enables nosetests to work with i18n _() blocks - -import __builtin__ -import sys -import os -from ConfigParser import MissingSectionHeaderError -from StringIO import StringIO - -from swift.common.utils import readconf - -setattr(__builtin__, '_', lambda x: x) - - -# Work around what seems to be a Python bug. -# c.f. https://bugs.launchpad.net/swift/+bug/820185. -import logging -logging.raiseExceptions = False - - -def get_config(section_name=None, defaults=None): - """ - Attempt to get a test config dictionary. - - :param section_name: the section to read (all sections if not defined) - :param defaults: an optional dictionary namespace of defaults - """ - config_file = os.environ.get('SWIFT_TEST_CONFIG_FILE', - '/etc/swift/test.conf') - config = {} - if defaults is not None: - config.update(defaults) - - try: - config = readconf(config_file, section_name) - except SystemExit: - if not os.path.exists(config_file): - print >>sys.stderr, \ - 'Unable to read test config %s - file not found' \ - % config_file - elif not os.access(config_file, os.R_OK): - print >>sys.stderr, \ - 'Unable to read test config %s - permission denied' \ - % config_file - else: - print >>sys.stderr, \ - 'Unable to read test config %s - section %s not found' \ - % (config_file, section_name) - return config diff --git a/ufo/test/unit/__init__.py b/ufo/test/unit/__init__.py deleted file mode 100644 index cb247643be2..00000000000 --- a/ufo/test/unit/__init__.py +++ /dev/null @@ -1,95 +0,0 @@ -""" Gluster Swift Unit Tests """ - -import logging -from collections import defaultdict -from test import get_config -from swift.common.utils import TRUE_VALUES - - -class NullLoggingHandler(logging.Handler): - - def emit(self, record): - pass - - -class FakeLogger(object): - # a thread safe logger - - def __init__(self, *args, **kwargs): - self._clear() - self.level = logging.NOTSET - if 'facility' in kwargs: - self.facility = kwargs['facility'] - - def _clear(self): - self.log_dict = defaultdict(list) - - def _store_in(store_name): - def stub_fn(self, *args, **kwargs): - self.log_dict[store_name].append((args, kwargs)) - return stub_fn - - error = _store_in('error') - info = _store_in('info') - warning = _store_in('warning') - debug = _store_in('debug') - - def exception(self, *args, **kwargs): - self.log_dict['exception'].append((args, kwargs, str(exc_info()[1]))) - - # mock out the StatsD logging methods: - increment = _store_in('increment') - decrement = _store_in('decrement') - timing = _store_in('timing') - timing_since = _store_in('timing_since') - update_stats = _store_in('update_stats') - set_statsd_prefix = _store_in('set_statsd_prefix') - - def setFormatter(self, obj): - self.formatter = obj - - def close(self): - self._clear() - - def set_name(self, name): - # don't touch _handlers - self._name = name - - def acquire(self): - pass - - def release(self): - pass - - def createLock(self): - pass - - def emit(self, record): - pass - - def handle(self, record): - pass - - def flush(self): - pass - - def handleError(self, record): - pass - - -original_syslog_handler = logging.handlers.SysLogHandler - - -def fake_syslog_handler(): - for attr in dir(original_syslog_handler): - if attr.startswith('LOG'): - setattr(FakeLogger, attr, - copy.copy(getattr(logging.handlers.SysLogHandler, attr))) - FakeLogger.priority_map = \ - copy.deepcopy(logging.handlers.SysLogHandler.priority_map) - - logging.handlers.SysLogHandler = FakeLogger - - -if get_config('unit_test').get('fake_syslog', 'False').lower() in TRUE_VALUES: - fake_syslog_handler() diff --git a/ufo/test/unit/common/__init__.py b/ufo/test/unit/common/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/ufo/test/unit/common/__init__.py +++ /dev/null diff --git a/ufo/test/unit/common/data/README.rings b/ufo/test/unit/common/data/README.rings deleted file mode 100644 index 6457501fb38..00000000000 --- a/ufo/test/unit/common/data/README.rings +++ /dev/null @@ -1,3 +0,0 @@ -The unit tests expect certain ring data built using the following command: - - ../../../../bin/gluster-swift-gen-builders test iops
\ No newline at end of file diff --git a/ufo/test/unit/common/data/account.builder b/ufo/test/unit/common/data/account.builder Binary files differdeleted file mode 100644 index 090ba4b74ff..00000000000 --- a/ufo/test/unit/common/data/account.builder +++ /dev/null diff --git a/ufo/test/unit/common/data/account.ring.gz b/ufo/test/unit/common/data/account.ring.gz Binary files differdeleted file mode 100644 index 6d4c8547437..00000000000 --- a/ufo/test/unit/common/data/account.ring.gz +++ /dev/null diff --git a/ufo/test/unit/common/data/account_tree.tar.bz2 b/ufo/test/unit/common/data/account_tree.tar.bz2 Binary files differdeleted file mode 100644 index cb23e4dd71b..00000000000 --- a/ufo/test/unit/common/data/account_tree.tar.bz2 +++ /dev/null diff --git a/ufo/test/unit/common/data/backups/1365124498.account.builder b/ufo/test/unit/common/data/backups/1365124498.account.builder Binary files differdeleted file mode 100644 index 090ba4b74ff..00000000000 --- a/ufo/test/unit/common/data/backups/1365124498.account.builder +++ /dev/null diff --git a/ufo/test/unit/common/data/backups/1365124498.container.builder b/ufo/test/unit/common/data/backups/1365124498.container.builder Binary files differdeleted file mode 100644 index 733d27dd9e3..00000000000 --- a/ufo/test/unit/common/data/backups/1365124498.container.builder +++ /dev/null diff --git a/ufo/test/unit/common/data/backups/1365124498.object.builder b/ufo/test/unit/common/data/backups/1365124498.object.builder Binary files differdeleted file mode 100644 index ff877ec957d..00000000000 --- a/ufo/test/unit/common/data/backups/1365124498.object.builder +++ /dev/null diff --git a/ufo/test/unit/common/data/backups/1365124499.object.builder b/ufo/test/unit/common/data/backups/1365124499.object.builder Binary files differdeleted file mode 100644 index 8b8cd6c1e34..00000000000 --- a/ufo/test/unit/common/data/backups/1365124499.object.builder +++ /dev/null diff --git a/ufo/test/unit/common/data/container.builder b/ufo/test/unit/common/data/container.builder Binary files differdeleted file mode 100644 index 733d27dd9e3..00000000000 --- a/ufo/test/unit/common/data/container.builder +++ /dev/null diff --git a/ufo/test/unit/common/data/container.ring.gz b/ufo/test/unit/common/data/container.ring.gz Binary files differdeleted file mode 100644 index 592b84bab9f..00000000000 --- a/ufo/test/unit/common/data/container.ring.gz +++ /dev/null diff --git a/ufo/test/unit/common/data/container_tree.tar.bz2 b/ufo/test/unit/common/data/container_tree.tar.bz2 Binary files differdeleted file mode 100644 index b4a14928570..00000000000 --- a/ufo/test/unit/common/data/container_tree.tar.bz2 +++ /dev/null diff --git a/ufo/test/unit/common/data/object.builder b/ufo/test/unit/common/data/object.builder Binary files differdeleted file mode 100644 index 8b8cd6c1e34..00000000000 --- a/ufo/test/unit/common/data/object.builder +++ /dev/null diff --git a/ufo/test/unit/common/data/object.ring.gz b/ufo/test/unit/common/data/object.ring.gz Binary files differdeleted file mode 100644 index d2f7192b6ce..00000000000 --- a/ufo/test/unit/common/data/object.ring.gz +++ /dev/null diff --git a/ufo/test/unit/common/test_Glusterfs.py b/ufo/test/unit/common/test_Glusterfs.py deleted file mode 100644 index 7de060aef99..00000000000 --- a/ufo/test/unit/common/test_Glusterfs.py +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright (c) 2013 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest -import os, fcntl, errno, shutil -from tempfile import mkdtemp -import gluster.swift.common.Glusterfs as gfs - -def mock_os_path_ismount(path): - return True - -def mock_get_export_list(): - return ['test', 'test2'] - -def mock_os_system(cmd): - return False - -def mock_fcntl_lockf(f, *a, **kw): - raise IOError(errno.EAGAIN) - -def _init(): - global _RUN_DIR, _OS_SYSTEM, _FCNTL_LOCKF - global _OS_PATH_ISMOUNT, __GET_EXPORT_LIST - - _RUN_DIR = gfs.RUN_DIR - _OS_SYSTEM = os.system - _FCNTL_LOCKF = fcntl.lockf - _OS_PATH_ISMOUNT = os.path.ismount - __GET_EXPORT_LIST = gfs._get_export_list - -def _init_mock_variables(tmpdir): - os.system = mock_os_system - os.path.ismount = mock_os_path_ismount - gfs.RUN_DIR = os.path.join(tmpdir, 'var/run/swift') - gfs._get_export_list = mock_get_export_list - -def _reset_mock_variables(): - gfs.RUN_DIR = _RUN_DIR - gfs._get_export_list = __GET_EXPORT_LIST - - os.system = _OS_SYSTEM - fcntl.lockf = _FCNTL_LOCKF - os.path.ismount = _OS_PATH_ISMOUNT - -class TestGlusterfs(unittest.TestCase): - """ Tests for common.GlusterFS """ - - def setUp(self): - _init() - - def test_mount(self): - try: - tmpdir = mkdtemp() - root = os.path.join(tmpdir, 'mnt/gluster-object') - drive = 'test' - - _init_mock_variables(tmpdir) - assert gfs.mount(root, drive) - finally: - _reset_mock_variables() - shutil.rmtree(tmpdir) - - def test_mount_egain(self): - try: - tmpdir = mkdtemp() - root = os.path.join(tmpdir, 'mnt/gluster-object') - drive = 'test' - - _init_mock_variables(tmpdir) - assert gfs.mount(root, drive) - fcntl.lockf = mock_fcntl_lockf - assert gfs.mount(root, drive) - finally: - _reset_mock_variables() - shutil.rmtree(tmpdir) - - def test_mount_get_export_list_err(self): - gfs._get_export_list = mock_get_export_list - assert not gfs.mount(None, 'drive') - _reset_mock_variables() - - def tearDown(self): - _reset_mock_variables() diff --git a/ufo/test/unit/common/test_diskfile.py b/ufo/test/unit/common/test_diskfile.py deleted file mode 100644 index 85d539a29f0..00000000000 --- a/ufo/test/unit/common/test_diskfile.py +++ /dev/null @@ -1,932 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" Tests for gluster.swift.common.DiskFile """ - -import os -import stat -import errno -import unittest -import tempfile -import shutil -from hashlib import md5 -from swift.common.utils import normalize_timestamp -from swift.common.exceptions import DiskFileNotExist -import gluster.swift.common.DiskFile -import gluster.swift.common.utils -from gluster.swift.common.DiskFile import Gluster_DiskFile, \ - AlreadyExistsAsDir -from gluster.swift.common.utils import DEFAULT_UID, DEFAULT_GID, X_TYPE, \ - X_OBJECT_TYPE -from test_utils import _initxattr, _destroyxattr -from test.unit import FakeLogger - - -_metadata = {} - -def _mock_read_metadata(filename): - if filename in _metadata: - md = _metadata[filename] - else: - md = {} - return md - -def _mock_write_metadata(filename, metadata): - _metadata[filename] = metadata - -def _mock_clear_metadata(): - _metadata = {} - - -class MockException(Exception): - pass - - -def _mock_rmdirs(p): - raise MockException("gluster.swift.common.DiskFile.rmdirs() called") - -def _mock_do_listdir(p): - raise MockException("gluster.swift.common.DiskFile.do_listdir() called") - -def _mock_do_unlink(f): - ose = OSError() - ose.errno = errno.ENOENT - raise ose - - -def _mock_do_unlink_eacces_err(f): - ose = OSError() - ose.errno = errno.EACCES - raise ose - -def _mock_getsize_eaccess_err(f): - ose = OSError() - ose.errno = errno.EACCES - raise ose - -def _mock_do_rmdir_eacces_err(f): - ose = OSError() - ose.errno = errno.EACCES - raise ose - -class MockRenamerCalled(Exception): - pass - - -def _mock_renamer(a, b): - raise MockRenamerCalled() - - -class TestDiskFile(unittest.TestCase): - """ Tests for gluster.swift.common.DiskFile """ - - def setUp(self): - self.lg = FakeLogger() - _initxattr() - _mock_clear_metadata() - self._saved_df_wm = gluster.swift.common.DiskFile.write_metadata - self._saved_df_rm = gluster.swift.common.DiskFile.read_metadata - gluster.swift.common.DiskFile.write_metadata = _mock_write_metadata - gluster.swift.common.DiskFile.read_metadata = _mock_read_metadata - self._saved_ut_wm = gluster.swift.common.utils.write_metadata - self._saved_ut_rm = gluster.swift.common.utils.read_metadata - gluster.swift.common.utils.write_metadata = _mock_write_metadata - gluster.swift.common.utils.read_metadata = _mock_read_metadata - - def tearDown(self): - self.lg = None - _destroyxattr() - gluster.swift.common.DiskFile.write_metadata = self._saved_df_wm - gluster.swift.common.DiskFile.read_metadata = self._saved_df_rm - gluster.swift.common.utils.write_metadata = self._saved_ut_wm - gluster.swift.common.utils.read_metadata = self._saved_ut_rm - - def test_constructor_no_slash(self): - assert not os.path.exists("/tmp/foo") - gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf._obj_path == "" - assert gdf.name == "bar" - assert gdf.datadir == "/tmp/foo/vol0/bar" - assert gdf.device_path == "/tmp/foo/vol0" - assert gdf._container_path == "/tmp/foo/vol0/bar" - assert gdf.disk_chunk_size == 65536 - assert gdf.iter_hook == None - assert gdf.logger == self.lg - assert gdf.uid == DEFAULT_UID - assert gdf.gid == DEFAULT_GID - assert gdf.metadata == {} - assert gdf.meta_file == None - assert gdf.data_file == None - assert gdf.fp == None - assert gdf.iter_etag == None - assert not gdf.started_at_0 - assert not gdf.read_to_eof - assert gdf.quarantined_dir == None - assert not gdf.keep_cache - assert not gdf._is_dir - - def test_constructor_leadtrail_slash(self): - assert not os.path.exists("/tmp/foo") - gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", - "/b/a/z/", self.lg) - assert gdf._obj == "z" - assert gdf._obj_path == "b/a" - assert gdf.name == "bar/b/a" - assert gdf.datadir == "/tmp/foo/vol0/bar/b/a" - assert gdf.device_path == "/tmp/foo/vol0" - - def test_constructor_no_metadata(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - stats = os.stat(the_file) - ts = normalize_timestamp(stats.st_ctime) - etag = md5() - etag.update("1234") - etag = etag.hexdigest() - exp_md = { - 'Content-Length': 4, - 'ETag': etag, - 'X-Timestamp': ts, - 'Content-Type': 'application/octet-stream'} - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - assert gdf.fp is None - assert gdf.metadata == exp_md - finally: - shutil.rmtree(td) - - def test_constructor_existing_metadata(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - ini_md = { - 'X-Type': 'Object', - 'X-Object-Type': 'file', - 'Content-Length': 5, - 'ETag': 'etag', - 'X-Timestamp': 'ts', - 'Content-Type': 'application/loctet-stream'} - _metadata[the_file] = ini_md - exp_md = ini_md.copy() - del exp_md['X-Type'] - del exp_md['X-Object-Type'] - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - assert gdf.fp is None - assert gdf.metadata == exp_md - finally: - shutil.rmtree(td) - - def test_constructor_invalid_existing_metadata(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - inv_md = { - 'Content-Length': 5, - 'ETag': 'etag', - 'X-Timestamp': 'ts', - 'Content-Type': 'application/loctet-stream'} - _metadata[the_file] = inv_md - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - assert gdf.fp is None - assert gdf.metadata != inv_md - finally: - shutil.rmtree(td) - - def test_constructor_isdir(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "d") - try: - os.makedirs(the_dir) - ini_md = { - 'X-Type': 'Object', - 'X-Object-Type': 'dir', - 'Content-Length': 5, - 'ETag': 'etag', - 'X-Timestamp': 'ts', - 'Content-Type': 'application/loctet-stream'} - _metadata[the_dir] = ini_md - exp_md = ini_md.copy() - del exp_md['X-Type'] - del exp_md['X-Object-Type'] - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "d", self.lg, keep_data_fp=True) - assert gdf._obj == "d" - assert gdf.data_file == the_dir - assert gdf._is_dir - assert gdf.fp is None - assert gdf.metadata == exp_md - finally: - shutil.rmtree(td) - - def test_constructor_keep_data_fp(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg, keep_data_fp=True) - assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - assert gdf.fp is not None - finally: - shutil.rmtree(td) - - def test_constructor_chunk_size(self): - assert not os.path.exists("/tmp/foo") - gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", - "z", self.lg, disk_chunk_size=8192) - assert gdf.disk_chunk_size == 8192 - - def test_constructor_iter_hook(self): - assert not os.path.exists("/tmp/foo") - gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", - "z", self.lg, iter_hook='hook') - assert gdf.iter_hook == 'hook' - - def test_close(self): - assert not os.path.exists("/tmp/foo") - gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", - "z", self.lg) - # Should be a no-op, as by default is_dir is False, but fp is None - gdf.close() - - gdf._is_dir = True - gdf.fp = "123" - # Should still be a no-op as is_dir is True (marker directory) - gdf.close() - assert gdf.fp == "123" - - gdf._is_dir = False - saved_dc = gluster.swift.common.DiskFile.do_close - self.called = False - def our_do_close(fp): - self.called = True - gluster.swift.common.DiskFile.do_close = our_do_close - try: - gdf.close() - assert self.called - assert gdf.fp is None - finally: - gluster.swift.common.DiskFile.do_close = saved_dc - - def test_is_deleted(self): - assert not os.path.exists("/tmp/foo") - gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf.is_deleted() - gdf.data_file = "/tmp/foo/bar" - assert not gdf.is_deleted() - - def test_create_dir_object(self): - td = tempfile.mkdtemp() - the_dir = os.path.join(td, "vol0", "bar", "dir") - try: - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "dir/z", self.lg) - # Not created, dir object path is different, just checking - assert gdf._obj == "z" - gdf._create_dir_object(the_dir) - assert os.path.isdir(the_dir) - assert the_dir in _metadata - finally: - shutil.rmtree(td) - - def test_create_dir_object_exists(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "dir") - try: - os.makedirs(the_path) - with open(the_dir, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "dir/z", self.lg) - # Not created, dir object path is different, just checking - assert gdf._obj == "z" - def _mock_do_chown(p, u, g): - assert u == DEFAULT_UID - assert g == DEFAULT_GID - dc = gluster.swift.common.DiskFile.do_chown - gluster.swift.common.DiskFile.do_chown = _mock_do_chown - try: - gdf._create_dir_object(the_dir) - finally: - gluster.swift.common.DiskFile.do_chown = dc - assert os.path.isdir(the_dir) - assert the_dir in _metadata - finally: - shutil.rmtree(td) - - def test_put_metadata(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "z") - try: - os.makedirs(the_dir) - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - md = { 'Content-Type': 'application/octet-stream', 'a': 'b' } - gdf.put_metadata(md.copy()) - assert gdf.metadata == md, "gdf.metadata = %r, md = %r" % (gdf.metadata, md) - assert _metadata[the_dir] == md - finally: - shutil.rmtree(td) - - def test_put_w_tombstone(self): - assert not os.path.exists("/tmp/foo") - gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf.metadata == {} - - gdf.put_metadata({'x': '1'}, tombstone=True) - assert gdf.metadata == {} - - def test_put_w_meta_file(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - newmd = gdf.metadata.copy() - newmd['X-Object-Meta-test'] = '1234' - gdf.put_metadata(newmd) - assert gdf.metadata == newmd - assert _metadata[the_file] == newmd - finally: - shutil.rmtree(td) - - def test_put_w_meta_file_no_content_type(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - newmd = gdf.metadata.copy() - newmd['Content-Type'] = '' - newmd['X-Object-Meta-test'] = '1234' - gdf.put_metadata(newmd) - assert gdf.metadata == newmd - assert _metadata[the_file] == newmd - finally: - shutil.rmtree(td) - - def test_put_w_meta_dir(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "dir") - try: - os.makedirs(the_dir) - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "dir", self.lg) - newmd = gdf.metadata.copy() - newmd['X-Object-Meta-test'] = '1234' - gdf.put_metadata(newmd) - assert gdf.metadata == newmd - assert _metadata[the_dir] == newmd - finally: - shutil.rmtree(td) - - def test_put_w_marker_dir(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "dir") - try: - os.makedirs(the_dir) - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "dir", self.lg) - newmd = gdf.metadata.copy() - newmd['X-Object-Meta-test'] = '1234' - gdf.put_metadata(newmd) - assert gdf.metadata == newmd - assert _metadata[the_dir] == newmd - finally: - shutil.rmtree(td) - - def test_put_w_marker_dir_create(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "dir") - try: - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "dir", self.lg) - assert gdf.metadata == {} - newmd = { - 'Content-Length': 0, - 'ETag': 'etag', - 'X-Timestamp': 'ts', - 'Content-Type': 'application/directory'} - gdf.put(None, newmd, extension='.dir') - assert gdf.data_file == the_dir - assert gdf.metadata == newmd - assert _metadata[the_dir] == newmd - finally: - shutil.rmtree(td) - - def test_put_is_dir(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "dir") - try: - os.makedirs(the_dir) - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "dir", self.lg) - origmd = gdf.metadata.copy() - origfmd = _metadata[the_dir] - newmd = gdf.metadata.copy() - # FIXME: This is a hack to get to the code-path; it is not clear - # how this can happen normally. - newmd['Content-Type'] = '' - newmd['X-Object-Meta-test'] = '1234' - try: - gdf.put(None, newmd, extension='.data') - except AlreadyExistsAsDir: - pass - else: - self.fail("Expected to encounter 'already-exists-as-dir' exception") - assert gdf.metadata == origmd - assert _metadata[the_dir] == origfmd - finally: - shutil.rmtree(td) - - def test_put(self): - td = tempfile.mkdtemp() - try: - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf._obj_path == "" - assert gdf.name == "bar" - assert gdf.datadir == os.path.join(td, "vol0", "bar") - assert gdf.data_file is None - - body = '1234\n' - etag = md5() - etag.update(body) - etag = etag.hexdigest() - metadata = { - 'X-Timestamp': '1234', - 'Content-Type': 'file', - 'ETag': etag, - 'Content-Length': '5', - } - - with gdf.mkstemp() as fd: - assert gdf.tmppath is not None - tmppath = gdf.tmppath - os.write(fd, body) - gdf.put(fd, metadata) - - assert gdf.data_file == os.path.join(td, "vol0", "bar", "z") - assert os.path.exists(gdf.data_file) - assert not os.path.exists(tmppath) - finally: - shutil.rmtree(td) - - def test_put_obj_path(self): - the_obj_path = os.path.join("b", "a") - the_file = os.path.join(the_obj_path, "z") - td = tempfile.mkdtemp() - try: - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - the_file, self.lg) - assert gdf._obj == "z" - assert gdf._obj_path == the_obj_path - assert gdf.name == os.path.join("bar", "b", "a") - assert gdf.datadir == os.path.join(td, "vol0", "bar", "b", "a") - assert gdf.data_file is None - - body = '1234\n' - etag = md5() - etag.update(body) - etag = etag.hexdigest() - metadata = { - 'X-Timestamp': '1234', - 'Content-Type': 'file', - 'ETag': etag, - 'Content-Length': '5', - } - - with gdf.mkstemp() as fd: - assert gdf.tmppath is not None - tmppath = gdf.tmppath - os.write(fd, body) - gdf.put(fd, metadata) - - assert gdf.data_file == os.path.join(td, "vol0", "bar", "b", "a", "z") - assert os.path.exists(gdf.data_file) - assert not os.path.exists(tmppath) - finally: - shutil.rmtree(td) - - def test_unlinkold_no_metadata(self): - assert not os.path.exists("/tmp/foo") - gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf.metadata == {} - _saved_rmdirs = gluster.swift.common.DiskFile.rmdirs - _saved_do_listdir = gluster.swift.common.DiskFile.do_listdir - gluster.swift.common.DiskFile.rmdirs = _mock_rmdirs - gluster.swift.common.DiskFile.do_listdir = _mock_do_listdir - try: - gdf.unlinkold(None) - except MockException as exp: - self.fail(str(exp)) - finally: - gluster.swift.common.DiskFile.rmdirs = _saved_rmdirs - gluster.swift.common.DiskFile.do_listdir = _saved_do_listdir - - def test_unlinkold_same_timestamp(self): - assert not os.path.exists("/tmp/foo") - gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf.metadata == {} - gdf.metadata['X-Timestamp'] = 1 - _saved_rmdirs = gluster.swift.common.DiskFile.rmdirs - _saved_do_listdir = gluster.swift.common.DiskFile.do_listdir - gluster.swift.common.DiskFile.rmdirs = _mock_rmdirs - gluster.swift.common.DiskFile.do_listdir = _mock_do_listdir - try: - gdf.unlinkold(1) - except MockException as exp: - self.fail(str(exp)) - finally: - gluster.swift.common.DiskFile.rmdirs = _saved_rmdirs - gluster.swift.common.DiskFile.do_listdir = _saved_do_listdir - - def test_unlinkold_file(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - - later = float(gdf.metadata['X-Timestamp']) + 1 - gdf.unlinkold(normalize_timestamp(later)) - assert os.path.isdir(gdf.datadir) - assert not os.path.exists(os.path.join(gdf.datadir, gdf._obj)) - finally: - shutil.rmtree(td) - - def test_unlinkold_file_not_found(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - - # Handle the case the file is not in the directory listing. - os.unlink(the_file) - - later = float(gdf.metadata['X-Timestamp']) + 1 - gdf.unlinkold(normalize_timestamp(later)) - assert os.path.isdir(gdf.datadir) - assert not os.path.exists(os.path.join(gdf.datadir, gdf._obj)) - finally: - shutil.rmtree(td) - - def test_unlinkold_file_unlink_error(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - - later = float(gdf.metadata['X-Timestamp']) + 1 - - stats = os.stat(the_path) - os.chmod(the_path, stats.st_mode & (~stat.S_IWUSR)) - - # Handle the case do_unlink() raises an OSError - __os_unlink = os.unlink - os.unlink = _mock_do_unlink_eacces_err - try: - gdf.unlinkold(normalize_timestamp(later)) - except OSError as e: - assert e.errno == errno.EACCES - else: - self.fail("Excepted an OSError when unlinking file") - finally: - os.unlink = __os_unlink - os.chmod(the_path, stats.st_mode) - - assert os.path.isdir(gdf.datadir) - assert os.path.exists(os.path.join(gdf.datadir, gdf._obj)) - finally: - shutil.rmtree(td) - - def test_unlinkold_is_dir(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "d") - try: - os.makedirs(the_dir) - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "d", self.lg, keep_data_fp=True) - assert gdf.data_file == the_dir - assert gdf._is_dir - - later = float(gdf.metadata['X-Timestamp']) + 1 - gdf.unlinkold(normalize_timestamp(later)) - assert os.path.isdir(gdf.datadir) - assert not os.path.exists(os.path.join(gdf.datadir, gdf._obj)) - finally: - shutil.rmtree(td) - - def test_unlinkold_is_dir_failure(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "d") - try: - os.makedirs(the_dir) - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "d", self.lg, keep_data_fp=True) - assert gdf.data_file == the_dir - assert gdf._is_dir - - stats = os.stat(gdf.datadir) - os.chmod(gdf.datadir, 0) - __os_rmdir = os.rmdir - os.rmdir = _mock_do_rmdir_eacces_err - try: - later = float(gdf.metadata['X-Timestamp']) + 1 - gdf.unlinkold(normalize_timestamp(later)) - finally: - os.chmod(gdf.datadir, stats.st_mode) - os.rmdir = __os_rmdir - assert os.path.isdir(gdf.datadir) - assert os.path.isdir(gdf.data_file) - finally: - shutil.rmtree(td) - - def test_get_data_file_size(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - assert 4 == gdf.get_data_file_size() - finally: - shutil.rmtree(td) - - def test_get_data_file_size(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - assert 4 == gdf.metadata['Content-Length'] - gdf.metadata['Content-Length'] = 3 - assert 4 == gdf.get_data_file_size() - assert 4 == gdf.metadata['Content-Length'] - finally: - shutil.rmtree(td) - - def test_get_data_file_size_dne(self): - assert not os.path.exists("/tmp/foo") - gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", - "/b/a/z/", self.lg) - try: - s = gdf.get_data_file_size() - except DiskFileNotExist: - pass - else: - self.fail("Expected DiskFileNotExist exception") - - def test_get_data_file_size_dne_os_err(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - gdf.data_file = gdf.data_file + ".dne" - try: - s = gdf.get_data_file_size() - except DiskFileNotExist: - pass - else: - self.fail("Expected DiskFileNotExist exception") - finally: - shutil.rmtree(td) - - def test_get_data_file_size_os_err(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_file = os.path.join(the_path, "z") - try: - os.makedirs(the_path) - with open(the_file, "wb") as fd: - fd.write("1234") - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf._obj == "z" - assert gdf.data_file == the_file - assert not gdf._is_dir - stats = os.stat(the_path) - os.chmod(the_path, 0) - __os_path_getsize = os.path.getsize - os.path.getsize = _mock_getsize_eaccess_err - try: - s = gdf.get_data_file_size() - except OSError as err: - assert err.errno == errno.EACCES - else: - self.fail("Expected OSError exception") - finally: - os.path.getsize = __os_path_getsize - os.chmod(the_path, stats.st_mode) - finally: - shutil.rmtree(td) - - def test_get_data_file_size_dir(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "d") - try: - os.makedirs(the_dir) - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "d", self.lg, keep_data_fp=True) - assert gdf._obj == "d" - assert gdf.data_file == the_dir - assert gdf._is_dir - assert 0 == gdf.get_data_file_size() - finally: - shutil.rmtree(td) - - def test_filter_metadata(self): - assert not os.path.exists("/tmp/foo") - gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", - "z", self.lg) - assert gdf.metadata == {} - gdf.filter_metadata() - assert gdf.metadata == {} - - gdf.metadata[X_TYPE] = 'a' - gdf.metadata[X_OBJECT_TYPE] = 'b' - gdf.metadata['foobar'] = 'c' - gdf.filter_metadata() - assert X_TYPE not in gdf.metadata - assert X_OBJECT_TYPE not in gdf.metadata - assert 'foobar' in gdf.metadata - - def test_mkstemp(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "dir") - try: - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "dir/z", self.lg) - saved_tmppath = '' - with gdf.mkstemp() as fd: - assert gdf.datadir == os.path.join(td, "vol0", "bar", "dir") - assert os.path.isdir(gdf.datadir) - saved_tmppath = gdf.tmppath - assert os.path.dirname(saved_tmppath) == gdf.datadir - assert os.path.basename(saved_tmppath)[:3] == '.z.' - assert os.path.exists(saved_tmppath) - os.write(fd, "123") - assert not os.path.exists(saved_tmppath) - finally: - shutil.rmtree(td) - - def test_mkstemp_err_on_close(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "dir") - try: - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "dir/z", self.lg) - saved_tmppath = '' - with gdf.mkstemp() as fd: - assert gdf.datadir == os.path.join(td, "vol0", "bar", "dir") - assert os.path.isdir(gdf.datadir) - saved_tmppath = gdf.tmppath - assert os.path.dirname(saved_tmppath) == gdf.datadir - assert os.path.basename(saved_tmppath)[:3] == '.z.' - assert os.path.exists(saved_tmppath) - os.write(fd, "123") - # At the end of previous with block a close on fd is called. - # Calling os.close on the same fd will raise an OSError - # exception and we must catch it. - try: - os.close(fd) - except OSError as err: - pass - else: - self.fail("Exception expected") - assert not os.path.exists(saved_tmppath) - finally: - shutil.rmtree(td) - - def test_mkstemp_err_on_unlink(self): - td = tempfile.mkdtemp() - the_path = os.path.join(td, "vol0", "bar") - the_dir = os.path.join(the_path, "dir") - try: - gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar", - "dir/z", self.lg) - saved_tmppath = '' - with gdf.mkstemp() as fd: - assert gdf.datadir == os.path.join(td, "vol0", "bar", "dir") - assert os.path.isdir(gdf.datadir) - saved_tmppath = gdf.tmppath - assert os.path.dirname(saved_tmppath) == gdf.datadir - assert os.path.basename(saved_tmppath)[:3] == '.z.' - assert os.path.exists(saved_tmppath) - os.write(fd, "123") - os.unlink(saved_tmppath) - assert not os.path.exists(saved_tmppath) - finally: - shutil.rmtree(td) diff --git a/ufo/test/unit/common/test_fs_utils.py b/ufo/test/unit/common/test_fs_utils.py deleted file mode 100644 index 186e07d59b6..00000000000 --- a/ufo/test/unit/common/test_fs_utils.py +++ /dev/null @@ -1,277 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import shutil -import random -import unittest -from tempfile import mkdtemp, mkstemp -from gluster.swift.common import fs_utils as fs -from gluster.swift.common.exceptions import NotDirectoryError, \ - FileOrDirNotFoundError - -class TestUtils(unittest.TestCase): - """ Tests for common.utils """ - - def test_do_walk(self): - try: - # create directory structure - tmpparent = mkdtemp() - tmpdirs = [] - tmpfiles = [] - for i in range(5): - tmpdirs.append(mkdtemp(dir=tmpparent).rsplit(os.path.sep, 1)[1]) - tmpfiles.append(mkstemp(dir=tmpparent)[1].rsplit(os.path.sep, \ - 1)[1]) - - for path, dirnames, filenames in fs.do_walk(tmpparent): - assert path == tmpparent - assert dirnames.sort() == tmpdirs.sort() - assert filenames.sort() == tmpfiles.sort() - break - finally: - shutil.rmtree(tmpparent) - - def test_do_open(self): - try: - fd, tmpfile = mkstemp() - f = fs.do_open(tmpfile, 'r') - try: - f.write('test') - except IOError as err: - pass - else: - self.fail("IOError expected") - finally: - f.close() - os.close(fd) - os.remove(tmpfile) - - def test_do_open_err(self): - try: - fs.do_open(os.path.join('/tmp', str(random.random())), 'r') - except IOError: - pass - else: - self.fail("IOError expected") - - def test_do_write(self): - try: - fd, tmpfile = mkstemp() - cnt = fs.do_write(fd, "test") - assert cnt == len("test") - finally: - os.close(fd) - os.remove(tmpfile) - - def test_do_write_err(self): - try: - fd, tmpfile = mkstemp() - fd1 = os.open(tmpfile, os.O_RDONLY) - fs.do_write(fd1, "test") - except OSError: - pass - else: - self.fail("OSError expected") - finally: - os.close(fd) - os.close(fd1) - - def test_do_mkdir(self): - try: - path = os.path.join('/tmp', str(random.random())) - fs.do_mkdir(path) - assert os.path.exists(path) - assert fs.do_mkdir(path) - finally: - os.rmdir(path) - - def test_do_mkdir_err(self): - try: - path = os.path.join('/tmp', str(random.random()), str(random.random())) - fs.do_mkdir(path) - except OSError: - pass - else: - self.fail("OSError expected") - - - def test_do_makedirs(self): - try: - subdir = os.path.join('/tmp', str(random.random())) - path = os.path.join(subdir, str(random.random())) - fs.do_makedirs(path) - assert os.path.exists(path) - assert fs.do_makedirs(path) - finally: - shutil.rmtree(subdir) - - def test_do_listdir(self): - try: - tmpdir = mkdtemp() - subdir = [] - for i in range(5): - subdir.append(mkdtemp(dir=tmpdir).rsplit(os.path.sep, 1)[1]) - - assert subdir.sort() == fs.do_listdir(tmpdir).sort() - finally: - shutil.rmtree(tmpdir) - - def test_do_listdir_err(self): - try: - path = os.path.join('/tmp', str(random.random())) - fs.do_listdir(path) - except OSError: - pass - else: - self.fail("OSError expected") - - def test_do_stat(self): - try: - tmpdir = mkdtemp() - fd, tmpfile = mkstemp(dir=tmpdir) - buf1 = os.stat(tmpfile) - buf2 = fs.do_stat(fd) - buf3 = fs.do_stat(tmpfile) - - assert buf1 == buf2 - assert buf1 == buf3 - finally: - os.close(fd) - os.remove(tmpfile) - os.rmdir(tmpdir) - - def test_do_stat_err(self): - try: - fs.do_stat(os.path.join('/tmp', str(random.random()))) - except OSError: - pass - else: - self.fail("OSError expected") - - def test_do_close(self): - try: - fd, tmpfile = mkstemp() - fs.do_close(fd); - try: - os.write(fd, "test") - except OSError: - pass - else: - self.fail("OSError expected") - fp = open(tmpfile) - fs.do_close(fp) - finally: - os.remove(tmpfile) - - def test_do_unlink(self): - try: - fd, tmpfile = mkstemp() - fs.do_unlink(tmpfile) - assert not os.path.exists(tmpfile) - assert fs.do_unlink(os.path.join('/tmp', str(random.random()))) - finally: - os.close(fd) - - def test_do_unlink_err(self): - try: - tmpdir = mkdtemp() - fs.do_unlink(tmpdir) - except OSError: - pass - else: - self.fail('OSError expected') - finally: - os.rmdir(tmpdir) - - def test_do_rmdir(self): - tmpdir = mkdtemp() - fs.do_rmdir(tmpdir) - assert not os.path.exists(tmpdir) - assert not fs.do_rmdir(os.path.join('/tmp', str(random.random()))) - - def test_do_rmdir_err(self): - try: - fd, tmpfile = mkstemp() - fs.do_rmdir(tmpfile) - except OSError: - pass - else: - self.fail('OSError expected') - finally: - os.close(fd) - os.remove(tmpfile) - - def test_do_rename(self): - try: - srcpath = mkdtemp() - destpath = os.path.join('/tmp', str(random.random())) - fs.do_rename(srcpath, destpath) - assert not os.path.exists(srcpath) - assert os.path.exists(destpath) - finally: - os.rmdir(destpath) - - def test_do_rename_err(self): - try: - srcpath = os.path.join('/tmp', str(random.random())) - destpath = os.path.join('/tmp', str(random.random())) - fs.do_rename(srcpath, destpath) - except OSError: - pass - else: - self.fail("OSError expected") - - def test_dir_empty(self): - try: - tmpdir = mkdtemp() - subdir = mkdtemp(dir=tmpdir) - assert not fs.dir_empty(tmpdir) - assert fs.dir_empty(subdir) - finally: - shutil.rmtree(tmpdir) - - def test_dir_empty_err(self): - try: - try: - assert fs.dir_empty(os.path.join('/tmp', str(random.random()))) - except FileOrDirNotFoundError: - pass - else: - self.fail("FileOrDirNotFoundError exception expected") - - fd, tmpfile = mkstemp() - try: - fs.dir_empty(tmpfile) - except NotDirectoryError: - pass - else: - self.fail("NotDirectoryError exception expected") - finally: - os.close(fd) - os.unlink(tmpfile) - - def test_rmdirs(self): - try: - tmpdir = mkdtemp() - subdir = mkdtemp(dir=tmpdir) - fd, tmpfile = mkstemp(dir=tmpdir) - assert not fs.rmdirs(tmpfile) - assert not fs.rmdirs(tmpdir) - assert fs.rmdirs(subdir) - assert not os.path.exists(subdir) - finally: - os.close(fd) - shutil.rmtree(tmpdir) diff --git a/ufo/test/unit/common/test_ring.py b/ufo/test/unit/common/test_ring.py deleted file mode 100644 index 8b7509ccea8..00000000000 --- a/ufo/test/unit/common/test_ring.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright (c) 2013 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import unittest -import gluster.swift.common.constraints -import swift.common.utils -from gluster.swift.common.ring import Ring - - -class TestRing(unittest.TestCase): - """ Tests for common.utils """ - - def setUp(self): - swift.common.utils.HASH_PATH_SUFFIX = 'endcap' - swiftdir = os.path.join(os.getcwd(), "common", "data") - self.ring = Ring(swiftdir, ring_name='object') - - def test_first_device(self): - part, node = self.ring.get_nodes('test') - assert node[0]['device'] == 'test' - node = self.ring.get_part_nodes(0) - assert node[0]['device'] == 'test' - for node in self.ring.get_more_nodes(0): - assert node['device'] == 'volume_not_in_ring' - - def test_invalid_device(self): - part, node = self.ring.get_nodes('test2') - assert node[0]['device'] == 'volume_not_in_ring' - node = self.ring.get_part_nodes(0) - assert node[0]['device'] == 'volume_not_in_ring' - - def test_second_device(self): - part, node = self.ring.get_nodes('iops') - assert node[0]['device'] == 'iops' - node = self.ring.get_part_nodes(0) - assert node[0]['device'] == 'iops' - for node in self.ring.get_more_nodes(0): - assert node['device'] == 'volume_not_in_ring' - - def test_second_device_with_reseller_prefix(self): - part, node = self.ring.get_nodes('AUTH_iops') - assert node[0]['device'] == 'iops' diff --git a/ufo/test/unit/common/test_utils.py b/ufo/test/unit/common/test_utils.py deleted file mode 100644 index c645509fa24..00000000000 --- a/ufo/test/unit/common/test_utils.py +++ /dev/null @@ -1,1020 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" Tests for common.utils """ - -import os -import unittest -import errno -import xattr -import cPickle as pickle -import tempfile -import hashlib -import tarfile -import shutil -from collections import defaultdict -from swift.common.utils import normalize_timestamp -from gluster.swift.common import utils, Glusterfs - -# -# Somewhat hacky way of emulating the operation of xattr calls. They are made -# against a dictionary that stores the xattr key/value pairs. -# -_xattrs = {} -_xattr_op_cnt = defaultdict(int) -_xattr_set_err = {} -_xattr_get_err = {} -_xattr_rem_err = {} - -def _xkey(path, key): - return "%s:%s" % (path, key) - -def _setxattr(path, key, value, *args, **kwargs): - _xattr_op_cnt['set'] += 1 - xkey = _xkey(path, key) - if xkey in _xattr_set_err: - e = IOError() - e.errno = _xattr_set_err[xkey] - raise e - global _xattrs - _xattrs[xkey] = value - -def _getxattr(path, key, *args, **kwargs): - _xattr_op_cnt['get'] += 1 - xkey = _xkey(path, key) - if xkey in _xattr_get_err: - e = IOError() - e.errno = _xattr_get_err[xkey] - raise e - global _xattrs - if xkey in _xattrs: - ret_val = _xattrs[xkey] - else: - e = IOError("Fake IOError") - e.errno = errno.ENODATA - raise e - return ret_val - -def _removexattr(path, key, *args, **kwargs): - _xattr_op_cnt['remove'] += 1 - xkey = _xkey(path, key) - if xkey in _xattr_rem_err: - e = IOError() - e.errno = _xattr_rem_err[xkey] - raise e - global _xattrs - if xkey in _xattrs: - del _xattrs[xkey] - else: - e = IOError("Fake IOError") - e.errno = errno.ENODATA - raise e - -def _initxattr(): - global _xattrs - _xattrs = {} - global _xattr_op_cnt - _xattr_op_cnt = defaultdict(int) - global _xattr_set_err, _xattr_get_err, _xattr_rem_err - _xattr_set_err = {} - _xattr_get_err = {} - _xattr_rem_err = {} - - # Save the current methods - global _xattr_set; _xattr_set = xattr.setxattr - global _xattr_get; _xattr_get = xattr.getxattr - global _xattr_remove; _xattr_remove = xattr.removexattr - - # Monkey patch the calls we use with our internal unit test versions - xattr.setxattr = _setxattr - xattr.getxattr = _getxattr - xattr.removexattr = _removexattr - -def _destroyxattr(): - # Restore the current methods just in case - global _xattr_set; xattr.setxattr = _xattr_set - global _xattr_get; xattr.getxattr = _xattr_get - global _xattr_remove; xattr.removexattr = _xattr_remove - # Destroy the stored values and - global _xattrs; _xattrs = None - - -class SimMemcache(object): - def __init__(self): - self._d = {} - - def get(self, key): - return self._d.get(key, None) - - def set(self, key, value): - self._d[key] = value - - -class TestUtils(unittest.TestCase): - """ Tests for common.utils """ - - def setUp(self): - _initxattr() - - def tearDown(self): - _destroyxattr() - - def test_write_metadata(self): - path = "/tmp/foo/w" - orig_d = { 'bar' : 'foo' } - utils.write_metadata(path, orig_d) - xkey = _xkey(path, utils.METADATA_KEY) - assert len(_xattrs.keys()) == 1 - assert xkey in _xattrs - assert orig_d == pickle.loads(_xattrs[xkey]) - assert _xattr_op_cnt['set'] == 1 - - def test_write_metadata_err(self): - path = "/tmp/foo/w" - orig_d = { 'bar' : 'foo' } - xkey = _xkey(path, utils.METADATA_KEY) - _xattr_set_err[xkey] = errno.EOPNOTSUPP - try: - utils.write_metadata(path, orig_d) - except IOError as e: - assert e.errno == errno.EOPNOTSUPP - assert len(_xattrs.keys()) == 0 - assert _xattr_op_cnt['set'] == 1 - else: - self.fail("Expected an IOError exception on write") - - def test_write_metadata_multiple(self): - # At 64 KB an xattr key/value pair, this should generate three keys. - path = "/tmp/foo/w" - orig_d = { 'bar' : 'x' * 150000 } - utils.write_metadata(path, orig_d) - assert len(_xattrs.keys()) == 3, "Expected 3 keys, found %d" % len(_xattrs.keys()) - payload = '' - for i in range(0,3): - xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or '')) - assert xkey in _xattrs - assert len(_xattrs[xkey]) <= utils.MAX_XATTR_SIZE - payload += _xattrs[xkey] - assert orig_d == pickle.loads(payload) - assert _xattr_op_cnt['set'] == 3, "%r" % _xattr_op_cnt - - def test_clean_metadata(self): - path = "/tmp/foo/c" - expected_d = { 'a': 'y' * 150000 } - expected_p = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL) - for i in range(0,3): - xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or '')) - _xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE] - expected_p = expected_p[utils.MAX_XATTR_SIZE:] - assert not expected_p - utils.clean_metadata(path) - assert _xattr_op_cnt['remove'] == 4, "%r" % _xattr_op_cnt - - def test_clean_metadata_err(self): - path = "/tmp/foo/c" - xkey = _xkey(path, utils.METADATA_KEY) - _xattrs[xkey] = pickle.dumps({ 'a': 'y' }, utils.PICKLE_PROTOCOL) - _xattr_rem_err[xkey] = errno.EOPNOTSUPP - try: - utils.clean_metadata(path) - except IOError as e: - assert e.errno == errno.EOPNOTSUPP - assert _xattr_op_cnt['remove'] == 1, "%r" % _xattr_op_cnt - else: - self.fail("Expected an IOError exception on remove") - - def test_read_metadata(self): - path = "/tmp/foo/r" - expected_d = { 'a': 'y' } - xkey = _xkey(path, utils.METADATA_KEY) - _xattrs[xkey] = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL) - res_d = utils.read_metadata(path) - assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d) - assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt - - def test_read_metadata_notfound(self): - path = "/tmp/foo/r" - res_d = utils.read_metadata(path) - assert res_d == {} - assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt - - def test_read_metadata_err(self): - path = "/tmp/foo/r" - expected_d = { 'a': 'y' } - xkey = _xkey(path, utils.METADATA_KEY) - _xattrs[xkey] = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL) - _xattr_get_err[xkey] = errno.EOPNOTSUPP - try: - res_d = utils.read_metadata(path) - except IOError as e: - assert e.errno == errno.EOPNOTSUPP - assert (_xattr_op_cnt['get'] == 1), "%r" % _xattr_op_cnt - else: - self.fail("Expected an IOError exception on get") - - def test_read_metadata_multiple(self): - path = "/tmp/foo/r" - expected_d = { 'a': 'y' * 150000 } - expected_p = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL) - for i in range(0,3): - xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or '')) - _xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE] - expected_p = expected_p[utils.MAX_XATTR_SIZE:] - assert not expected_p - res_d = utils.read_metadata(path) - assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d) - assert _xattr_op_cnt['get'] == 3, "%r" % _xattr_op_cnt - - def test_read_metadata_multiple_one_missing(self): - path = "/tmp/foo/r" - expected_d = { 'a': 'y' * 150000 } - expected_p = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL) - for i in range(0,2): - xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or '')) - _xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE] - expected_p = expected_p[utils.MAX_XATTR_SIZE:] - assert len(expected_p) <= utils.MAX_XATTR_SIZE - res_d = utils.read_metadata(path) - assert res_d == {} - assert _xattr_op_cnt['get'] == 3, "%r" % _xattr_op_cnt - assert len(_xattrs.keys()) == 0, "Expected 0 keys, found %d" % len(_xattrs.keys()) - - def test_restore_metadata_none(self): - # No initial metadata - path = "/tmp/foo/i" - res_d = utils.restore_metadata(path, { 'b': 'y' }) - expected_d = { 'b': 'y' } - assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d) - assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt - assert _xattr_op_cnt['set'] == 1, "%r" % _xattr_op_cnt - - def test_restore_metadata(self): - # Initial metadata - path = "/tmp/foo/i" - initial_d = { 'a': 'z' } - xkey = _xkey(path, utils.METADATA_KEY) - _xattrs[xkey] = pickle.dumps(initial_d, utils.PICKLE_PROTOCOL) - res_d = utils.restore_metadata(path, { 'b': 'y' }) - expected_d = { 'a': 'z', 'b': 'y' } - assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d) - assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt - assert _xattr_op_cnt['set'] == 1, "%r" % _xattr_op_cnt - - def test_restore_metadata_nochange(self): - # Initial metadata but no changes - path = "/tmp/foo/i" - initial_d = { 'a': 'z' } - xkey = _xkey(path, utils.METADATA_KEY) - _xattrs[xkey] = pickle.dumps(initial_d, utils.PICKLE_PROTOCOL) - res_d = utils.restore_metadata(path, {}) - expected_d = { 'a': 'z' } - assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d) - assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt - assert _xattr_op_cnt['set'] == 0, "%r" % _xattr_op_cnt - - def test_add_timestamp_empty(self): - orig = {} - res = utils._add_timestamp(orig) - assert res == {} - - def test_add_timestamp_none(self): - orig = { 'a': 1, 'b': 2, 'c': 3 } - exp = { 'a': (1, 0), 'b': (2, 0), 'c': (3, 0) } - res = utils._add_timestamp(orig) - assert res == exp - - def test_add_timestamp_mixed(self): - orig = { 'a': 1, 'b': (2, 1), 'c': 3 } - exp = { 'a': (1, 0), 'b': (2, 1), 'c': (3, 0) } - res = utils._add_timestamp(orig) - assert res == exp - - def test_add_timestamp_all(self): - orig = { 'a': (1, 0), 'b': (2, 1), 'c': (3, 0) } - res = utils._add_timestamp(orig) - assert res == orig - - def test_get_etag_empty(self): - tf = tempfile.NamedTemporaryFile() - hd = utils._get_etag(tf.name) - assert hd == hashlib.md5().hexdigest() - - def test_get_etag(self): - tf = tempfile.NamedTemporaryFile() - tf.file.write('123' * utils.CHUNK_SIZE) - tf.file.flush() - hd = utils._get_etag(tf.name) - tf.file.seek(0) - md5 = hashlib.md5() - while True: - chunk = tf.file.read(utils.CHUNK_SIZE) - if not chunk: - break - md5.update(chunk) - assert hd == md5.hexdigest() - - def test_get_object_metadata_dne(self): - md = utils.get_object_metadata("/tmp/doesNotEx1st") - assert md == {} - - def test_get_object_metadata_err(self): - tf = tempfile.NamedTemporaryFile() - try: - md = utils.get_object_metadata(os.path.join(tf.name,"doesNotEx1st")) - except OSError as e: - assert e.errno != errno.ENOENT - else: - self.fail("Expected exception") - - obj_keys = (utils.X_TIMESTAMP, utils.X_CONTENT_TYPE, utils.X_ETAG, - utils.X_CONTENT_LENGTH, utils.X_TYPE, utils.X_OBJECT_TYPE) - - def test_get_object_metadata_file(self): - tf = tempfile.NamedTemporaryFile() - tf.file.write('123'); tf.file.flush() - md = utils.get_object_metadata(tf.name) - for key in self.obj_keys: - assert key in md, "Expected key %s in %r" % (key, md) - assert md[utils.X_TYPE] == utils.OBJECT - assert md[utils.X_OBJECT_TYPE] == utils.FILE - assert md[utils.X_CONTENT_TYPE] == utils.FILE_TYPE - assert md[utils.X_CONTENT_LENGTH] == os.path.getsize(tf.name) - assert md[utils.X_TIMESTAMP] == normalize_timestamp(os.path.getctime(tf.name)) - assert md[utils.X_ETAG] == utils._get_etag(tf.name) - - def test_get_object_metadata_dir(self): - td = tempfile.mkdtemp() - try: - md = utils.get_object_metadata(td) - for key in self.obj_keys: - assert key in md, "Expected key %s in %r" % (key, md) - assert md[utils.X_TYPE] == utils.OBJECT - assert md[utils.X_OBJECT_TYPE] == utils.DIR - assert md[utils.X_CONTENT_TYPE] == utils.DIR_TYPE - assert md[utils.X_CONTENT_LENGTH] == 0 - assert md[utils.X_TIMESTAMP] == normalize_timestamp(os.path.getctime(td)) - assert md[utils.X_ETAG] == hashlib.md5().hexdigest() - finally: - os.rmdir(td) - - def test_create_object_metadata_file(self): - tf = tempfile.NamedTemporaryFile() - tf.file.write('4567'); tf.file.flush() - r_md = utils.create_object_metadata(tf.name) - - xkey = _xkey(tf.name, utils.METADATA_KEY) - assert len(_xattrs.keys()) == 1 - assert xkey in _xattrs - assert _xattr_op_cnt['get'] == 1 - assert _xattr_op_cnt['set'] == 1 - md = pickle.loads(_xattrs[xkey]) - assert r_md == md - - for key in self.obj_keys: - assert key in md, "Expected key %s in %r" % (key, md) - assert md[utils.X_TYPE] == utils.OBJECT - assert md[utils.X_OBJECT_TYPE] == utils.FILE - assert md[utils.X_CONTENT_TYPE] == utils.FILE_TYPE - assert md[utils.X_CONTENT_LENGTH] == os.path.getsize(tf.name) - assert md[utils.X_TIMESTAMP] == normalize_timestamp(os.path.getctime(tf.name)) - assert md[utils.X_ETAG] == utils._get_etag(tf.name) - - def test_create_object_metadata_dir(self): - td = tempfile.mkdtemp() - try: - r_md = utils.create_object_metadata(td) - - xkey = _xkey(td, utils.METADATA_KEY) - assert len(_xattrs.keys()) == 1 - assert xkey in _xattrs - assert _xattr_op_cnt['get'] == 1 - assert _xattr_op_cnt['set'] == 1 - md = pickle.loads(_xattrs[xkey]) - assert r_md == md - - for key in self.obj_keys: - assert key in md, "Expected key %s in %r" % (key, md) - assert md[utils.X_TYPE] == utils.OBJECT - assert md[utils.X_OBJECT_TYPE] == utils.DIR - assert md[utils.X_CONTENT_TYPE] == utils.DIR_TYPE - assert md[utils.X_CONTENT_LENGTH] == 0 - assert md[utils.X_TIMESTAMP] == normalize_timestamp(os.path.getctime(td)) - assert md[utils.X_ETAG] == hashlib.md5().hexdigest() - finally: - os.rmdir(td) - - def test_get_container_metadata(self): - def _mock_get_container_details(path, memcache=None): - o_list = [ 'a', 'b', 'c' ] - o_count = 3 - b_used = 47 - return o_list, o_count, b_used - orig_gcd = utils.get_container_details - utils.get_container_details = _mock_get_container_details - td = tempfile.mkdtemp() - try: - exp_md = { - utils.X_TYPE: (utils.CONTAINER, 0), - utils.X_TIMESTAMP: (normalize_timestamp(os.path.getctime(td)), 0), - utils.X_PUT_TIMESTAMP: (normalize_timestamp(os.path.getmtime(td)), 0), - utils.X_OBJECTS_COUNT: (3, 0), - utils.X_BYTES_USED: (47, 0), - } - md = utils.get_container_metadata(td) - assert md == exp_md - finally: - utils.get_container_details = orig_gcd - os.rmdir(td) - - def test_get_account_metadata(self): - def _mock_get_account_details(path, memcache=None): - c_list = [ '123', 'abc' ] - c_count = 2 - return c_list, c_count - orig_gad = utils.get_account_details - utils.get_account_details = _mock_get_account_details - td = tempfile.mkdtemp() - try: - exp_md = { - utils.X_TYPE: (utils.ACCOUNT, 0), - utils.X_TIMESTAMP: (normalize_timestamp(os.path.getctime(td)), 0), - utils.X_PUT_TIMESTAMP: (normalize_timestamp(os.path.getmtime(td)), 0), - utils.X_OBJECTS_COUNT: (0, 0), - utils.X_BYTES_USED: (0, 0), - utils.X_CONTAINER_COUNT: (2, 0), - } - md = utils.get_account_metadata(td) - assert md == exp_md - finally: - utils.get_account_details = orig_gad - os.rmdir(td) - - cont_keys = [utils.X_TYPE, utils.X_TIMESTAMP, utils.X_PUT_TIMESTAMP, - utils.X_OBJECTS_COUNT, utils.X_BYTES_USED] - - def test_create_container_metadata(self): - td = tempfile.mkdtemp() - try: - r_md = utils.create_container_metadata(td) - - xkey = _xkey(td, utils.METADATA_KEY) - assert len(_xattrs.keys()) == 1 - assert xkey in _xattrs - assert _xattr_op_cnt['get'] == 1 - assert _xattr_op_cnt['set'] == 1 - md = pickle.loads(_xattrs[xkey]) - assert r_md == md - - for key in self.cont_keys: - assert key in md, "Expected key %s in %r" % (key, md) - assert md[utils.X_TYPE] == (utils.CONTAINER, 0) - assert md[utils.X_TIMESTAMP] == (normalize_timestamp(os.path.getctime(td)), 0) - assert md[utils.X_PUT_TIMESTAMP] == (normalize_timestamp(os.path.getmtime(td)), 0) - assert md[utils.X_OBJECTS_COUNT] == (0, 0) - assert md[utils.X_BYTES_USED] == (0, 0) - finally: - os.rmdir(td) - - acct_keys = [val for val in cont_keys] - acct_keys.append(utils.X_CONTAINER_COUNT) - - def test_create_account_metadata(self): - td = tempfile.mkdtemp() - try: - r_md = utils.create_account_metadata(td) - - xkey = _xkey(td, utils.METADATA_KEY) - assert len(_xattrs.keys()) == 1 - assert xkey in _xattrs - assert _xattr_op_cnt['get'] == 1 - assert _xattr_op_cnt['set'] == 1 - md = pickle.loads(_xattrs[xkey]) - assert r_md == md - - for key in self.acct_keys: - assert key in md, "Expected key %s in %r" % (key, md) - assert md[utils.X_TYPE] == (utils.ACCOUNT, 0) - assert md[utils.X_TIMESTAMP] == (normalize_timestamp(os.path.getctime(td)), 0) - assert md[utils.X_PUT_TIMESTAMP] == (normalize_timestamp(os.path.getmtime(td)), 0) - assert md[utils.X_OBJECTS_COUNT] == (0, 0) - assert md[utils.X_BYTES_USED] == (0, 0) - assert md[utils.X_CONTAINER_COUNT] == (0, 0) - finally: - os.rmdir(td) - - def test_container_details_uncached(self): - the_path = "/tmp/bar" - def mock_get_container_details_from_fs(cont_path): - bu = 5 - oc = 1 - ol = ['foo',] - dl = [('a',100),] - return utils.ContainerDetails(bu, oc, ol, dl) - orig_gcdff = utils._get_container_details_from_fs - utils._get_container_details_from_fs = mock_get_container_details_from_fs - try: - retval = utils.get_container_details(the_path) - cd = mock_get_container_details_from_fs(the_path) - assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) - finally: - utils._get_container_details_from_fs = orig_gcdff - - def test_container_details_cached_hit(self): - mc = SimMemcache() - the_path = "/tmp/bar" - def mock_get_container_details_from_fs(cont_path, bu_p=5): - bu = bu_p - oc = 1 - ol = ['foo',] - dl = [('a',100),] - return utils.ContainerDetails(bu, oc, ol, dl) - def mock_do_stat(path): - class MockStat(object): - def __init__(self, mtime): - self.st_mtime = mtime - return MockStat(100) - cd = mock_get_container_details_from_fs(the_path, bu_p=6) - mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path, cd) - orig_gcdff = utils._get_container_details_from_fs - utils._get_container_details_from_fs = mock_get_container_details_from_fs - orig_ds = utils.do_stat - utils.do_stat = mock_do_stat - try: - retval = utils.get_container_details(the_path, memcache=mc) - # If it did not properly use memcache, the default mocked version - # of get details from fs would return 5 bytes used instead of the - # 6 we specified above. - cd = mock_get_container_details_from_fs(the_path, bu_p=6) - assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) - finally: - utils._get_container_details_from_fs = orig_gcdff - utils.do_stat = orig_ds - - def test_container_details_cached_miss_key(self): - mc = SimMemcache() - the_path = "/tmp/bar" - def mock_get_container_details_from_fs(cont_path, bu_p=5): - bu = bu_p - oc = 1 - ol = ['foo',] - dl = [('a',100),] - return utils.ContainerDetails(bu, oc, ol, dl) - def mock_do_stat(path): - # Be sure we don't miss due to mtimes not matching - self.fail("do_stat should not have been called") - cd = mock_get_container_details_from_fs(the_path + "u", bu_p=6) - mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path + "u", cd) - orig_gcdff = utils._get_container_details_from_fs - utils._get_container_details_from_fs = mock_get_container_details_from_fs - orig_ds = utils.do_stat - utils.do_stat = mock_do_stat - try: - retval = utils.get_container_details(the_path, memcache=mc) - cd = mock_get_container_details_from_fs(the_path) - assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) - mkey = utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path - assert mkey in mc._d - finally: - utils._get_container_details_from_fs = orig_gcdff - utils.do_stat = orig_ds - - def test_container_details_cached_miss_dir_list(self): - mc = SimMemcache() - the_path = "/tmp/bar" - def mock_get_container_details_from_fs(cont_path, bu_p=5): - bu = bu_p - oc = 1 - ol = ['foo',] - dl = [] - return utils.ContainerDetails(bu, oc, ol, dl) - def mock_do_stat(path): - # Be sure we don't miss due to mtimes not matching - self.fail("do_stat should not have been called") - cd = mock_get_container_details_from_fs(the_path, bu_p=6) - mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path, cd) - orig_gcdff = utils._get_container_details_from_fs - utils._get_container_details_from_fs = mock_get_container_details_from_fs - orig_ds = utils.do_stat - utils.do_stat = mock_do_stat - try: - retval = utils.get_container_details(the_path, memcache=mc) - cd = mock_get_container_details_from_fs(the_path) - assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) - mkey = utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path - assert mkey in mc._d - assert 5 == mc._d[mkey].bytes_used - finally: - utils._get_container_details_from_fs = orig_gcdff - utils.do_stat = orig_ds - - def test_container_details_cached_miss_mtime(self): - mc = SimMemcache() - the_path = "/tmp/bar" - def mock_get_container_details_from_fs(cont_path, bu_p=5): - bu = bu_p - oc = 1 - ol = ['foo',] - dl = [('a',100),] - return utils.ContainerDetails(bu, oc, ol, dl) - def mock_do_stat(path): - # Be sure we miss due to mtimes not matching - class MockStat(object): - def __init__(self, mtime): - self.st_mtime = mtime - return MockStat(200) - cd = mock_get_container_details_from_fs(the_path, bu_p=6) - mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path, cd) - orig_gcdff = utils._get_container_details_from_fs - utils._get_container_details_from_fs = mock_get_container_details_from_fs - orig_ds = utils.do_stat - utils.do_stat = mock_do_stat - try: - retval = utils.get_container_details(the_path, memcache=mc) - cd = mock_get_container_details_from_fs(the_path) - assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) - mkey = utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path - assert mkey in mc._d - assert 5 == mc._d[mkey].bytes_used - finally: - utils._get_container_details_from_fs = orig_gcdff - utils.do_stat = orig_ds - - def test_account_details_uncached(self): - the_path = "/tmp/bar" - def mock_get_account_details_from_fs(acc_path, acc_stats): - mt = 100 - cc = 2 - cl = ['a', 'b'] - return utils.AccountDetails(mt, cc, cl) - orig_gcdff = utils._get_account_details_from_fs - utils._get_account_details_from_fs = mock_get_account_details_from_fs - try: - retval = utils.get_account_details(the_path) - ad = mock_get_account_details_from_fs(the_path, None) - assert retval == (ad.container_list, ad.container_count) - finally: - utils._get_account_details_from_fs = orig_gcdff - - def test_account_details_cached_hit(self): - mc = SimMemcache() - the_path = "/tmp/bar" - def mock_get_account_details_from_fs(acc_path, acc_stats): - mt = 100 - cc = 2 - cl = ['a', 'b'] - return utils.AccountDetails(mt, cc, cl) - def mock_do_stat(path): - class MockStat(object): - def __init__(self, mtime): - self.st_mtime = mtime - return MockStat(100) - ad = mock_get_account_details_from_fs(the_path, None) - ad.container_list = ['x', 'y'] - mc.set(utils.MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX + the_path, ad) - orig_gcdff = utils._get_account_details_from_fs - orig_ds = utils.do_stat - utils._get_account_details_from_fs = mock_get_account_details_from_fs - utils.do_stat = mock_do_stat - try: - retval = utils.get_account_details(the_path, memcache=mc) - assert retval == (ad.container_list, ad.container_count) - wrong_ad = mock_get_account_details_from_fs(the_path, None) - assert wrong_ad != ad - finally: - utils._get_account_details_from_fs = orig_gcdff - utils.do_stat = orig_ds - - def test_account_details_cached_miss(self): - mc = SimMemcache() - the_path = "/tmp/bar" - def mock_get_account_details_from_fs(acc_path, acc_stats): - mt = 100 - cc = 2 - cl = ['a', 'b'] - return utils.AccountDetails(mt, cc, cl) - def mock_do_stat(path): - class MockStat(object): - def __init__(self, mtime): - self.st_mtime = mtime - return MockStat(100) - ad = mock_get_account_details_from_fs(the_path, None) - ad.container_list = ['x', 'y'] - mc.set(utils.MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX + the_path + 'u', ad) - orig_gcdff = utils._get_account_details_from_fs - orig_ds = utils.do_stat - utils._get_account_details_from_fs = mock_get_account_details_from_fs - utils.do_stat = mock_do_stat - try: - retval = utils.get_account_details(the_path, memcache=mc) - correct_ad = mock_get_account_details_from_fs(the_path, None) - assert retval == (correct_ad.container_list, correct_ad.container_count) - assert correct_ad != ad - finally: - utils._get_account_details_from_fs = orig_gcdff - utils.do_stat = orig_ds - - def test_account_details_cached_miss_mtime(self): - mc = SimMemcache() - the_path = "/tmp/bar" - def mock_get_account_details_from_fs(acc_path, acc_stats): - mt = 100 - cc = 2 - cl = ['a', 'b'] - return utils.AccountDetails(mt, cc, cl) - def mock_do_stat(path): - class MockStat(object): - def __init__(self, mtime): - self.st_mtime = mtime - return MockStat(100) - ad = mock_get_account_details_from_fs(the_path, None) - ad.container_list = ['x', 'y'] - ad.mtime = 200 - mc.set(utils.MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX + the_path, ad) - orig_gcdff = utils._get_account_details_from_fs - orig_ds = utils.do_stat - utils._get_account_details_from_fs = mock_get_account_details_from_fs - utils.do_stat = mock_do_stat - try: - retval = utils.get_account_details(the_path, memcache=mc) - correct_ad = mock_get_account_details_from_fs(the_path, None) - assert retval == (correct_ad.container_list, correct_ad.container_count) - assert correct_ad != ad - finally: - utils._get_account_details_from_fs = orig_gcdff - utils.do_stat = orig_ds - - def test_get_account_details_from_fs(self): - orig_cwd = os.getcwd() - td = tempfile.mkdtemp() - try: - tf = tarfile.open("common/data/account_tree.tar.bz2", "r:bz2") - os.chdir(td) - tf.extractall() - - ad = utils._get_account_details_from_fs(td, None) - assert ad.mtime == os.path.getmtime(td) - assert ad.container_count == 3 - assert set(ad.container_list) == set(['c1', 'c2', 'c3']) - finally: - os.chdir(orig_cwd) - shutil.rmtree(td) - - def test_get_container_details_from_fs_notadir(self): - tf = tempfile.NamedTemporaryFile() - cd = utils._get_container_details_from_fs(tf.name) - assert cd.bytes_used == 0 - assert cd.object_count == 0 - assert cd.obj_list == [] - assert cd.dir_list == [] - - def test_get_container_details_from_fs(self): - orig_cwd = os.getcwd() - td = tempfile.mkdtemp() - try: - tf = tarfile.open("common/data/container_tree.tar.bz2", "r:bz2") - os.chdir(td) - tf.extractall() - - cd = utils._get_container_details_from_fs(td) - assert cd.bytes_used == 0, repr(cd.bytes_used) - assert cd.object_count == 8, repr(cd.object_count) - assert set(cd.obj_list) == set(['file1', 'file3', 'file2', - 'dir3', 'dir1', 'dir2', - 'dir1/file1', 'dir1/file2' - ]), repr(cd.obj_list) - - full_dir1 = os.path.join(td, 'dir1') - full_dir2 = os.path.join(td, 'dir2') - full_dir3 = os.path.join(td, 'dir3') - exp_dir_dict = { td: os.path.getmtime(td), - full_dir1: os.path.getmtime(full_dir1), - full_dir2: os.path.getmtime(full_dir2), - full_dir3: os.path.getmtime(full_dir3), - } - for d,m in cd.dir_list: - assert d in exp_dir_dict - assert exp_dir_dict[d] == m - finally: - os.chdir(orig_cwd) - shutil.rmtree(td) - - - def test_get_container_details_from_fs_do_getsize_true(self): - orig_cwd = os.getcwd() - td = tempfile.mkdtemp() - try: - tf = tarfile.open("common/data/container_tree.tar.bz2", "r:bz2") - os.chdir(td) - tf.extractall() - - __do_getsize = Glusterfs._do_getsize - Glusterfs._do_getsize = True - - cd = utils._get_container_details_from_fs(td) - assert cd.bytes_used == 30, repr(cd.bytes_used) - assert cd.object_count == 8, repr(cd.object_count) - assert set(cd.obj_list) == set(['file1', 'file3', 'file2', - 'dir3', 'dir1', 'dir2', - 'dir1/file1', 'dir1/file2' - ]), repr(cd.obj_list) - - full_dir1 = os.path.join(td, 'dir1') - full_dir2 = os.path.join(td, 'dir2') - full_dir3 = os.path.join(td, 'dir3') - exp_dir_dict = { td: os.path.getmtime(td), - full_dir1: os.path.getmtime(full_dir1), - full_dir2: os.path.getmtime(full_dir2), - full_dir3: os.path.getmtime(full_dir3), - } - for d,m in cd.dir_list: - assert d in exp_dir_dict - assert exp_dir_dict[d] == m - finally: - Glusterfs._do_getsize = __do_getsize - os.chdir(orig_cwd) - shutil.rmtree(td) - - def test_get_account_details_from_fs_notadir_w_stats(self): - tf = tempfile.NamedTemporaryFile() - ad = utils._get_account_details_from_fs(tf.name, os.stat(tf.name)) - assert ad.mtime == os.path.getmtime(tf.name) - assert ad.container_count == 0 - assert ad.container_list == [] - - def test_get_account_details_from_fs_notadir(self): - tf = tempfile.NamedTemporaryFile() - ad = utils._get_account_details_from_fs(tf.name, None) - assert ad.mtime == os.path.getmtime(tf.name) - assert ad.container_count == 0 - assert ad.container_list == [] - - def test_write_pickle(self): - td = tempfile.mkdtemp() - try: - fpp = os.path.join(td, 'pp') - utils.write_pickle('pickled peppers', fpp) - with open(fpp, "rb") as f: - contents = f.read() - s = pickle.loads(contents) - assert s == 'pickled peppers', repr(s) - finally: - shutil.rmtree(td) - - def test_write_pickle_ignore_tmp(self): - tf = tempfile.NamedTemporaryFile() - td = tempfile.mkdtemp() - try: - fpp = os.path.join(td, 'pp') - # Also test an explicity pickle protocol - utils.write_pickle('pickled peppers', fpp, tmp=tf.name, pickle_protocol=2) - with open(fpp, "rb") as f: - contents = f.read() - s = pickle.loads(contents) - assert s == 'pickled peppers', repr(s) - with open(tf.name, "rb") as f: - contents = f.read() - assert contents == '' - finally: - shutil.rmtree(td) - - def test_check_user_xattr_bad_path(self): - assert False == utils.check_user_xattr("/tmp/foo/bar/check/user/xattr") - - def test_check_user_xattr_bad_set(self): - td = tempfile.mkdtemp() - xkey = _xkey(td, 'user.test.key1') - _xattr_set_err[xkey] = errno.EOPNOTSUPP - try: - assert False == utils.check_user_xattr(td) - except IOError: - pass - else: - self.fail("Expected IOError") - finally: - shutil.rmtree(td) - - def test_check_user_xattr_bad_remove(self): - td = tempfile.mkdtemp() - xkey = _xkey(td, 'user.test.key1') - _xattr_rem_err[xkey] = errno.EOPNOTSUPP - try: - utils.check_user_xattr(td) - except IOError: - self.fail("Unexpected IOError") - finally: - shutil.rmtree(td) - - def test_check_user_xattr(self): - td = tempfile.mkdtemp() - try: - assert utils.check_user_xattr(td) - finally: - shutil.rmtree(td) - - def test_validate_container_empty(self): - ret = utils.validate_container({}) - assert ret == False - - def test_validate_container_missing_keys(self): - ret = utils.validate_container({ 'foo': 'bar' }) - assert ret == False - - def test_validate_container_bad_type(self): - md = { utils.X_TYPE: ('bad', 0), - utils.X_TIMESTAMP: ('na', 0), - utils.X_PUT_TIMESTAMP: ('na', 0), - utils.X_OBJECTS_COUNT: ('na', 0), - utils.X_BYTES_USED: ('na', 0) } - ret = utils.validate_container(md) - assert ret == False - - def test_validate_container_good_type(self): - md = { utils.X_TYPE: (utils.CONTAINER, 0), - utils.X_TIMESTAMP: ('na', 0), - utils.X_PUT_TIMESTAMP: ('na', 0), - utils.X_OBJECTS_COUNT: ('na', 0), - utils.X_BYTES_USED: ('na', 0) } - ret = utils.validate_container(md) - assert ret - - def test_validate_account_empty(self): - ret = utils.validate_account({}) - assert ret == False - - def test_validate_account_missing_keys(self): - ret = utils.validate_account({ 'foo': 'bar' }) - assert ret == False - - def test_validate_account_bad_type(self): - md = { utils.X_TYPE: ('bad', 0), - utils.X_TIMESTAMP: ('na', 0), - utils.X_PUT_TIMESTAMP: ('na', 0), - utils.X_OBJECTS_COUNT: ('na', 0), - utils.X_BYTES_USED: ('na', 0), - utils.X_CONTAINER_COUNT: ('na', 0) } - ret = utils.validate_account(md) - assert ret == False - - def test_validate_account_good_type(self): - md = { utils.X_TYPE: (utils.ACCOUNT, 0), - utils.X_TIMESTAMP: ('na', 0), - utils.X_PUT_TIMESTAMP: ('na', 0), - utils.X_OBJECTS_COUNT: ('na', 0), - utils.X_BYTES_USED: ('na', 0), - utils.X_CONTAINER_COUNT: ('na', 0) } - ret = utils.validate_account(md) - assert ret - - def test_validate_object_empty(self): - ret = utils.validate_object({}) - assert ret == False - - def test_validate_object_missing_keys(self): - ret = utils.validate_object({ 'foo': 'bar' }) - assert ret == False - - def test_validate_object_bad_type(self): - md = { utils.X_TIMESTAMP: 'na', - utils.X_CONTENT_TYPE: 'na', - utils.X_ETAG: 'bad', - utils.X_CONTENT_LENGTH: 'na', - utils.X_TYPE: 'bad', - utils.X_OBJECT_TYPE: 'na' } - ret = utils.validate_object(md) - assert ret == False - - def test_validate_object_good_type(self): - md = { utils.X_TIMESTAMP: 'na', - utils.X_CONTENT_TYPE: 'na', - utils.X_ETAG: 'bad', - utils.X_CONTENT_LENGTH: 'na', - utils.X_TYPE: utils.OBJECT, - utils.X_OBJECT_TYPE: 'na' } - ret = utils.validate_object(md) - assert ret - - def test_is_marker_empty(self): - assert False == utils.is_marker(None) - - def test_is_marker_missing(self): - assert False == utils.is_marker( { 'foo': 'bar' } ) - - def test_is_marker_not_marker(self): - md = { utils.X_OBJECT_TYPE: utils.DIR } - assert False == utils.is_marker(md) - - def test_is_marker(self): - md = { utils.X_OBJECT_TYPE: utils.MARKER_DIR } - assert utils.is_marker(md) diff --git a/ufo/tools/test-requires b/ufo/tools/test-requires deleted file mode 100644 index ef8bf14bfb0..00000000000 --- a/ufo/tools/test-requires +++ /dev/null @@ -1,6 +0,0 @@ -coverage -nose -nosexcover -openstack.nose_plugin -nosehtmloutput -mock>=0.8.0 diff --git a/ufo/tox.ini b/ufo/tox.ini deleted file mode 100644 index 792bb417f36..00000000000 --- a/ufo/tox.ini +++ /dev/null @@ -1,25 +0,0 @@ -[tox] -envlist = py26,py27 - -[testenv] -setenv = VIRTUAL_ENV={envdir} - NOSE_WITH_OPENSTACK=1 - NOSE_OPENSTACK_COLOR=1 - NOSE_OPENSTACK_RED=0.05 - NOSE_OPENSTACK_YELLOW=0.025 - NOSE_OPENSTACK_SHOW_ELAPSED=1 - NOSE_OPENSTACK_STDOUT=1 -deps = - https://launchpad.net/swift/grizzly/1.8.0/+download/swift-1.8.0.tar.gz - -r{toxinidir}/tools/test-requires -changedir = {toxinidir}/test/unit -commands = nosetests -v --exe --with-coverage --cover-package gluster --cover-erase {posargs} - -[tox:jenkins] -downloadcache = ~/cache/pip - -[testenv:cover] -setenv = NOSE_WITH_COVERAGE=1 - -[testenv:venv] -commands = {posargs} diff --git a/ufo/unittests.sh b/ufo/unittests.sh deleted file mode 100755 index a02be77966f..00000000000 --- a/ufo/unittests.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -cd $(dirname $0)/test/unit -nosetests --exe --with-coverage --cover-package gluster --cover-erase $@ -saved_status=$? -rm -f .coverage -exit $saved_status |