From 1129698a490ac9266b1f613fababdf524e11f887 Mon Sep 17 00:00:00 2001 From: Peter Portante Date: Thu, 8 Nov 2012 01:16:56 -0500 Subject: object-storage: final changes to remove swift diff Final set of changes to remove the diffs carried to make UFO work with OpenStack Swift. The code is now a complete layering on top of OpenStack Swift where we either "monkey patch" or subclass as necessary. See BZ 870589 (https://bugzilla.redhat.com/show_bug.cgi?id=870589). There are a lot of changes here due for the most part to rearranging the directory hierarchy to have create a proper python module hierarchy under the "gluster" namespace. Plugin references have been removed. The differences that used to be in the swift.diff file are now replaced with server implementations for account, container, object, and proxy that subclass the swift versions. Additionally, the plugins/conf directory has been moved to the "etc" directory, and the plugins/bin directory promoted a level. Unit tests pass. A new setup.py file is provided so that the install process can use it for creating all the necessary python install infrastructure (eggs and paste support). A new RPM spec file is provided which to properly install the new code, and the sample configuration files have been modified to reference the new python egg. Change-Id: I4316c1b66dca80f847fe9b0d583174689c175599 BUG: 870589 Signed-off-by: Peter Portante Reviewed-on: http://review.gluster.org/4180 Reviewed-by: Kaleb KEITHLEY Reviewed-by: Mohammed Junaid Tested-by: Kaleb KEITHLEY --- swift/1.4.8/.unittests | 2 +- swift/1.4.8/README | 12 +- swift/1.4.8/bin/gluster-swift-gen-builders | 19 + swift/1.4.8/etc/account-server/1.conf-gluster | 19 + swift/1.4.8/etc/container-server/1.conf-gluster | 21 + swift/1.4.8/etc/fs.conf-gluster | 13 + swift/1.4.8/etc/object-server/1.conf-gluster | 19 + swift/1.4.8/etc/proxy-server.conf-gluster | 33 + swift/1.4.8/etc/swift.conf-gluster | 91 +++ swift/1.4.8/gluster-swift-plugin.spec | 74 -- swift/1.4.8/gluster-swift-ufo.spec | 87 +++ swift/1.4.8/gluster-swift.spec | 396 ---------- swift/1.4.8/gluster/__init__.py | 0 swift/1.4.8/gluster/swift/__init__.py | 18 + swift/1.4.8/gluster/swift/account/__init__.py | 0 swift/1.4.8/gluster/swift/account/server.py | 45 ++ swift/1.4.8/gluster/swift/common/DiskDir.py | 500 +++++++++++++ swift/1.4.8/gluster/swift/common/DiskFile.py | 323 ++++++++ swift/1.4.8/gluster/swift/common/Glusterfs.py | 126 ++++ swift/1.4.8/gluster/swift/common/__init__.py | 0 swift/1.4.8/gluster/swift/common/constraints.py | 82 +++ swift/1.4.8/gluster/swift/common/fs_utils.py | 156 ++++ .../gluster/swift/common/middleware/__init__.py | 0 .../gluster/swift/common/middleware/gluster.py | 40 + swift/1.4.8/gluster/swift/common/utils.py | 457 ++++++++++++ swift/1.4.8/gluster/swift/container/__init__.py | 0 swift/1.4.8/gluster/swift/container/server.py | 46 ++ swift/1.4.8/gluster/swift/obj/__init__.py | 0 swift/1.4.8/gluster/swift/obj/server.py | 33 + swift/1.4.8/gluster/swift/proxy/__init__.py | 0 swift/1.4.8/gluster/swift/proxy/server.py | 27 + swift/1.4.8/plugins/DiskDir.py | 499 ------------- swift/1.4.8/plugins/DiskFile.py | 323 -------- swift/1.4.8/plugins/Glusterfs.py | 131 ---- swift/1.4.8/plugins/__init__.py | 0 swift/1.4.8/plugins/bin/gluster-swift-gen-builders | 19 - .../plugins/conf/account-server/1.conf-gluster | 19 - .../plugins/conf/container-server/1.conf-gluster | 21 - swift/1.4.8/plugins/conf/fs.conf-gluster | 8 - .../plugins/conf/object-server/1.conf-gluster | 19 - swift/1.4.8/plugins/conf/proxy-server.conf-gluster | 36 - swift/1.4.8/plugins/conf/swift.conf-gluster | 92 --- swift/1.4.8/plugins/constraints.py | 82 --- swift/1.4.8/plugins/fs_utils.py | 156 ---- swift/1.4.8/plugins/middleware/__init__.py | 0 swift/1.4.8/plugins/middleware/gluster.py | 40 - swift/1.4.8/plugins/utils.py | 475 ------------ swift/1.4.8/setup.py | 57 ++ swift/1.4.8/swift.diff | 174 ----- swift/1.4.8/test/unit/common/__init__.py | 0 .../test/unit/common/data/account_tree.tar.bz2 | Bin 0 -> 228 bytes .../test/unit/common/data/container_tree.tar.bz2 | Bin 0 -> 282 bytes swift/1.4.8/test/unit/common/test_utils.py | 818 +++++++++++++++++++++ swift/1.4.8/test/unit/plugins/__init__.py | 0 .../test/unit/plugins/data/account_tree.tar.bz2 | Bin 228 -> 0 bytes .../test/unit/plugins/data/container_tree.tar.bz2 | Bin 282 -> 0 bytes swift/1.4.8/test/unit/plugins/test_utils.py | 818 --------------------- 57 files changed, 3034 insertions(+), 3392 deletions(-) create mode 100755 swift/1.4.8/bin/gluster-swift-gen-builders create mode 100644 swift/1.4.8/etc/account-server/1.conf-gluster create mode 100644 swift/1.4.8/etc/container-server/1.conf-gluster create mode 100644 swift/1.4.8/etc/fs.conf-gluster create mode 100644 swift/1.4.8/etc/object-server/1.conf-gluster create mode 100644 swift/1.4.8/etc/proxy-server.conf-gluster create mode 100644 swift/1.4.8/etc/swift.conf-gluster delete mode 100644 swift/1.4.8/gluster-swift-plugin.spec create mode 100644 swift/1.4.8/gluster-swift-ufo.spec delete mode 100644 swift/1.4.8/gluster-swift.spec create mode 100644 swift/1.4.8/gluster/__init__.py create mode 100644 swift/1.4.8/gluster/swift/__init__.py create mode 100644 swift/1.4.8/gluster/swift/account/__init__.py create mode 100644 swift/1.4.8/gluster/swift/account/server.py create mode 100644 swift/1.4.8/gluster/swift/common/DiskDir.py create mode 100644 swift/1.4.8/gluster/swift/common/DiskFile.py create mode 100644 swift/1.4.8/gluster/swift/common/Glusterfs.py create mode 100644 swift/1.4.8/gluster/swift/common/__init__.py create mode 100644 swift/1.4.8/gluster/swift/common/constraints.py create mode 100644 swift/1.4.8/gluster/swift/common/fs_utils.py create mode 100644 swift/1.4.8/gluster/swift/common/middleware/__init__.py create mode 100644 swift/1.4.8/gluster/swift/common/middleware/gluster.py create mode 100644 swift/1.4.8/gluster/swift/common/utils.py create mode 100644 swift/1.4.8/gluster/swift/container/__init__.py create mode 100644 swift/1.4.8/gluster/swift/container/server.py create mode 100644 swift/1.4.8/gluster/swift/obj/__init__.py create mode 100644 swift/1.4.8/gluster/swift/obj/server.py create mode 100644 swift/1.4.8/gluster/swift/proxy/__init__.py create mode 100644 swift/1.4.8/gluster/swift/proxy/server.py delete mode 100644 swift/1.4.8/plugins/DiskDir.py delete mode 100644 swift/1.4.8/plugins/DiskFile.py delete mode 100644 swift/1.4.8/plugins/Glusterfs.py delete mode 100644 swift/1.4.8/plugins/__init__.py delete mode 100755 swift/1.4.8/plugins/bin/gluster-swift-gen-builders delete mode 100644 swift/1.4.8/plugins/conf/account-server/1.conf-gluster delete mode 100644 swift/1.4.8/plugins/conf/container-server/1.conf-gluster delete mode 100644 swift/1.4.8/plugins/conf/fs.conf-gluster delete mode 100644 swift/1.4.8/plugins/conf/object-server/1.conf-gluster delete mode 100644 swift/1.4.8/plugins/conf/proxy-server.conf-gluster delete mode 100644 swift/1.4.8/plugins/conf/swift.conf-gluster delete mode 100644 swift/1.4.8/plugins/constraints.py delete mode 100644 swift/1.4.8/plugins/fs_utils.py delete mode 100644 swift/1.4.8/plugins/middleware/__init__.py delete mode 100644 swift/1.4.8/plugins/middleware/gluster.py delete mode 100644 swift/1.4.8/plugins/utils.py create mode 100644 swift/1.4.8/setup.py delete mode 100644 swift/1.4.8/swift.diff create mode 100644 swift/1.4.8/test/unit/common/__init__.py create mode 100644 swift/1.4.8/test/unit/common/data/account_tree.tar.bz2 create mode 100644 swift/1.4.8/test/unit/common/data/container_tree.tar.bz2 create mode 100644 swift/1.4.8/test/unit/common/test_utils.py delete mode 100644 swift/1.4.8/test/unit/plugins/__init__.py delete mode 100644 swift/1.4.8/test/unit/plugins/data/account_tree.tar.bz2 delete mode 100644 swift/1.4.8/test/unit/plugins/data/container_tree.tar.bz2 delete mode 100644 swift/1.4.8/test/unit/plugins/test_utils.py (limited to 'swift') diff --git a/swift/1.4.8/.unittests b/swift/1.4.8/.unittests index 113e5dd1f..2cee10be5 100755 --- a/swift/1.4.8/.unittests +++ b/swift/1.4.8/.unittests @@ -1,6 +1,6 @@ #!/bin/bash cd test/unit -nosetests --exe --with-coverage --cover-package plugins --cover-erase $@ +nosetests --exe --with-coverage --cover-package gluster --cover-erase $@ rm -f .coverage cd - diff --git a/swift/1.4.8/README b/swift/1.4.8/README index 18435e08a..9efd918b5 100644 --- a/swift/1.4.8/README +++ b/swift/1.4.8/README @@ -3,20 +3,14 @@ via gluster-native/nfs mount to be accessed as containers and objects. It is a plugin for OpenStack Swift project. Install -* Clone the swift repo from git://github.com/openstack/swift.git -* Apply the swift.diff present in glusterfs.git/swift/1.4.8 to the swift repo. -* Create a directory named "plugins" under swift.git/swift directory. -* Copy the contents of glusterfs.git/swift/1.4.8/plugins/ under swift.git/swift/ - except the conf directory. -* Copy the contents of glusterfs.git/swift/1.4.8/plugins/conf under /etc/swift/. -* Run python setup.py install + * TBD Once this is done, you can access the GlusterFS volumes as Swift accounts. Add the Volume names with the user-name and its corresponding password to the /etc/swift/proxy-server.conf (follow the syntax used in the sample conf file). -Command to start the servers +Command to start the servers (TBD) swift-init main start -Command to stop the servers +Command to stop the servers (TBD) swift-init main stop diff --git a/swift/1.4.8/bin/gluster-swift-gen-builders b/swift/1.4.8/bin/gluster-swift-gen-builders new file mode 100755 index 000000000..b89cd15fb --- /dev/null +++ b/swift/1.4.8/bin/gluster-swift-gen-builders @@ -0,0 +1,19 @@ +#!/bin/bash + +function create { + swift-ring-builder $1 create 0 1 1 + swift-ring-builder $1 add z1-127.0.0.1:$2/$3_ 100.0 + swift-ring-builder $1 rebalance + swift-ring-builder $1 +} + +if [ "$1x" = "x" ]; then + echo "Please specify the gluster volume name to use." + exit 1 +fi + +# Note that these port numbers must match the configured values for the +# various servers in their configuration files. +create account.builder 6012 $1 +create container.builder 6011 $1 +create object.builder 6010 $1 diff --git a/swift/1.4.8/etc/account-server/1.conf-gluster b/swift/1.4.8/etc/account-server/1.conf-gluster new file mode 100644 index 000000000..da8f31726 --- /dev/null +++ b/swift/1.4.8/etc/account-server/1.conf-gluster @@ -0,0 +1,19 @@ +[DEFAULT] +devices = /mnt/gluster-object +mount_check = true +bind_port = 6012 +user = root +log_facility = LOG_LOCAL2 + +[pipeline:main] +pipeline = account-server + +[app:account-server] +use = egg:gluster_swift_ufo#account + +[account-replicator] +vm_test_mode = yes + +[account-auditor] + +[account-reaper] diff --git a/swift/1.4.8/etc/container-server/1.conf-gluster b/swift/1.4.8/etc/container-server/1.conf-gluster new file mode 100644 index 000000000..acad62135 --- /dev/null +++ b/swift/1.4.8/etc/container-server/1.conf-gluster @@ -0,0 +1,21 @@ +[DEFAULT] +devices = /mnt/gluster-object +mount_check = true +bind_port = 6011 +user = root +log_facility = LOG_LOCAL2 + +[pipeline:main] +pipeline = container-server + +[app:container-server] +use = egg:gluster_swift_ufo#container + +[container-replicator] +vm_test_mode = yes + +[container-updater] + +[container-auditor] + +[container-sync] diff --git a/swift/1.4.8/etc/fs.conf-gluster b/swift/1.4.8/etc/fs.conf-gluster new file mode 100644 index 000000000..bbbdc2bd1 --- /dev/null +++ b/swift/1.4.8/etc/fs.conf-gluster @@ -0,0 +1,13 @@ +[DEFAULT] +# IP address of a GlusterFS volume server member. By default, we assume the +# local host. +mount_ip = localhost + +# The GlusterFS server need not be local, a remote server can also be used +# by setting "remote_cluster = yes". +remote_cluster = no + +# By default it is assumed the Gluster volumes can be accessed using other +# methods besides UFO (not object only), which disables a caching +# optimizations in order to keep in sync with file system changes. +object_only = no diff --git a/swift/1.4.8/etc/object-server/1.conf-gluster b/swift/1.4.8/etc/object-server/1.conf-gluster new file mode 100644 index 000000000..fe157a9b5 --- /dev/null +++ b/swift/1.4.8/etc/object-server/1.conf-gluster @@ -0,0 +1,19 @@ +[DEFAULT] +devices = /mnt/gluster-object +mount_check = true +bind_port = 6010 +user = root +log_facility = LOG_LOCAL2 + +[pipeline:main] +pipeline = object-server + +[app:object-server] +use = egg:gluster_swift_ufo#object + +[object-replicator] +vm_test_mode = yes + +[object-updater] + +[object-auditor] diff --git a/swift/1.4.8/etc/proxy-server.conf-gluster b/swift/1.4.8/etc/proxy-server.conf-gluster new file mode 100644 index 000000000..30eb745bf --- /dev/null +++ b/swift/1.4.8/etc/proxy-server.conf-gluster @@ -0,0 +1,33 @@ +[DEFAULT] +bind_port = 8080 +user = root +log_facility = LOG_LOCAL1 + +[pipeline:main] +pipeline = healthcheck cache tempauth proxy-server + +[app:proxy-server] +use = egg:gluster_swift_ufo#proxy +allow_account_management = true +account_autocreate = true + +[filter:tempauth] +use = egg:swift#tempauth +# Here you need to add users explicitly. See the OpenStack Swift Deployment +# Guide for more information. The user and user64 directives take the +# following form: +# user__ = [group] [group] [...] [storage_url] +# user64__ = [group] [group] [...] [storage_url] +# Where you use user64 for accounts and/or usernames that include underscores. +# +# NOTE (and WARNING): The account name must match the device name specified +# when generating the account, container, and object build rings. +# +# E.g. +# user_ufo0_admin = abc123 .admin + +[filter:healthcheck] +use = egg:swift#healthcheck + +[filter:cache] +use = egg:swift#memcache diff --git a/swift/1.4.8/etc/swift.conf-gluster b/swift/1.4.8/etc/swift.conf-gluster new file mode 100644 index 000000000..25c3ca157 --- /dev/null +++ b/swift/1.4.8/etc/swift.conf-gluster @@ -0,0 +1,91 @@ +[DEFAULT] + + +[swift-hash] +# random unique string that can never change (DO NOT LOSE) +swift_hash_path_suffix = gluster + + +# The swift-constraints section sets the basic constraints on data +# saved in the swift cluster. + +[swift-constraints] + +# max_file_size is the largest "normal" object that can be saved in +# the cluster. This is also the limit on the size of each segment of +# a "large" object when using the large object manifest support. +# This value is set in bytes. Setting it to lower than 1MiB will cause +# some tests to fail. It is STRONGLY recommended to leave this value at +# the default (5 * 2**30 + 2). + +# FIXME: Really? Gluster can handle a 2^64 sized file? And can the fronting +# web service handle such a size? I think with UFO, we need to keep with the +# default size from Swift and encourage users to research what size their web +# services infrastructure can handle. + +max_file_size = 18446744073709551616 + + +# max_meta_name_length is the max number of bytes in the utf8 encoding +# of the name portion of a metadata header. + +#max_meta_name_length = 128 + + +# max_meta_value_length is the max number of bytes in the utf8 encoding +# of a metadata value + +#max_meta_value_length = 256 + + +# max_meta_count is the max number of metadata keys that can be stored +# on a single account, container, or object + +#max_meta_count = 90 + + +# max_meta_overall_size is the max number of bytes in the utf8 encoding +# of the metadata (keys + values) + +#max_meta_overall_size = 4096 + + +# max_object_name_length is the max number of bytes in the utf8 encoding of an +# object name: Gluster FS can handle much longer file names, but the length +# between the slashes of the URL is handled below. Remember that most web +# clients can't handle anything greater than 2048, and those that do are +# rather clumsy. + +max_object_name_length = 2048 + +# max_object_name_component_length (GlusterFS) is the max number of bytes in +# the utf8 encoding of an object name component (the part between the +# slashes); this is a limit imposed by the underlying file system (for XFS it +# is 255 bytes). + +max_object_name_component_length = 255 + +# container_listing_limit is the default (and max) number of items +# returned for a container listing request + +#container_listing_limit = 10000 + + +# account_listing_limit is the default (and max) number of items returned +# for an account listing request + +#account_listing_limit = 10000 + + +# max_account_name_length is the max number of bytes in the utf8 encoding of +# an account name: Gluster FS Filename limit (XFS limit?), must be the same +# size as max_object_name_component_length above. + +max_account_name_length = 255 + + +# max_container_name_length is the max number of bytes in the utf8 encoding +# of a container name: Gluster FS Filename limit (XFS limit?), must be the same +# size as max_object_name_component_length above. + +max_container_name_length = 255 diff --git a/swift/1.4.8/gluster-swift-plugin.spec b/swift/1.4.8/gluster-swift-plugin.spec deleted file mode 100644 index 0a1637dc6..000000000 --- a/swift/1.4.8/gluster-swift-plugin.spec +++ /dev/null @@ -1,74 +0,0 @@ -############################################################################################################ -# Command to build rpms.# -# $ rpmbuild -ta %{name}-%{version}-%{release}.tar.gz # -############################################################################################################ -# Setting up the environment. # -# * Create a directory %{name}-%{version} under $HOME/rpmbuild/SOURCES # -# * Copy the contents of plugins directory into $HOME/rpmbuild/SOURCES/%{name}-%{version} # -# * tar zcvf %{name}-%{version}-%{release}.tar.gz $HOME/rpmbuild/SOURCES/%{name}-%{version} %{name}.spec # -# For more information refer # -# http://fedoraproject.org/wiki/How_to_create_an_RPM_package # -############################################################################################################ - -%define _confdir /etc/swift -%define _swiftdir /usr/lib/python2.6/site-packages/swift -%define _ufo_version 1.0 -%define _ufo_release 12 - -Summary : GlusterFS Unified File and Object Storage. -Name : gluster-swift-plugin -Version : %{_ufo_version} -Release : %{_ufo_release} -Group : Application/File -Vendor : Red Hat Inc. -Source0 : %{name}-%{version}-%{release}.tar.gz -Packager : gluster-users@gluster.org -License : Apache -BuildArch: noarch -Requires : memcached -Requires : openssl -Requires : python -Requires : gluster-swift - -%description -Gluster Unified File and Object Storage unifies NAS and object storage -technology. This provides a system for data storage that enables users to access -the same data as an object and as a file, simplifying management and controlling -storage costs. - -%prep -%setup -q - -%install -rm -rf %{buildroot} - -mkdir -p %{buildroot}/%{_swiftdir}/plugins/middleware -mkdir -p %{buildroot}/%{_confdir}/ -mkdir -p %{buildroot}/%{_bindir}/ - -cp constraints.py %{buildroot}/%{_swiftdir}/plugins -cp DiskDir.py %{buildroot}/%{_swiftdir}/plugins -cp DiskFile.py %{buildroot}/%{_swiftdir}/plugins -cp Glusterfs.py %{buildroot}/%{_swiftdir}/plugins -cp __init__.py %{buildroot}/%{_swiftdir}/plugins -cp utils.py %{buildroot}/%{_swiftdir}/plugins -cp fs_utils.py %{buildroot}/%{_swiftdir}/plugins - -cp middleware/__init__.py %{buildroot}/%{_swiftdir}/plugins/middleware -cp middleware/gluster.py %{buildroot}/%{_swiftdir}/plugins/middleware - -cp -r conf/* %{buildroot}/%{_confdir}/ - -cp bin/gluster-swift-gen-builders %{buildroot}/%{_bindir}/ - -%files -%defattr(-,root,root) -%docdir conf -%{_swiftdir}/plugins -%{_bindir}/gluster-swift-gen-builders -%config %{_confdir}/account-server/1.conf-gluster -%config %{_confdir}/container-server/1.conf-gluster -%config %{_confdir}/object-server/1.conf-gluster -%config %{_confdir}/swift.conf-gluster -%config %{_confdir}/proxy-server.conf-gluster -%config %{_confdir}/fs.conf-gluster diff --git a/swift/1.4.8/gluster-swift-ufo.spec b/swift/1.4.8/gluster-swift-ufo.spec new file mode 100644 index 000000000..55b192a27 --- /dev/null +++ b/swift/1.4.8/gluster-swift-ufo.spec @@ -0,0 +1,87 @@ +############################################################################################################ +# Command to build rpms.# +# $ rpmbuild -ta %{name}-%{version}-%{release}.tar.gz # +############################################################################################################ +# Setting up the environment. # +# * Create a directory %{name}-%{version} under $HOME/rpmbuild/SOURCES # +# * Copy the contents of gluster directory into $HOME/rpmbuild/SOURCES/%{name}-%{version} # +# * tar zcvf %{name}-%{version}-%{release}.tar.gz $HOME/rpmbuild/SOURCES/%{name}-%{version} %{name}.spec # +# For more information refer # +# http://fedoraproject.org/wiki/How_to_create_an_RPM_package # +############################################################################################################ + +%if ! (0%{?fedora} > 12 || 0%{?rhel} > 5) +%{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")} +%endif + +%define _confdir /etc/swift +%define _ufo_version 1.1 +%define _ufo_release 1 + +Summary : GlusterFS Unified File and Object Storage. +Name : gluster-swift-ufo +Version : %{_ufo_version} +Release : %{_ufo_release} +Group : Application/File +Vendor : Red Hat Inc. +Source0 : %{name}-%{version}-%{release}.tar.gz +Packager : gluster-users@gluster.org +License : Apache +BuildArch: noarch +Requires : memcached +Requires : openssl +Requires : python +#Requires : openstack-swift >= 1.4.8 +#Requires : openstack-swift-account >= 1.4.8 +#Requires : openstack-swift-auth >= 1.4.8 +#Requires : openstack-swift-container >= 1.4.8 +#Requires : openstack-swift-object >= 1.4.8 +#Requires : openstack-swift-proxy >= 1.4.8 +#Obsoletes: gluster-swift +#Obsoletes: gluster-swift-plugin + +%description +Gluster Unified File and Object Storage unifies NAS and object storage +technology. This provides a system for data storage that enables users to access +the same data as an object and as a file, simplifying management and controlling +storage costs. + +%prep +%setup -q + +%build +%{__python} setup.py build + +%install +rm -rf %{buildroot} + +%{__python} setup.py install -O1 --skip-build --root %{buildroot} + +mkdir -p %{buildroot}/%{_confdir}/ +cp -r etc/* %{buildroot}/%{_confdir}/ + +mkdir -p %{buildroot}/%{_bindir}/ +cp bin/gluster-swift-gen-builders %{buildroot}/%{_bindir}/ + +%clean +rm -rf %{buildroot} + +%files +%defattr(-,root,root) +%{python_sitelib}/gluster +#%{python_sitelib}/gluster/swift/*.py* +#%{python_sitelib}/gluster/swift/common/*.py* +#%{python_sitelib}/gluster/swift/common/middleware +#%{python_sitelib}/gluster/swift/proxy +#%{python_sitelib}/gluster/swift/obj +#%{python_sitelib}/gluster/swift/container +#%{python_sitelib}/gluster/swift/account +%{python_sitelib}/gluster_swift_ufo-%{version}-*.egg-info +%{_bindir}/gluster-swift-gen-builders +%dir %{_sysconfdir}/swift +%config %{_confdir}/account-server/1.conf-gluster +%config %{_confdir}/container-server/1.conf-gluster +%config %{_confdir}/object-server/1.conf-gluster +%config %{_confdir}/swift.conf-gluster +%config %{_confdir}/proxy-server.conf-gluster +%config %{_confdir}/fs.conf-gluster diff --git a/swift/1.4.8/gluster-swift.spec b/swift/1.4.8/gluster-swift.spec deleted file mode 100644 index a525e9e4a..000000000 --- a/swift/1.4.8/gluster-swift.spec +++ /dev/null @@ -1,396 +0,0 @@ -%if ! (0%{?fedora} > 12 || 0%{?rhel} > 5) -%{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")} -%endif - -Name: gluster-swift -Version: 1.4.8 -Release: 12%{?dist} -Summary: OpenStack Object Storage (swift) - -Group: Development/Languages -License: ASL 2.0 -URL: http://launchpad.net/swift -Source0: http://launchpad.net/swift/essex/%{version}/+download/swift-%{version}.tar.gz -Source1: %{name}-functions -Source2: %{name}-account.init -Source4: %{name}-container.init -Source5: %{name}-object.init -Source6: %{name}-proxy.init -Patch0: openstack-swift-newdeps.patch -Patch1: openstack-swift-docmod.patch -Patch2: openstack-swift-nonet.patch -Patch3: gluster.patch - -BuildRoot: %{_tmppath}/swift-%{version}-%{release}-root-%(%{__id_u} -n) - -BuildArch: noarch -BuildRequires: dos2unix -BuildRequires: python-devel -BuildRequires: python-setuptools -BuildRequires: python-netifaces -BuildRequires: python-paste-deploy -Requires: python-configobj -Requires: python-eventlet >= 0.9.8 -Requires: python-greenlet >= 0.3.1 -Requires: python-paste-deploy -Requires: python-simplejson -Requires: python-webob1.0 -Requires: pyxattr -Requires: python-setuptools -Requires: python-netifaces -Requires: python-netifaces - -Conflicts: openstack-swift - -Requires(post): chkconfig -Requires(postun): initscripts -Requires(preun): chkconfig -Requires(pre): shadow-utils -Obsoletes: openstack-swift-auth <= 1.4.0 - -%description -OpenStack Object Storage (swift) aggregates commodity servers to work together -in clusters for reliable, redundant, and large-scale storage of static objects. -Objects are written to multiple hardware devices in the data center, with the -OpenStack software responsible for ensuring data replication and integrity -across the cluster. Storage clusters can scale horizontally by adding new nodes, -which are automatically configured. Should a node fail, OpenStack works to -replicate its content from other active nodes. Because OpenStack uses software -logic to ensure data replication and distribution across different devices, -inexpensive commodity hard drives and servers can be used in lieu of more -expensive equipment. - -%package account -Summary: A swift account server -Group: Applications/System - -Requires: %{name} = %{version}-%{release} - -%description account -OpenStack Object Storage (swift) aggregates commodity servers to work together -in clusters for reliable, redundant, and large-scale storage of static objects. - -This package contains the %{name} account server. - -%package container -Summary: A swift container server -Group: Applications/System - -Requires: %{name} = %{version}-%{release} - -%description container -OpenStack Object Storage (swift) aggregates commodity servers to work together -in clusters for reliable, redundant, and large-scale storage of static objects. - -This package contains the %{name} container server. - -%package object -Summary: A swift object server -Group: Applications/System - -Requires: %{name} = %{version}-%{release} -Requires: rsync >= 3.0 - -%description object -OpenStack Object Storage (swift) aggregates commodity servers to work together -in clusters for reliable, redundant, and large-scale storage of static objects. - -This package contains the %{name} object server. - -%package proxy -Summary: A swift proxy server -Group: Applications/System - -Requires: %{name} = %{version}-%{release} - -%description proxy -OpenStack Object Storage (swift) aggregates commodity servers to work together -in clusters for reliable, redundant, and large-scale storage of static objects. - -This package contains the %{name} proxy server. - -%package doc -Summary: Documentation for %{name} -Group: Documentation -#%if 0%{?rhel} >= 6 -#BuildRequires: python-sphinx10 >= 1.0 -#%endif -%if 0%{?fedora} >= 14 -BuildRequires: python-sphinx >= 1.0 -%endif -# Required for generating docs -BuildRequires: python-eventlet -BuildRequires: python-simplejson -BuildRequires: python-webob1.0 -BuildRequires: pyxattr - -%description doc -OpenStack Object Storage (swift) aggregates commodity servers to work together -in clusters for reliable, redundant, and large-scale storage of static objects. - -This package contains documentation files for %{name}. - -%prep -%setup -q -n swift-%{version} -%patch0 -p1 -b .newdeps -%patch1 -p1 -b .docmod -%patch2 -p1 -b .nonet -%patch3 -p1 -b .gluster -# Fix wrong-file-end-of-line-encoding warning -dos2unix LICENSE - -%build -%{__python} setup.py build -# Fails unless we create the build directory -mkdir -p doc/build -# Build docs -%if 0%{?fedora} >= 14 -%{__python} setup.py build_sphinx -%endif -#%if 0%{?rhel} >= 6 -#export PYTHONPATH="$( pwd ):$PYTHONPATH" -#SPHINX_DEBUG=1 sphinx-1.0-build -b html doc/source doc/build/html -#SPHINX_DEBUG=1 sphinx-1.0-build -b man doc/source doc/build/man -#%endif -# Fix hidden-file-or-dir warning -#rm doc/build/html/.buildinfo - -%install -rm -rf %{buildroot} -%{__python} setup.py install -O1 --skip-build --root %{buildroot} -# Init helper functions -install -p -D -m 644 %{SOURCE1} %{buildroot}%{_datarootdir}/%{name}/functions -# Init scripts -install -p -D -m 755 %{SOURCE2} %{buildroot}%{_initrddir}/%{name}-account -install -p -D -m 755 %{SOURCE4} %{buildroot}%{_initrddir}/%{name}-container -install -p -D -m 755 %{SOURCE5} %{buildroot}%{_initrddir}/%{name}-object -install -p -D -m 755 %{SOURCE6} %{buildroot}%{_initrddir}/%{name}-proxy -# Remove tests -rm -fr %{buildroot}/%{python_sitelib}/test -# Misc other -install -d -m 755 %{buildroot}%{_sysconfdir}/swift -install -d -m 755 %{buildroot}%{_sysconfdir}/swift/account-server -install -d -m 755 %{buildroot}%{_sysconfdir}/swift/container-server -install -d -m 755 %{buildroot}%{_sysconfdir}/swift/object-server -install -d -m 755 %{buildroot}%{_sysconfdir}/swift/proxy-server -# Install pid directory -install -d -m 755 %{buildroot}%{_localstatedir}/run/swift -install -d -m 755 %{buildroot}%{_localstatedir}/run/swift/account-server -install -d -m 755 %{buildroot}%{_localstatedir}/run/swift/container-server -install -d -m 755 %{buildroot}%{_localstatedir}/run/swift/object-server -install -d -m 755 %{buildroot}%{_localstatedir}/run/swift/proxy-server - -%clean -rm -rf %{buildroot} - -%pre -getent group swift >/dev/null || groupadd -r swift -g 160 -getent passwd swift >/dev/null || \ -useradd -r -g swift -u 160 -d %{_sharedstatedir}/swift -s /sbin/nologin \ --c "OpenStack Swift Daemons" swift -exit 0 - -%post account -/sbin/chkconfig --add %{name}-account - -%preun account -if [ $1 = 0 ] ; then - /sbin/service %{name}-account stop >/dev/null 2>&1 - /sbin/chkconfig --del %{name}-account -fi - -%postun account -if [ "$1" -ge "1" ] ; then - /sbin/service %{name}-account condrestart >/dev/null 2>&1 || : -fi - -%post container -/sbin/chkconfig --add %{name}-container - -%preun container -if [ $1 = 0 ] ; then - /sbin/service %{name}-container stop >/dev/null 2>&1 - /sbin/chkconfig --del %{name}-container -fi - -%postun container -if [ "$1" -ge "1" ] ; then - /sbin/service %{name}-container condrestart >/dev/null 2>&1 || : -fi - -%post object -/sbin/chkconfig --add %{name}-object - -%preun object -if [ $1 = 0 ] ; then - /sbin/service %{name}-object stop >/dev/null 2>&1 - /sbin/chkconfig --del %{name}-object -fi - -%postun object -if [ "$1" -ge "1" ] ; then - /sbin/service %{name}-object condrestart >/dev/null 2>&1 || : -fi - -%post proxy -/sbin/chkconfig --add %{name}-proxy - -%preun proxy -if [ $1 = 0 ] ; then - /sbin/service %{name}-proxy stop >/dev/null 2>&1 - /sbin/chkconfig --del %{name}-proxy -fi - -%postun proxy -if [ "$1" -ge "1" ] ; then - /sbin/service %{name}-proxy condrestart >/dev/null 2>&1 || : -fi - -%files -%defattr(-,root,root,-) -%doc AUTHORS LICENSE README -%doc etc/dispersion.conf-sample etc/drive-audit.conf-sample etc/object-expirer.conf-sample -%doc etc/swift.conf-sample -%dir %{_datarootdir}/%{name}/functions -%dir %attr(0755, swift, swift) %{_localstatedir}/run/swift -%dir %{_sysconfdir}/swift -%dir %{python_sitelib}/swift -%{_bindir}/swift -%{_bindir}/swift-account-audit -%{_bindir}/swift-bench -%{_bindir}/swift-drive-audit -%{_bindir}/swift-get-nodes -%{_bindir}/swift-init -%{_bindir}/swift-ring-builder -%{_bindir}/swift-dispersion-populate -%{_bindir}/swift-dispersion-report -%{_bindir}/swift-recon* -%{_bindir}/swift-object-expirer -%{_bindir}/swift-oldies -%{_bindir}/swift-orphans -%{_bindir}/swift-form-signature -%{_bindir}/swift-temp-url -%{python_sitelib}/swift/*.py* -%{python_sitelib}/swift/common -%{python_sitelib}/swift-%{version}-*.egg-info - -%files account -%defattr(-,root,root,-) -%doc etc/account-server.conf-sample -%dir %{_initrddir}/%{name}-account -%dir %attr(0755, swift, swift) %{_localstatedir}/run/swift/account-server -%dir %{_sysconfdir}/swift/account-server -%{_bindir}/swift-account-auditor -%{_bindir}/swift-account-reaper -%{_bindir}/swift-account-replicator -%{_bindir}/swift-account-server -%{python_sitelib}/swift/account - - -%files container -%defattr(-,root,root,-) -%doc etc/container-server.conf-sample -%dir %{_initrddir}/%{name}-container -%dir %attr(0755, swift, swift) %{_localstatedir}/run/swift/container-server -%dir %{_sysconfdir}/swift/container-server -%{_bindir}/swift-container-auditor -%{_bindir}/swift-container-server -%{_bindir}/swift-container-replicator -%{_bindir}/swift-container-updater -%{_bindir}/swift-container-sync -%{python_sitelib}/swift/container - -%files object -%defattr(-,root,root,-) -%doc etc/object-server.conf-sample etc/rsyncd.conf-sample -%dir %{_initrddir}/%{name}-object -%dir %attr(0755, swift, swift) %{_localstatedir}/run/swift/object-server -%dir %{_sysconfdir}/swift/object-server -%{_bindir}/swift-object-auditor -%{_bindir}/swift-object-info -%{_bindir}/swift-object-replicator -%{_bindir}/swift-object-server -%{_bindir}/swift-object-updater -%{python_sitelib}/swift/obj - -%files proxy -%defattr(-,root,root,-) -%doc etc/proxy-server.conf-sample -%dir %{_initrddir}/%{name}-proxy -%dir %attr(0755, swift, swift) %{_localstatedir}/run/swift/proxy-server -%dir %{_sysconfdir}/swift/proxy-server -%{_bindir}/swift-proxy-server -%{python_sitelib}/swift/proxy - -%files doc -%defattr(-,root,root,-) -%doc LICENSE -#%doc doc/build/html - -%changelog -* Thu Apr 26 2012 Anthony Towns 1.4.8-2 -- Apply gluster patches -- Rename to gluster-swift - -* Thu Mar 22 2012 Alan Pevec 1.4.8-1 -- Update to 1.4.8 - -* Fri Mar 09 2012 Alan Pevec 1.4.7-1 -- Update to 1.4.7 - -* Mon Feb 13 2012 Alan Pevec 1.4.6-1 -- Update to 1.4.6 - -* Thu Jan 12 2012 Alan Pevec 1.4.4-2 -- add back /var/run/swift for el6 - -* Wed Jan 04 2012 Alan Pevec 1.4.4-1 -- Use updated parallel install versions of epel packages (pbrady) -- Ensure the docs aren't built with the system glance module (pbrady) -- Ensure we don't access the net when building docs (pbrady) -- Update to 1.4.4 - -* Wed Nov 23 2011 David Nalley -1.4.3-2 -* fixed some missing requires - -* Sat Nov 05 2011 David Nalley - 1.4.3-1 -- Update to 1.4.3 -- fix init script add, registration, deletion BZ 685155 -- fixing BR to facilitate epel6 building - -* Tue Aug 23 2011 David Nalley - 1.4.0-2 -- adding uid:gid for bz 732693 - -* Wed Jun 22 2011 David Nalley - 1.4.1-1 -- Update to 1.4.0 -- change the name of swift binary from st to swift - -* Sat Jun 04 2011 David Nalley - 1.4.0-1 -- Update to 1.4.0 - -* Fri May 20 2011 David Nalley - 1.3.0-1 -- Update to 1.3.0 - -* Tue Feb 08 2011 Fedora Release Engineering - 1.1.0-2 -- Rebuilt for https://fedoraproject.org/wiki/Fedora_15_Mass_Rebuild - -* Sun Dec 05 2010 Silas Sewell - 1.1.0-1 -- Update to 1.1.0 - -* Sun Aug 08 2010 Silas Sewell - 1.0.2-5 -- Update for new Python macro guidelines -- Use dos2unix instead of sed -- Make gecos field more descriptive - -* Wed Jul 28 2010 Silas Sewell - 1.0.2-4 -- Rename to openstack-swift - -* Wed Jul 28 2010 Silas Sewell - 1.0.2-3 -- Fix return value in swift-functions - -* Tue Jul 27 2010 Silas Sewell - 1.0.2-2 -- Add swift user -- Update init scripts - -* Sun Jul 18 2010 Silas Sewell - 1.0.2-1 -- Initial build diff --git a/swift/1.4.8/gluster/__init__.py b/swift/1.4.8/gluster/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/swift/1.4.8/gluster/swift/__init__.py b/swift/1.4.8/gluster/swift/__init__.py new file mode 100644 index 000000000..17578ebee --- /dev/null +++ b/swift/1.4.8/gluster/swift/__init__.py @@ -0,0 +1,18 @@ +""" Gluster Swift UFO """ + +class Version(object): + def __init__(self, canonical_version, final): + self.canonical_version = canonical_version + self.final = final + + @property + def pretty_version(self): + if self.final: + return self.canonical_version + else: + return '%s-dev' % (self.canonical_version,) + + +_version = Version('1.1', True) +__version__ = _version.pretty_version +__canonical_version__ = _version.canonical_version diff --git a/swift/1.4.8/gluster/swift/account/__init__.py b/swift/1.4.8/gluster/swift/account/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/swift/1.4.8/gluster/swift/account/server.py b/swift/1.4.8/gluster/swift/account/server.py new file mode 100644 index 000000000..8b9831579 --- /dev/null +++ b/swift/1.4.8/gluster/swift/account/server.py @@ -0,0 +1,45 @@ +# Copyright (c) 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Account Server for Gluster Swift UFO """ + +# Simply importing this monkey patches the constraint handling to fit our +# needs +import gluster.swift.common.constraints + +from swift.account import server +from gluster.swift.common.DiskDir import DiskAccount + + +class AccountController(server.AccountController): + def _get_account_broker(self, drive, part, account): + """ + Overriden to provide the GlusterFS specific broker that talks to + Gluster for the information related to servicing a given request + instead of talking to a database. + + :param drive: drive that holds the container + :param part: partition the container is in + :param account: account name + :returns: DiskDir object + """ + return DiskAccount(self.root, account, self.logger) + + +def app_factory(global_conf, **local_conf): + """paste.deploy app factory for creating WSGI account server apps.""" + conf = global_conf.copy() + conf.update(local_conf) + return AccountController(conf) diff --git a/swift/1.4.8/gluster/swift/common/DiskDir.py b/swift/1.4.8/gluster/swift/common/DiskDir.py new file mode 100644 index 000000000..8602fabae --- /dev/null +++ b/swift/1.4.8/gluster/swift/common/DiskDir.py @@ -0,0 +1,500 @@ +# Copyright (c) 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os, errno + +from gluster.swift.common.utils import clean_metadata, dir_empty, rmdirs, \ + mkdirs, validate_account, validate_container, is_marker, \ + get_container_details, get_account_details, get_container_metadata, \ + create_container_metadata, create_account_metadata, DEFAULT_GID, \ + DEFAULT_UID, validate_object, create_object_metadata, read_metadata, \ + write_metadata, X_CONTENT_TYPE, X_CONTENT_LENGTH, X_TIMESTAMP, \ + X_PUT_TIMESTAMP, X_TYPE, X_ETAG, X_OBJECTS_COUNT, X_BYTES_USED, \ + X_CONTAINER_COUNT, CONTAINER +from gluster.swift.common import Glusterfs + +from swift.common.constraints import CONTAINER_LISTING_LIMIT +from swift.common.utils import normalize_timestamp, TRUE_VALUES + + +DATADIR = 'containers' + +# Create a dummy db_file in /etc/swift +_unittests_enabled = os.getenv('GLUSTER_UNIT_TEST_ENABLED', 'no') +if _unittests_enabled in TRUE_VALUES: + _tmp_dir = '/tmp/gluster_unit_tests' + try: + os.mkdir(_tmp_dir) + except OSError as e: + if e.errno != errno.EEXIST: + raise + _db_file = os.path.join(_tmp_dir, 'db_file.db') +else: + _db_file = '/etc/swift/db_file.db' +if not os.path.exists(_db_file): + file(_db_file, 'w+') + + +def _read_metadata(dd): + """ Filter read metadata so that it always returns a tuple that includes + some kind of timestamp. With 1.4.8 of the Swift integration the + timestamps were not stored. Here we fabricate timestamps for volumes + where the existing data has no timestamp (that is, stored data is not + a tuple), allowing us a measure of backward compatibility. + + FIXME: At this time it does not appear that the timestamps on each + metadata are used for much, so this should not hurt anything. + """ + metadata_i = read_metadata(dd) + metadata = {} + timestamp = 0 + for key, value in metadata_i.iteritems(): + if not isinstance(value, tuple): + value = (value, timestamp) + metadata[key] = value + return metadata + + +class DiskCommon(object): + def is_deleted(self): + return not os.path.exists(self.datadir) + + def filter_prefix(self, objects, prefix): + """ + Accept sorted list. + """ + found = 0 + filtered_objs = [] + for object_name in objects: + if object_name.startswith(prefix): + filtered_objs.append(object_name) + found = 1 + else: + if found: + break + return filtered_objs + + def filter_delimiter(self, objects, delimiter, prefix): + """ + Accept sorted list. + Objects should start with prefix. + """ + filtered_objs=[] + for object_name in objects: + tmp_obj = object_name.replace(prefix, '', 1) + sufix = tmp_obj.split(delimiter, 1) + new_obj = prefix + sufix[0] + if new_obj and new_obj not in filtered_objs: + filtered_objs.append(new_obj) + + return filtered_objs + + def filter_marker(self, objects, marker): + """ + TODO: We can traverse in reverse order to optimize. + Accept sorted list. + """ + filtered_objs=[] + found = 0 + if objects[-1] < marker: + return filtered_objs + for object_name in objects: + if object_name > marker: + filtered_objs.append(object_name) + + return filtered_objs + + def filter_end_marker(self, objects, end_marker): + """ + Accept sorted list. + """ + filtered_objs=[] + for object_name in objects: + if object_name < end_marker: + filtered_objs.append(object_name) + else: + break + + return filtered_objs + + def filter_limit(self, objects, limit): + filtered_objs=[] + for i in range(0, limit): + filtered_objs.append(objects[i]) + + return filtered_objs + + +class DiskDir(DiskCommon): + """ + Manage object files on disk. + + :param path: path to devices on the node + :param account: account name for the object + :param container: container name for the object + :param logger: account or container server logging object + :param uid: user ID container object should assume + :param gid: group ID container object should assume + """ + + def __init__(self, path, account, container, logger, + uid=DEFAULT_UID, gid=DEFAULT_GID): + self.root = path + if container: + self.container = container + else: + self.container = None + if self.container: + self.datadir = os.path.join(path, account, self.container) + else: + self.datadir = os.path.join(path, account) + # Note that the account name has a one-to-one mapping to the gluster + # mount point, or volume name. + self.account = account + assert logger is not None + self.logger = logger + self.metadata = {} + self.container_info = None + self.object_info = None + self.uid = int(uid) + self.gid = int(gid) + self.db_file = _db_file + self.dir_exists = os.path.exists(self.datadir) + if self.dir_exists: + try: + self.metadata = _read_metadata(self.datadir) + except EOFError: + create_container_metadata(self.datadir) + else: + return + if self.container: + if not self.metadata: + create_container_metadata(self.datadir) + self.metadata = _read_metadata(self.datadir) + else: + if not validate_container(self.metadata): + create_container_metadata(self.datadir) + self.metadata = _read_metadata(self.datadir) + else: + if not self.metadata: + create_account_metadata(self.datadir) + self.metadata = _read_metadata(self.datadir) + else: + if not validate_account(self.metadata): + create_account_metadata(self.datadir) + self.metadata = _read_metadata(self.datadir) + + def empty(self): + return dir_empty(self.datadir) + + def delete(self): + if self.empty(): + #For delete account. + if os.path.ismount(self.datadir): + clean_metadata(self.datadir) + else: + rmdirs(self.datadir) + self.dir_exists = False + + def put_metadata(self, metadata): + """ + Write metadata to directory/container. + """ + write_metadata(self.datadir, metadata) + self.metadata = metadata + + def put(self, metadata): + """ + Create and write metatdata to directory/container. + :param metadata: Metadata to write. + """ + if not self.dir_exists: + mkdirs(self.datadir) + + os.chown(self.datadir, self.uid, self.gid) + write_metadata(self.datadir, metadata) + self.metadata = metadata + self.dir_exists = True + + def put_obj(self, content_length, timestamp): + ocnt = self.metadata[X_OBJECTS_COUNT][0] + self.metadata[X_OBJECTS_COUNT] = (int(ocnt) + 1, timestamp) + self.metadata[X_PUT_TIMESTAMP] = timestamp + bused = self.metadata[X_BYTES_USED][0] + self.metadata[X_BYTES_USED] = (int(bused) + int(content_length), timestamp) + #TODO: define update_metadata instad of writing whole metadata again. + self.put_metadata(self.metadata) + + def delete_obj(self, content_length): + ocnt, timestamp = self.metadata[X_OBJECTS_COUNT][0] + self.metadata[X_OBJECTS_COUNT] = (int(ocnt) - 1, timestamp) + bused, timestamp = self.metadata[X_BYTES_USED] + self.metadata[X_BYTES_USED] = (int(bused) - int(content_length), timestamp) + self.put_metadata(self.metadata) + + def put_container(self, container, put_timestamp, del_timestamp, object_count, bytes_used): + """ + For account server. + """ + self.metadata[X_OBJECTS_COUNT] = (0, put_timestamp) + self.metadata[X_BYTES_USED] = (0, put_timestamp) + ccnt = self.metadata[X_CONTAINER_COUNT][0] + self.metadata[X_CONTAINER_COUNT] = (int(ccnt) + 1, put_timestamp) + self.metadata[X_PUT_TIMESTAMP] = (1, put_timestamp) + self.put_metadata(self.metadata) + + def delete_container(self, object_count, bytes_used): + """ + For account server. + """ + self.metadata[X_OBJECTS_COUNT] = (0, 0) + self.metadata[X_BYTES_USED] = (0, 0) + ccnt, timestamp = self.metadata[X_CONTAINER_COUNT] + self.metadata[X_CONTAINER_COUNT] = (int(ccnt) - 1, timestamp) + self.put_metadata(self.metadata) + + def unlink(self): + """ + Remove directory/container if empty. + """ + if dir_empty(self.datadir): + rmdirs(self.datadir) + + def list_objects_iter(self, limit, marker, end_marker, + prefix, delimiter, path): + """ + Returns tuple of name, created_at, size, content_type, etag. + """ + if path: + prefix = path = path.rstrip('/') + '/' + delimiter = '/' + if delimiter and not prefix: + prefix = '' + + self.update_object_count() + + objects, object_count, bytes_used = self.object_info + + if objects: + objects.sort() + + if objects and prefix: + objects = self.filter_prefix(objects, prefix) + + if objects and delimiter: + objects = self.filter_delimiter(objects, delimiter, prefix) + + if objects and marker: + objects = self.filter_marker(objects, marker) + + if objects and end_marker: + objects = self.filter_end_marker(objects, end_marker) + + if objects and limit: + if len(objects) > limit: + objects = self.filter_limit(objects, limit) + + container_list = [] + if objects: + for obj in objects: + list_item = [] + list_item.append(obj) + obj_path = os.path.join(self.datadir, obj) + metadata = read_metadata(obj_path) + if not metadata or not validate_object(metadata): + metadata = create_object_metadata(obj_path) + if metadata: + list_item.append(metadata[X_TIMESTAMP]) + list_item.append(int(metadata[X_CONTENT_LENGTH])) + list_item.append(metadata[X_CONTENT_TYPE]) + list_item.append(metadata[X_ETAG]) + container_list.append(list_item) + + return container_list + + def update_object_count(self): + if not self.object_info: + self.object_info = get_container_details(self.datadir) + + objects, object_count, bytes_used = self.object_info + + if X_OBJECTS_COUNT not in self.metadata \ + or int(self.metadata[X_OBJECTS_COUNT][0]) != object_count \ + or X_BYTES_USED not in self.metadata \ + or int(self.metadata[X_BYTES_USED][0]) != bytes_used: + self.metadata[X_OBJECTS_COUNT] = (object_count, 0) + self.metadata[X_BYTES_USED] = (bytes_used, 0) + write_metadata(self.datadir, self.metadata) + + def update_container_count(self): + if not self.container_info: + self.container_info = get_account_details(self.datadir) + + containers, container_count = self.container_info + + if X_CONTAINER_COUNT not in self.metadata \ + or int(self.metadata[X_CONTAINER_COUNT][0]) != container_count: + self.metadata[X_CONTAINER_COUNT] = (container_count, 0) + write_metadata(self.datadir, self.metadata) + + def get_info(self, include_metadata=False): + """ + Get global data for the container. + :returns: dict with keys: account, container, object_count, bytes_used, + hash, id, created_at, put_timestamp, delete_timestamp, + reported_put_timestamp, reported_delete_timestamp, + reported_object_count, and reported_bytes_used. + If include_metadata is set, metadata is included as a key + pointing to a dict of tuples of the metadata + """ + # TODO: delete_timestamp, reported_put_timestamp + # reported_delete_timestamp, reported_object_count, + # reported_bytes_used, created_at + if not Glusterfs.OBJECT_ONLY: + # If we are not configured for object only environments, we should + # update the object counts in case they changed behind our back. + self.update_object_count() + + data = {'account' : self.account, 'container' : self.container, + 'object_count' : self.metadata.get(X_OBJECTS_COUNT, ('0', 0))[0], + 'bytes_used' : self.metadata.get(X_BYTES_USED, ('0',0))[0], + 'hash': '', 'id' : '', 'created_at' : '1', + 'put_timestamp' : self.metadata.get(X_PUT_TIMESTAMP, ('0',0))[0], + 'delete_timestamp' : '1', + 'reported_put_timestamp' : '1', 'reported_delete_timestamp' : '1', + 'reported_object_count' : '1', 'reported_bytes_used' : '1'} + if include_metadata: + data['metadata'] = self.metadata + return data + + def put_object(self, name, timestamp, size, content_type, + etag, deleted=0): + # TODO: Implement the specifics of this func. + pass + + def initialize(self, timestamp): + pass + + def update_put_timestamp(self, timestamp): + """ + Create the container if it doesn't exist and update the timestamp + """ + if not os.path.exists(self.datadir): + self.put(self.metadata) + + def delete_object(self, name, timestamp): + # TODO: Implement the delete object + pass + + def delete_db(self, timestamp): + """ + Delete the container + """ + self.unlink() + + def update_metadata(self, metadata): + assert self.metadata, "Valid container/account metadata should have been created by now" + if metadata: + new_metadata = self.metadata.copy() + new_metadata.update(metadata) + if new_metadata != self.metadata: + write_metadata(self.datadir, new_metadata) + self.metadata = new_metadata + + +class DiskAccount(DiskDir): + def __init__(self, root, account, logger): + super(DiskAccount, self).__init__(root, account, None, logger) + assert self.dir_exists + + def list_containers_iter(self, limit, marker, end_marker, + prefix, delimiter): + """ + Return tuple of name, object_count, bytes_used, 0(is_subdir). + Used by account server. + """ + if delimiter and not prefix: + prefix = '' + + self.update_container_count() + + containers, container_count = self.container_info + + if containers: + containers.sort() + + if containers and prefix: + containers = self.filter_prefix(containers, prefix) + + if containers and delimiter: + containers = self.filter_delimiter(containers, delimiter, prefix) + + if containers and marker: + containers = self.filter_marker(containers, marker) + + if containers and end_marker: + containers = self.filter_end_marker(containers, end_marker) + + if containers and limit: + if len(containers) > limit: + containers = self.filter_limit(containers, limit) + + account_list = [] + if containers: + for cont in containers: + list_item = [] + metadata = None + list_item.append(cont) + cont_path = os.path.join(self.datadir, cont) + metadata = _read_metadata(cont_path) + if not metadata or not validate_container(metadata): + metadata = create_container_metadata(cont_path) + + if metadata: + list_item.append(metadata[X_OBJECTS_COUNT][0]) + list_item.append(metadata[X_BYTES_USED][0]) + list_item.append(0) + account_list.append(list_item) + + return account_list + + def get_info(self, include_metadata=False): + """ + Get global data for the account. + :returns: dict with keys: account, created_at, put_timestamp, + delete_timestamp, container_count, object_count, + bytes_used, hash, id + """ + if not Glusterfs.OBJECT_ONLY: + # If we are not configured for object only environments, we should + # update the container counts in case they changed behind our back. + self.update_container_count() + + data = {'account' : self.account, 'created_at' : '1', + 'put_timestamp' : '1', 'delete_timestamp' : '1', + 'container_count' : self.metadata.get(X_CONTAINER_COUNT, (0,0))[0], + 'object_count' : self.metadata.get(X_OBJECTS_COUNT, (0,0))[0], + 'bytes_used' : self.metadata.get(X_BYTES_USED, (0,0))[0], + 'hash' : '', 'id' : ''} + + if include_metadata: + data['metadata'] = self.metadata + return data + + def get_container_timestamp(self, container): + cont_path = os.path.join(self.datadir, container) + metadata = read_metadata(cont_path) + + return int(metadata.get(X_PUT_TIMESTAMP, ('0',0))[0]) or None diff --git a/swift/1.4.8/gluster/swift/common/DiskFile.py b/swift/1.4.8/gluster/swift/common/DiskFile.py new file mode 100644 index 000000000..6404be6d6 --- /dev/null +++ b/swift/1.4.8/gluster/swift/common/DiskFile.py @@ -0,0 +1,323 @@ +# Copyright (c) 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from eventlet import tpool +from tempfile import mkstemp +from contextlib import contextmanager +from swift.common.utils import normalize_timestamp, renamer +from gluster.swift.common.utils import mkdirs, rmdirs, validate_object, \ + create_object_metadata, do_open, do_close, do_unlink, do_chown, \ + do_stat, do_listdir, read_metadata, write_metadata +from gluster.swift.common.utils import X_CONTENT_TYPE, X_CONTENT_LENGTH, \ + X_TIMESTAMP, X_PUT_TIMESTAMP, X_TYPE, X_ETAG, X_OBJECTS_COUNT, \ + X_BYTES_USED, X_OBJECT_TYPE, FILE, DIR, MARKER_DIR, OBJECT, DIR_TYPE, \ + FILE_TYPE, DEFAULT_UID, DEFAULT_GID + +import logging +from swift.obj.server import DiskFile + + +DATADIR = 'objects' +ASYNCDIR = 'async_pending' +KEEP_CACHE_SIZE = (5 * 1024 * 1024) +# keep these lower-case +DISALLOWED_HEADERS = set('content-length content-type deleted etag'.split()) + + +class Gluster_DiskFile(DiskFile): + """ + Manage object files on disk. + + :param path: path to devices on the node/mount path for UFO. + :param device: device name/account_name for UFO. + :param partition: partition on the device the object lives in + :param account: account name for the object + :param container: container name for the object + :param obj: object name for the object + :param keep_data_fp: if True, don't close the fp, otherwise close it + :param disk_chunk_Size: size of chunks on file reads + :param uid: user ID disk object should assume (file or directory) + :param gid: group ID disk object should assume (file or directory) + """ + + def __init__(self, path, device, partition, account, container, obj, + logger, keep_data_fp=False, disk_chunk_size=65536, + uid=DEFAULT_UID, gid=DEFAULT_GID): + self.disk_chunk_size = disk_chunk_size + device = account + #Don't support obj_name ending/begining with '/', like /a, a/, /a/b/ etc + obj = obj.strip('/') + if '/' in obj: + self.obj_path, self.obj = obj.rsplit('/', 1) + else: + self.obj_path = '' + self.obj = obj + + if self.obj_path: + self.name = '/'.join((container, self.obj_path)) + else: + self.name = container + #Absolute path for obj directory. + self.datadir = os.path.join(path, device, self.name) + self.device_path = os.path.join(path, device) + self.container_path = os.path.join(path, device, container) + self.tmpdir = os.path.join(path, device, 'tmp') + self.logger = logger + self.metadata = {} + self.data_file = None + self.fp = None + self.iter_etag = None + self.started_at_0 = False + self.read_to_eof = False + self.quarantined_dir = None + self.keep_cache = False + self.is_dir = False + self.is_valid = True + self.uid = int(uid) + self.gid = int(gid) + if not os.path.exists(self.datadir + '/' + self.obj): + return + + self.data_file = os.path.join(self.datadir, self.obj) + self.metadata = read_metadata(self.datadir + '/' + self.obj) + if not self.metadata: + create_object_metadata(self.datadir + '/' + self.obj) + self.metadata = read_metadata(self.datadir + '/' + self.obj) + + if not validate_object(self.metadata): + create_object_metadata(self.datadir + '/' + self.obj) + self.metadata = read_metadata(self.datadir + '/' + + self.obj) + + self.filter_metadata() + + if os.path.isdir(self.datadir + '/' + self.obj): + self.is_dir = True + else: + self.fp = do_open(self.data_file, 'rb') + if not keep_data_fp: + self.close(verify_file=False) + + def close(self, verify_file=True): + """ + Close the file. Will handle quarantining file if necessary. + + :param verify_file: Defaults to True. If false, will not check + file to see if it needs quarantining. + """ + #Marker directory + if self.is_dir: + return + if self.fp: + do_close(self.fp) + self.fp = None + + def is_deleted(self): + """ + Check if the file is deleted. + + :returns: True if the file doesn't exist or has been flagged as + deleted. + """ + return not self.data_file + + def create_dir_object(self, dir_path): + #TODO: if object already exists??? + if os.path.exists(dir_path) and not os.path.isdir(dir_path): + self.logger.error("Deleting file %s", dir_path) + do_unlink(dir_path) + #If dir aleady exist just override metadata. + mkdirs(dir_path) + do_chown(dir_path, self.uid, self.gid) + create_object_metadata(dir_path) + return True + + def put_metadata(self, metadata): + obj_path = self.datadir + '/' + self.obj + write_metadata(obj_path, metadata) + self.metadata = metadata + + def put(self, fd, tmppath, metadata, extension=''): + """ + Finalize writing the file on disk, and renames it from the temp file to + the real location. This should be called after the data has been + written to the temp file. + + :params fd: file descriptor of the temp file + :param tmppath: path to the temporary file being used + :param metadata: dictionary of metadata to be written + :param extention: extension to be used when making the file + """ + if extension == '.ts': + # TombStone marker (deleted) + return True + + # Fix up the metadata to ensure it has a proper value for the + # Content-Type metadata, as well as an X_TYPE and X_OBJECT_TYPE + # metadata values. + + content_type = metadata['Content-Type'] + if not content_type: + metadata['Content-Type'] = FILE_TYPE + x_object_type = FILE + else: + x_object_type = MARKER_DIR if content_type.lower() == DIR_TYPE else FILE + metadata[X_TYPE] = OBJECT + metadata[X_OBJECT_TYPE] = x_object_type + + if extension == '.meta': + # Metadata recorded separately from the file + self.put_metadata(metadata) + return True + + extension = '' + + if metadata[X_OBJECT_TYPE] == MARKER_DIR: + self.create_dir_object(os.path.join(self.datadir, self.obj)) + self.put_metadata(metadata) + self.data_file = self.datadir + '/' + self.obj + return True + + # Check if directory already exists. + if self.is_dir: + self.logger.error('Directory already exists %s/%s' % \ + (self.datadir , self.obj)) + return False + + timestamp = normalize_timestamp(metadata[X_TIMESTAMP]) + write_metadata(tmppath, metadata) + if X_CONTENT_LENGTH in metadata: + self.drop_cache(fd, 0, int(metadata[X_CONTENT_LENGTH])) + tpool.execute(os.fsync, fd) + if self.obj_path: + dir_objs = self.obj_path.split('/') + tmp_path = '' + if len(dir_objs): + for dir_name in dir_objs: + if tmp_path: + tmp_path = tmp_path + '/' + dir_name + else: + tmp_path = dir_name + if not self.create_dir_object(os.path.join(self.container_path, + tmp_path)): + self.logger.error("Failed in subdir %s",\ + os.path.join(self.container_path,tmp_path)) + return False + + renamer(tmppath, os.path.join(self.datadir, + self.obj + extension)) + do_chown(os.path.join(self.datadir, self.obj + extension), \ + self.uid, self.gid) + self.metadata = metadata + self.data_file = self.datadir + '/' + self.obj + extension + return True + + def unlinkold(self, timestamp): + """ + Remove any older versions of the object file. Any file that has an + older timestamp than timestamp will be deleted. + + :param timestamp: timestamp to compare with each file + """ + if self.metadata and self.metadata['X-Timestamp'] != timestamp: + self.unlink() + + def unlink(self): + """ + Remove the file. + """ + #Marker dir. + if self.is_dir: + rmdirs(os.path.join(self.datadir, self.obj)) + if not os.path.isdir(os.path.join(self.datadir, self.obj)): + self.metadata = {} + self.data_file = None + else: + logging.error('Unable to delete dir %s' % os.path.join(self.datadir, self.obj)) + return + + for fname in do_listdir(self.datadir): + if fname == self.obj: + try: + do_unlink(os.path.join(self.datadir, fname)) + except OSError, err: + if err.errno != errno.ENOENT: + raise + + #Remove entire path for object. + #remove_dir_path(self.obj_path, self.container_path) + + self.metadata = {} + self.data_file = None + + def get_data_file_size(self): + """ + Returns the os.path.getsize for the file. Raises an exception if this + file does not match the Content-Length stored in the metadata. Or if + self.data_file does not exist. + + :returns: file size as an int + :raises DiskFileError: on file size mismatch. + :raises DiskFileNotExist: on file not existing (including deleted) + """ + #Marker directory. + if self.is_dir: + return 0 + try: + file_size = 0 + if self.data_file: + file_size = os.path.getsize(self.data_file) + if X_CONTENT_LENGTH in self.metadata: + metadata_size = int(self.metadata[X_CONTENT_LENGTH]) + if file_size != metadata_size: + self.metadata[X_CONTENT_LENGTH] = file_size + self.update_object(self.metadata) + + return file_size + except OSError, err: + if err.errno != errno.ENOENT: + raise + raise DiskFileNotExist('Data File does not exist.') + + def update_object(self, metadata): + obj_path = self.datadir + '/' + self.obj + write_metadata(obj_path, metadata) + self.metadata = metadata + + def filter_metadata(self): + if X_TYPE in self.metadata: + self.metadata.pop(X_TYPE) + if X_OBJECT_TYPE in self.metadata: + self.metadata.pop(X_OBJECT_TYPE) + + @contextmanager + def mkstemp(self): + """Contextmanager to make a temporary file.""" + + if not os.path.exists(self.tmpdir): + mkdirs(self.tmpdir) + fd, tmppath = mkstemp(dir=self.tmpdir) + try: + yield fd, tmppath + finally: + try: + os.close(fd) + except OSError: + pass + try: + os.unlink(tmppath) + except OSError: + pass diff --git a/swift/1.4.8/gluster/swift/common/Glusterfs.py b/swift/1.4.8/gluster/swift/common/Glusterfs.py new file mode 100644 index 000000000..2d3273ed4 --- /dev/null +++ b/swift/1.4.8/gluster/swift/common/Glusterfs.py @@ -0,0 +1,126 @@ +# Copyright (c) 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import os, fcntl, time +from ConfigParser import ConfigParser +from swift.common.utils import TRUE_VALUES +from gluster.swift.common.fs_utils import mkdirs + + +# +# Read the fs.conf file once at startup (module load) +# +_fs_conf = ConfigParser() +MOUNT_IP = 'localhost' +REMOTE_CLUSTER = False +OBJECT_ONLY = False +if _fs_conf.read(os.path.join('/etc/swift', 'fs.conf')): + try: + MOUNT_IP = _fs_conf.get('DEFAULT', 'mount_ip', 'localhost') + except (NoSectionError, NoOptionError): + pass + try: + REMOTE_CLUSTER = _fs_conf.get('DEFAULT', 'remote_cluster', False) in TRUE_VALUES + except (NoSectionError, NoOptionError): + pass + try: + OBJECT_ONLY = _fs_conf.get('DEFAULT', 'object_only', "no") in TRUE_VALUES + except (NoSectionError, NoOptionError): + pass +NAME = 'glusterfs' + + +def _busy_wait(full_mount_path): + # Iterate for definite number of time over a given + # interval for successful mount + for i in range(0, 5): + if os.path.ismount(os.path.join(full_mount_path)): + return True + time.sleep(2) + logging.error('Busy wait for mount timed out for mount %s', full_mount_path) + return False + +def mount(root, drive): + # FIXME: Possible thundering herd problem here + + el = _get_export_list() + for export in el: + if drive == export: + break + else: + logging.error('No export found in %r matching drive %s', el, drive) + return False + + # NOTE: root is typically the default value of /mnt/gluster-object + full_mount_path = os.path.join(root, drive) + if not os.path.isdir(full_mount_path): + mkdirs(full_mount_path) + + pid_dir = "/var/lib/glusterd/vols/%s/run/" % drive + pid_file = os.path.join(pid_dir, 'swift.pid'); + + if not os.path.exists(pid_dir): + mkdirs(pid_dir) + + fd = os.open(pid_file, os.O_CREAT|os.O_RDWR) + with os.fdopen(fd, 'r+b') as f: + try: + fcntl.lockf(f, fcntl.LOCK_EX|fcntl.LOCK_NB) + except: + ex = sys.exc_info()[1] + if isinstance(ex, IOError) and ex.errno in (EACCES, EAGAIN): + # This means that some other process is mounting the + # filesystem, so wait for the mount process to complete + return _busy_wait(full_mount_path) + + mnt_cmd = 'mount -t glusterfs %s:%s %s' % (MOUNT_IP, export, \ + full_mount_path) + if os.system(mnt_cmd) or not _busy_wait(full_mount_path): + logging.error('Mount failed %s: %s', NAME, mnt_cmd) + return False + return True + +def unmount(full_mount_path): + # FIXME: Possible thundering herd problem here + + umnt_cmd = 'umount %s 2>> /dev/null' % full_mount_path + if os.system(umnt_cmd): + logging.error('Unable to unmount %s %s' % (full_mount_path, NAME)) + +def _get_export_list(): + if REMOTE_CLUSTER: + cmnd = 'ssh %s gluster volume info' % MOUNT_IP + else: + cmnd = 'gluster volume info' + + export_list = [] + + if os.system(cmnd + ' >> /dev/null'): + if REMOTE_CLUSTER: + logging.error('Getting volume info failed %s, make sure to have '\ + 'passwordless ssh on %s', NAME, MOUNT_IP) + else: + logging.error('Getting volume failed %s', NAME) + else: + fp = os.popen(cmnd) + while True: + item = fp.readline() + if not item: + break + item = item.strip('\n').strip(' ') + if item.lower().startswith('volume name:'): + export_list.append(item.split(':')[1].strip(' ')) + + return export_list diff --git a/swift/1.4.8/gluster/swift/common/__init__.py b/swift/1.4.8/gluster/swift/common/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/swift/1.4.8/gluster/swift/common/constraints.py b/swift/1.4.8/gluster/swift/common/constraints.py new file mode 100644 index 000000000..a4fc8008c --- /dev/null +++ b/swift/1.4.8/gluster/swift/common/constraints.py @@ -0,0 +1,82 @@ +# Copyright (c) 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from webob.exc import HTTPBadRequest + +import swift.common.constraints +from gluster.swift.common import Glusterfs + + +MAX_OBJECT_NAME_COMPONENT_LENGTH = swift.common.constraints.constraints_conf_int( + 'max_object_name_component_length', 255) + +def validate_obj_name_component(obj): + if len(obj) > MAX_OBJECT_NAME_COMPONENT_LENGTH: + return 'too long (%d)' % len(obj) + if obj == '.' or obj == '..': + return 'cannot be . or ..' + return '' + +# Save the original check object creation +__check_object_creation = swift.common.constraints.check_object_creation + +# Define our new one which invokes the original +def gluster_check_object_creation(req, object_name): + """ + Check to ensure that everything is alright about an object to be created. + Monkey patches swift.common.constraints.check_object_creation, invoking + the original, and then adding an additional check for individual object + name components. + + :param req: HTTP request object + :param object_name: name of object to be created + :raises HTTPRequestEntityTooLarge: the object is too large + :raises HTTPLengthRequered: missing content-length header and not + a chunked request + :raises HTTPBadRequest: missing or bad content-type header, or + bad metadata + """ + ret = __check_object_creation(req, object_name) + + if ret is None: + for obj in object_name.split('/'): + reason = validate_obj_name_component(obj) + if reason: + bdy = 'Invalid object name "%s", component "%s" %s' \ + % (object_name, obj, reason) + ret = HTTPBadRequest(body=bdy, + request=req, + content_type='text/plain') + + return ret + +# Replace the original check object creation with ours +swift.common.constraints.check_object_creation = gluster_check_object_creation + +# Save the original check mount +__check_mount = swift.common.constraints.check_mount + +# Define our new one which invokes the original +def gluster_check_mount(root, drive): + # FIXME: Potential performance optimization here to not call the original + # check mount which makes two stat calls. We could do what they do with + # just one. + if __check_mount(root, drive): + return True + + return Glusterfs.mount(root, drive) + +# Replace the original check mount with ours +swift.common.constraints.check_mount = gluster_check_mount diff --git a/swift/1.4.8/gluster/swift/common/fs_utils.py b/swift/1.4.8/gluster/swift/common/fs_utils.py new file mode 100644 index 000000000..7f5292c2b --- /dev/null +++ b/swift/1.4.8/gluster/swift/common/fs_utils.py @@ -0,0 +1,156 @@ +# Copyright (c) 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import errno + +def do_mkdir(path): + try: + os.mkdir(path) + except Exception, err: + logging.exception("Mkdir failed on %s err: %s", path, str(err)) + if err.errno != errno.EEXIST: + raise + return True + +def do_makedirs(path): + try: + os.makedirs(path) + except Exception, err: + logging.exception("Makedirs failed on %s err: %s", path, str(err)) + if err.errno != errno.EEXIST: + raise + return True + +def do_listdir(path): + try: + buf = os.listdir(path) + except Exception, err: + logging.exception("Listdir failed on %s err: %s", path, str(err)) + raise + return buf + +def do_chown(path, uid, gid): + try: + os.chown(path, uid, gid) + except Exception, err: + logging.exception("Chown failed on %s err: %s", path, str(err)) + raise + return True + +def do_stat(path): + try: + #Check for fd. + if isinstance(path, int): + buf = os.fstat(path) + else: + buf = os.stat(path) + except Exception, err: + logging.exception("Stat failed on %s err: %s", path, str(err)) + raise + + return buf + +def do_open(path, mode): + try: + fd = open(path, mode) + except Exception, err: + logging.exception("Open failed on %s err: %s", path, str(err)) + raise + return fd + +def do_close(fd): + #fd could be file or int type. + try: + if isinstance(fd, int): + os.close(fd) + else: + fd.close() + except Exception, err: + logging.exception("Close failed on %s err: %s", fd, str(err)) + raise + return True + +def do_unlink(path, log = True): + try: + os.unlink(path) + except Exception, err: + if log: + logging.exception("Unlink failed on %s err: %s", path, str(err)) + if err.errno != errno.ENOENT: + raise + return True + +def do_rmdir(path): + try: + os.rmdir(path) + except Exception, err: + logging.exception("Rmdir failed on %s err: %s", path, str(err)) + if err.errno != errno.ENOENT: + raise + return True + +def do_rename(old_path, new_path): + try: + os.rename(old_path, new_path) + except Exception, err: + logging.exception("Rename failed on %s to %s err: %s", old_path, new_path, \ + str(err)) + raise + return True + +def mkdirs(path): + """ + Ensures the path is a directory or makes it if not. Errors if the path + exists but is a file or on permissions failure. + + :param path: path to create + """ + if not os.path.isdir(path): + try: + do_makedirs(path) + except OSError, err: + #TODO: check, isdir will fail if mounted and volume stopped. + #if err.errno != errno.EEXIST or not os.path.isdir(path) + if err.errno != errno.EEXIST: + raise + +def dir_empty(path): + """ + Return true if directory/container is empty. + :param path: Directory path. + :returns: True/False. + """ + if os.path.isdir(path): + try: + files = do_listdir(path) + except Exception, err: + logging.exception("listdir failed on %s err: %s", path, str(err)) + raise + if not files: + return True + else: + return False + else: + if not os.path.exists(path): + return True + +def rmdirs(path): + if os.path.isdir(path) and dir_empty(path): + do_rmdir(path) + else: + logging.error("rmdirs failed dir may not be empty or not valid dir") + return False diff --git a/swift/1.4.8/gluster/swift/common/middleware/__init__.py b/swift/1.4.8/gluster/swift/common/middleware/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/swift/1.4.8/gluster/swift/common/middleware/gluster.py b/swift/1.4.8/gluster/swift/common/middleware/gluster.py new file mode 100644 index 000000000..ab63c51e1 --- /dev/null +++ b/swift/1.4.8/gluster/swift/common/middleware/gluster.py @@ -0,0 +1,40 @@ +# Copyright (c) 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Noop Middleware that simply allows us to monkey patch the constraints +import gluster.swift.common.constraints + +class Gluster(object): + """ + Noop middleware for use with the proxy server to get paste.deploy to load + this middleware such that the plugin constraints monkey patch the common + constraints ahead of their use. + """ + def __init__(self, app, conf): + self.app = app + self.conf = conf + + def __call__(self, env, start_response): + return self.app(env, start_response) + + +def filter_factory(global_conf, **local_conf): + """Returns a WSGI filter app for use with paste.deploy.""" + conf = global_conf.copy() + conf.update(local_conf) + + def gluster_filter(app): + return Gluster(app, conf) + return gluster_filter diff --git a/swift/1.4.8/gluster/swift/common/utils.py b/swift/1.4.8/gluster/swift/common/utils.py new file mode 100644 index 000000000..56376f8ee --- /dev/null +++ b/swift/1.4.8/gluster/swift/common/utils.py @@ -0,0 +1,457 @@ +# Copyright (c) 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import errno +import xattr +from hashlib import md5 +import cPickle as pickle +from ConfigParser import ConfigParser, NoSectionError, NoOptionError +from swift.common.utils import normalize_timestamp, TRUE_VALUES +from gluster.swift.common.fs_utils import * +from gluster.swift.common import Glusterfs + +X_CONTENT_TYPE = 'Content-Type' +X_CONTENT_LENGTH = 'Content-Length' +X_TIMESTAMP = 'X-Timestamp' +X_PUT_TIMESTAMP = 'X-PUT-Timestamp' +X_TYPE = 'X-Type' +X_ETAG = 'ETag' +X_OBJECTS_COUNT = 'X-Object-Count' +X_BYTES_USED = 'X-Bytes-Used' +X_CONTAINER_COUNT = 'X-Container-Count' +X_OBJECT_TYPE = 'X-Object-Type' +DIR_TYPE = 'application/directory' +ACCOUNT = 'Account' +METADATA_KEY = 'user.swift.metadata' +MAX_XATTR_SIZE = 65536 +CONTAINER = 'container' +DIR = 'dir' +MARKER_DIR = 'marker_dir' +TEMP_DIR = 'tmp' +ASYNCDIR = 'async_pending' # Keep in sync with swift.obj.server.ASYNCDIR +FILE = 'file' +FILE_TYPE = 'application/octet-stream' +OBJECT = 'Object' +OBJECT_TYPE = 'application/octet-stream' +DEFAULT_UID = -1 +DEFAULT_GID = -1 +PICKLE_PROTOCOL = 2 +CHUNK_SIZE = 65536 +MEMCACHE_KEY_PREFIX = 'gluster.swift.' +MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX = MEMCACHE_KEY_PREFIX + 'account.details.' +MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX = MEMCACHE_KEY_PREFIX + 'container.details.' + + +def read_metadata(path): + """ + Helper function to read the pickled metadata from a File/Directory. + + :param path: File/Directory to read metadata from. + + :returns: dictionary of metadata + """ + metadata = None + metadata_s = '' + key = 0 + while metadata is None: + try: + metadata_s += xattr.get(path, '%s%s' % (METADATA_KEY, (key or ''))) + except IOError as err: + if err.errno == errno.ENODATA: + if key > 0: + # No errors reading the xattr keys, but since we have not + # been able to find enough chunks to get a successful + # unpickle operation, we consider the metadata lost, and + # drop the existing data so that the internal state can be + # recreated. + clean_metadata(path) + # We either could not find any metadata key, or we could find + # some keys, but were not successful in performing the + # unpickling (missing keys perhaps)? Either way, just report + # to the caller we have no metadata. + metadata = {} + else: + logging.exception("xattr.get failed on %s key %s err: %s", + path, key, str(err)) + # Note that we don't touch the keys on errors fetching the + # data since it could be a transient state. + raise + else: + try: + # If this key provides all or the remaining part of the pickle + # data, we don't need to keep searching for more keys. This + # means if we only need to store data in N xattr key/value + # pair, we only need to invoke xattr get N times. With large + # keys sizes we are shooting for N = 1. + metadata = pickle.loads(metadata_s) + assert isinstance(metadata, dict) + except EOFError, pickle.UnpicklingError: + # We still are not able recognize this existing data collected + # as a pickled object. Make sure we loop around to try to get + # more from another xattr key. + metadata = None + key += 1 + return metadata + +def write_metadata(path, metadata): + """ + Helper function to write pickled metadata for a File/Directory. + + :param path: File/Directory path to write the metadata + :param metadata: dictionary to metadata write + """ + assert isinstance(metadata, dict) + metastr = pickle.dumps(metadata, PICKLE_PROTOCOL) + key = 0 + while metastr: + try: + xattr.set(path, '%s%s' % (METADATA_KEY, key or ''), metastr[:MAX_XATTR_SIZE]) + except IOError as err: + logging.exception("xattr.set failed on %s key %s err: %s", path, key, str(err)) + raise + metastr = metastr[MAX_XATTR_SIZE:] + key += 1 + +def clean_metadata(path): + key = 0 + while True: + try: + xattr.remove(path, '%s%s' % (METADATA_KEY, (key or ''))) + except IOError as err: + if err.errno == errno.ENODATA: + break + raise + key += 1 + +def check_user_xattr(path): + if not os.path.exists(path): + return False + try: + xattr.set(path, 'user.test.key1', 'value1') + except IOError as err: + logging.exception("check_user_xattr: set failed on %s err: %s", path, str(err)) + raise + try: + xattr.remove(path, 'user.test.key1') + except IOError as err: + logging.exception("check_user_xattr: remove failed on %s err: %s", path, str(err)) + #Remove xattr may fail in case of concurrent remove. + return True + +def validate_container(metadata): + if not metadata: + logging.warn('validate_container: No metadata') + return False + + if X_TYPE not in metadata.keys() or \ + X_TIMESTAMP not in metadata.keys() or \ + X_PUT_TIMESTAMP not in metadata.keys() or \ + X_OBJECTS_COUNT not in metadata.keys() or \ + X_BYTES_USED not in metadata.keys(): + #logging.warn('validate_container: Metadata missing entries: %s' % metadata) + return False + + (value, timestamp) = metadata[X_TYPE] + if value == CONTAINER: + return True + + logging.warn('validate_container: metadata type is not CONTAINER (%r)' % (value,)) + return False + +def validate_account(metadata): + if not metadata: + logging.warn('validate_account: No metadata') + return False + + if X_TYPE not in metadata.keys() or \ + X_TIMESTAMP not in metadata.keys() or \ + X_PUT_TIMESTAMP not in metadata.keys() or \ + X_OBJECTS_COUNT not in metadata.keys() or \ + X_BYTES_USED not in metadata.keys() or \ + X_CONTAINER_COUNT not in metadata.keys(): + #logging.warn('validate_account: Metadata missing entries: %s' % metadata) + return False + + (value, timestamp) = metadata[X_TYPE] + if value == ACCOUNT: + return True + + logging.warn('validate_account: metadata type is not ACCOUNT (%r)' % (value,)) + return False + +def validate_object(metadata): + if not metadata: + logging.warn('validate_object: No metadata') + return False + + if X_TIMESTAMP not in metadata.keys() or \ + X_CONTENT_TYPE not in metadata.keys() or \ + X_ETAG not in metadata.keys() or \ + X_CONTENT_LENGTH not in metadata.keys() or \ + X_TYPE not in metadata.keys() or \ + X_OBJECT_TYPE not in metadata.keys(): + #logging.warn('validate_object: Metadata missing entries: %s' % metadata) + return False + + if metadata[X_TYPE] == OBJECT: + return True + + logging.warn('validate_object: metadata type is not OBJECT (%r)' % (metadata[X_TYPE],)) + return False + +def is_marker(metadata): + if not metadata: + logging.warn('is_marker: No metadata') + return False + + if X_OBJECT_TYPE not in metadata.keys(): + logging.warn('is_marker: X_OBJECT_TYPE missing from metadata: %s' % metadata) + return False + + if metadata[X_OBJECT_TYPE] == MARKER_DIR: + return True + else: + return False + +def _update_list(path, cont_path, src_list, reg_file=True, object_count=0, + bytes_used=0, obj_list=[]): + # strip the prefix off, also stripping the leading and trailing slashes + obj_path = path.replace(cont_path, '').strip(os.path.sep) + + for i in src_list: + if obj_path: + obj_list.append(os.path.join(obj_path, i)) + else: + obj_list.append(i) + + object_count += 1 + + if reg_file: + bytes_used += os.path.getsize(path + '/' + i) + + return object_count, bytes_used + +def update_list(path, cont_path, dirs=[], files=[], object_count=0, + bytes_used=0, obj_list=[]): + object_count, bytes_used = _update_list(path, cont_path, files, True, + object_count, bytes_used, + obj_list) + object_count, bytes_used = _update_list(path, cont_path, dirs, False, + object_count, bytes_used, + obj_list) + return object_count, bytes_used + + +class ContainerDetails(object): + def __init__(self, bytes_used, object_count, obj_list, dir_list): + self.bytes_used = bytes_used + self.object_count = object_count + self.obj_list = obj_list + self.dir_list = dir_list + + +def _get_container_details_from_fs(cont_path): + """ + get container details by traversing the filesystem + """ + bytes_used = 0 + object_count = 0 + obj_list = [] + dir_list = [] + + if os.path.isdir(cont_path): + for (path, dirs, files) in os.walk(cont_path): + object_count, bytes_used = update_list(path, cont_path, dirs, files, + object_count, bytes_used, + obj_list) + + dir_list.append((path, do_stat(path).st_mtime)) + + return ContainerDetails(bytes_used, object_count, obj_list, dir_list) + +def get_container_details(cont_path, memcache=None): + """ + Return object_list, object_count and bytes_used. + """ + mkey = '' + if memcache: + mkey = MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + cont_path + cd = memcache.get(mkey) + if cd: + if not cd.dir_list: + cd = None + else: + for (path, mtime) in cd.dir_list: + if mtime != do_stat(path).st_mtime: + cd = None + else: + cd = None + if not cd: + cd = _get_container_details_from_fs(cont_path) + if memcache: + memcache.set(mkey, cd) + return cd.obj_list, cd.object_count, cd.bytes_used + + +class AccountDetails(object): + """ A simple class to store the three pieces of information associated + with an account: + + 1. The last known modification time + 2. The count of containers in the following list + 3. The list of containers + """ + def __init__(self, mtime, container_count, container_list): + self.mtime = mtime + self.container_count = container_count + self.container_list = container_list + + +def _get_account_details_from_fs(acc_path, acc_stats): + container_list = [] + container_count = 0 + + if not acc_stats: + acc_stats = do_stat(acc_path) + is_dir = (acc_stats.st_mode & 0040000) != 0 + if is_dir: + for name in do_listdir(acc_path): + if name.lower() == TEMP_DIR \ + or name.lower() == ASYNCDIR \ + or not os.path.isdir(os.path.join(acc_path, name)): + continue + container_count += 1 + container_list.append(name) + + return AccountDetails(acc_stats.st_mtime, container_count, container_list) + +def get_account_details(acc_path, memcache=None): + """ + Return container_list and container_count. + """ + acc_stats = None + mkey = '' + if memcache: + mkey = MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX + acc_path + ad = memcache.get(mkey) + if ad: + # FIXME: Do we really need to stat the file? If we are object + # only, then we can track the other Swift HTTP APIs that would + # modify the account and invalidate the cached entry there. If we + # are not object only, are we even called on this path? + acc_stats = do_stat(acc_path) + if ad.mtime != acc_stats.st_mtime: + ad = None + else: + ad = None + if not ad: + ad = _get_account_details_from_fs(acc_path, acc_stats) + if memcache: + memcache.set(mkey, ad) + return ad.container_list, ad.container_count + +def _get_etag(path): + etag = md5() + with open(path, 'rb') as fp: + while True: + chunk = fp.read(CHUNK_SIZE) + if chunk: + etag.update(chunk) + else: + break + return etag.hexdigest() + +def get_object_metadata(obj_path): + """ + Return metadata of object. + """ + try: + stats = os.stat(obj_path) + except OSError as e: + if e.errno != errno.ENOENT: + raise + metadata = {} + else: + is_dir = (stats.st_mode & 0040000) != 0 + metadata = { + X_TYPE: OBJECT, + X_TIMESTAMP: normalize_timestamp(stats.st_ctime), + X_CONTENT_TYPE: DIR_TYPE if is_dir else FILE_TYPE, + X_OBJECT_TYPE: DIR if is_dir else FILE, + X_CONTENT_LENGTH: 0 if is_dir else stats.st_size, + X_ETAG: md5().hexdigest() if is_dir else _get_etag(obj_path), + } + return metadata + +def _add_timestamp(metadata_i): + # At this point we have a simple key/value dictionary, turn it into + # key/(value,timestamp) pairs. + timestamp = 0 + metadata = {} + for key, value_i in metadata_i.iteritems(): + if not isinstance(value_i, tuple): + metadata[key] = (value_i, timestamp) + else: + metadata[key] = value_i + return metadata + +def get_container_metadata(cont_path, memcache=None): + objects = [] + object_count = 0 + bytes_used = 0 + objects, object_count, bytes_used = get_container_details(cont_path, memcache) + metadata = {X_TYPE: CONTAINER, + X_TIMESTAMP: normalize_timestamp(os.path.getctime(cont_path)), + X_PUT_TIMESTAMP: normalize_timestamp(os.path.getmtime(cont_path)), + X_OBJECTS_COUNT: object_count, + X_BYTES_USED: bytes_used} + return _add_timestamp(metadata) + +def get_account_metadata(acc_path, memcache=None): + containers = [] + container_count = 0 + containers, container_count = get_account_details(acc_path, memcache) + metadata = {X_TYPE: ACCOUNT, + X_TIMESTAMP: normalize_timestamp(os.path.getctime(acc_path)), + X_PUT_TIMESTAMP: normalize_timestamp(os.path.getmtime(acc_path)), + X_OBJECTS_COUNT: 0, + X_BYTES_USED: 0, + X_CONTAINER_COUNT: container_count} + return _add_timestamp(metadata) + +def restore_metadata(path, metadata): + meta_orig = read_metadata(path) + if meta_orig: + meta_new = meta_orig.copy() + meta_new.update(metadata) + else: + meta_new = metadata + if meta_orig != meta_new: + write_metadata(path, meta_new) + return meta_new + +def create_object_metadata(obj_path): + metadata = get_object_metadata(obj_path) + return restore_metadata(obj_path, metadata) + +def create_container_metadata(cont_path, memcache=None): + metadata = get_container_metadata(cont_path, memcache) + return restore_metadata(cont_path, metadata) + +def create_account_metadata(acc_path, memcache=None): + metadata = get_account_metadata(acc_path, memcache) + return restore_metadata(acc_path, metadata) diff --git a/swift/1.4.8/gluster/swift/container/__init__.py b/swift/1.4.8/gluster/swift/container/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/swift/1.4.8/gluster/swift/container/server.py b/swift/1.4.8/gluster/swift/container/server.py new file mode 100644 index 000000000..e2a197302 --- /dev/null +++ b/swift/1.4.8/gluster/swift/container/server.py @@ -0,0 +1,46 @@ +# Copyright (c) 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Container Server for Gluster Swift UFO """ + +# Simply importing this monkey patches the constraint handling to fit our +# needs +import gluster.swift.common.constraints + +from swift.container import server +from gluster.swift.common.DiskDir import DiskDir + + +class ContainerController(server.ContainerController): + def _get_container_broker(self, drive, part, account, container): + """ + Overriden to provide the GlusterFS specific broker that talks to + Gluster for the information related to servicing a given request + instead of talking to a database. + + :param drive: drive that holds the container + :param part: partition the container is in + :param account: account name + :param container: container name + :returns: DiskDir object + """ + return DiskDir(self.root, account, container, self.logger) + + +def app_factory(global_conf, **local_conf): + """paste.deploy app factory for creating WSGI container server apps.""" + conf = global_conf.copy() + conf.update(local_conf) + return ContainerController(conf) diff --git a/swift/1.4.8/gluster/swift/obj/__init__.py b/swift/1.4.8/gluster/swift/obj/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/swift/1.4.8/gluster/swift/obj/server.py b/swift/1.4.8/gluster/swift/obj/server.py new file mode 100644 index 000000000..43cdd8890 --- /dev/null +++ b/swift/1.4.8/gluster/swift/obj/server.py @@ -0,0 +1,33 @@ +# Copyright (c) 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Object Server for Gluster Swift UFO """ + +# Simply importing this monkey patches the constraint handling to fit our +# needs +import gluster.swift.common.constraints + +from swift.obj import server +from gluster.swift.common.DiskFile import Gluster_DiskFile + +# Monkey patch the object server module to use Gluster's DiskFile definition +server.DiskFile = Gluster_DiskFile + + +def app_factory(global_conf, **local_conf): + """paste.deploy app factory for creating WSGI object server apps""" + conf = global_conf.copy() + conf.update(local_conf) + return server.ObjectController(conf) diff --git a/swift/1.4.8/gluster/swift/proxy/__init__.py b/swift/1.4.8/gluster/swift/proxy/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/swift/1.4.8/gluster/swift/proxy/server.py b/swift/1.4.8/gluster/swift/proxy/server.py new file mode 100644 index 000000000..792a97df9 --- /dev/null +++ b/swift/1.4.8/gluster/swift/proxy/server.py @@ -0,0 +1,27 @@ +# Copyright (c) 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Simply importing this monkey patches the constraint handling to fit our +# needs +import gluster.swift.common.constraints + +from swift.proxy import server + +def app_factory(global_conf, **local_conf): + """paste.deploy app factory for creating WSGI proxy apps.""" + conf = global_conf.copy() + conf.update(local_conf) + return server.Application(conf) diff --git a/swift/1.4.8/plugins/DiskDir.py b/swift/1.4.8/plugins/DiskDir.py deleted file mode 100644 index b12827c58..000000000 --- a/swift/1.4.8/plugins/DiskDir.py +++ /dev/null @@ -1,499 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os, errno - -from swift.plugins.utils import clean_metadata, dir_empty, rmdirs, mkdirs, \ - validate_account, validate_container, is_marker, get_container_details, \ - get_account_details, get_container_metadata, create_container_metadata, \ - create_account_metadata, DEFAULT_GID, DEFAULT_UID, validate_object, \ - create_object_metadata, read_metadata, write_metadata, X_CONTENT_TYPE, \ - X_CONTENT_LENGTH, X_TIMESTAMP, X_PUT_TIMESTAMP, X_TYPE, X_ETAG, \ - X_OBJECTS_COUNT, X_BYTES_USED, X_CONTAINER_COUNT, CONTAINER -from swift.plugins import Glusterfs - -from swift.common.constraints import CONTAINER_LISTING_LIMIT -from swift.common.utils import normalize_timestamp, TRUE_VALUES - - -DATADIR = 'containers' - -# Create a dummy db_file in /etc/swift -_unittests_enabled = os.getenv('GLUSTER_UNIT_TEST_ENABLED', 'no') -if _unittests_enabled in TRUE_VALUES: - _tmp_dir = '/tmp/gluster_unit_tests' - try: - os.mkdir(_tmp_dir) - except OSError as e: - if e.errno != errno.EEXIST: - raise - _db_file = os.path.join(_tmp_dir, 'db_file.db') -else: - _db_file = '/etc/swift/db_file.db' -if not os.path.exists(_db_file): - file(_db_file, 'w+') - - -def _read_metadata(dd): - """ Filter read metadata so that it always returns a tuple that includes - some kind of timestamp. With 1.4.8 of the Swift integration the - timestamps were not stored. Here we fabricate timestamps for volumes - where the existing data has no timestamp (that is, stored data is not - a tuple), allowing us a measure of backward compatibility. - - FIXME: At this time it does not appear that the timestamps on each - metadata are used for much, so this should not hurt anything. - """ - metadata_i = read_metadata(dd) - metadata = {} - timestamp = 0 - for key, value in metadata_i.iteritems(): - if not isinstance(value, tuple): - value = (value, timestamp) - metadata[key] = value - return metadata - - -class DiskCommon(object): - def is_deleted(self): - return not os.path.exists(self.datadir) - - def filter_prefix(self, objects, prefix): - """ - Accept sorted list. - """ - found = 0 - filtered_objs = [] - for object_name in objects: - if object_name.startswith(prefix): - filtered_objs.append(object_name) - found = 1 - else: - if found: - break - return filtered_objs - - def filter_delimiter(self, objects, delimiter, prefix): - """ - Accept sorted list. - Objects should start with prefix. - """ - filtered_objs=[] - for object_name in objects: - tmp_obj = object_name.replace(prefix, '', 1) - sufix = tmp_obj.split(delimiter, 1) - new_obj = prefix + sufix[0] - if new_obj and new_obj not in filtered_objs: - filtered_objs.append(new_obj) - - return filtered_objs - - def filter_marker(self, objects, marker): - """ - TODO: We can traverse in reverse order to optimize. - Accept sorted list. - """ - filtered_objs=[] - found = 0 - if objects[-1] < marker: - return filtered_objs - for object_name in objects: - if object_name > marker: - filtered_objs.append(object_name) - - return filtered_objs - - def filter_end_marker(self, objects, end_marker): - """ - Accept sorted list. - """ - filtered_objs=[] - for object_name in objects: - if object_name < end_marker: - filtered_objs.append(object_name) - else: - break - - return filtered_objs - - def filter_limit(self, objects, limit): - filtered_objs=[] - for i in range(0, limit): - filtered_objs.append(objects[i]) - - return filtered_objs - - -class DiskDir(DiskCommon): - """ - Manage object files on disk. - - :param path: path to devices on the node - :param account: account name for the object - :param container: container name for the object - :param logger: account or container server logging object - :param uid: user ID container object should assume - :param gid: group ID container object should assume - """ - - def __init__(self, path, account, container, logger, - uid=DEFAULT_UID, gid=DEFAULT_GID): - self.root = path - if container: - self.container = container - else: - self.container = None - if self.container: - self.datadir = os.path.join(path, account, self.container) - else: - self.datadir = os.path.join(path, account) - # Note that the account name has a one-to-one mapping to the gluster - # mount point, or volume name. - self.account = account - assert logger is not None - self.logger = logger - self.metadata = {} - self.container_info = None - self.object_info = None - self.uid = int(uid) - self.gid = int(gid) - self.db_file = _db_file - self.dir_exists = os.path.exists(self.datadir) - if self.dir_exists: - try: - self.metadata = _read_metadata(self.datadir) - except EOFError: - create_container_metadata(self.datadir) - else: - return - if self.container: - if not self.metadata: - create_container_metadata(self.datadir) - self.metadata = _read_metadata(self.datadir) - else: - if not validate_container(self.metadata): - create_container_metadata(self.datadir) - self.metadata = _read_metadata(self.datadir) - else: - if not self.metadata: - create_account_metadata(self.datadir) - self.metadata = _read_metadata(self.datadir) - else: - if not validate_account(self.metadata): - create_account_metadata(self.datadir) - self.metadata = _read_metadata(self.datadir) - - def empty(self): - return dir_empty(self.datadir) - - def delete(self): - if self.empty(): - #For delete account. - if os.path.ismount(self.datadir): - clean_metadata(self.datadir) - else: - rmdirs(self.datadir) - self.dir_exists = False - - def put_metadata(self, metadata): - """ - Write metadata to directory/container. - """ - write_metadata(self.datadir, metadata) - self.metadata = metadata - - def put(self, metadata): - """ - Create and write metatdata to directory/container. - :param metadata: Metadata to write. - """ - if not self.dir_exists: - mkdirs(self.datadir) - - os.chown(self.datadir, self.uid, self.gid) - write_metadata(self.datadir, metadata) - self.metadata = metadata - self.dir_exists = True - - def put_obj(self, content_length, timestamp): - ocnt = self.metadata[X_OBJECTS_COUNT][0] - self.metadata[X_OBJECTS_COUNT] = (int(ocnt) + 1, timestamp) - self.metadata[X_PUT_TIMESTAMP] = timestamp - bused = self.metadata[X_BYTES_USED][0] - self.metadata[X_BYTES_USED] = (int(bused) + int(content_length), timestamp) - #TODO: define update_metadata instad of writing whole metadata again. - self.put_metadata(self.metadata) - - def delete_obj(self, content_length): - ocnt, timestamp = self.metadata[X_OBJECTS_COUNT][0] - self.metadata[X_OBJECTS_COUNT] = (int(ocnt) - 1, timestamp) - bused, timestamp = self.metadata[X_BYTES_USED] - self.metadata[X_BYTES_USED] = (int(bused) - int(content_length), timestamp) - self.put_metadata(self.metadata) - - def put_container(self, container, put_timestamp, del_timestamp, object_count, bytes_used): - """ - For account server. - """ - self.metadata[X_OBJECTS_COUNT] = (0, put_timestamp) - self.metadata[X_BYTES_USED] = (0, put_timestamp) - ccnt = self.metadata[X_CONTAINER_COUNT][0] - self.metadata[X_CONTAINER_COUNT] = (int(ccnt) + 1, put_timestamp) - self.metadata[X_PUT_TIMESTAMP] = (1, put_timestamp) - self.put_metadata(self.metadata) - - def delete_container(self, object_count, bytes_used): - """ - For account server. - """ - self.metadata[X_OBJECTS_COUNT] = (0, 0) - self.metadata[X_BYTES_USED] = (0, 0) - ccnt, timestamp = self.metadata[X_CONTAINER_COUNT] - self.metadata[X_CONTAINER_COUNT] = (int(ccnt) - 1, timestamp) - self.put_metadata(self.metadata) - - def unlink(self): - """ - Remove directory/container if empty. - """ - if dir_empty(self.datadir): - rmdirs(self.datadir) - - def list_objects_iter(self, limit, marker, end_marker, - prefix, delimiter, path): - """ - Returns tuple of name, created_at, size, content_type, etag. - """ - if path: - prefix = path = path.rstrip('/') + '/' - delimiter = '/' - if delimiter and not prefix: - prefix = '' - - self.update_object_count() - - objects, object_count, bytes_used = self.object_info - - if objects: - objects.sort() - - if objects and prefix: - objects = self.filter_prefix(objects, prefix) - - if objects and delimiter: - objects = self.filter_delimiter(objects, delimiter, prefix) - - if objects and marker: - objects = self.filter_marker(objects, marker) - - if objects and end_marker: - objects = self.filter_end_marker(objects, end_marker) - - if objects and limit: - if len(objects) > limit: - objects = self.filter_limit(objects, limit) - - container_list = [] - if objects: - for obj in objects: - list_item = [] - list_item.append(obj) - obj_path = os.path.join(self.datadir, obj) - metadata = read_metadata(obj_path) - if not metadata or not validate_object(metadata): - metadata = create_object_metadata(obj_path) - if metadata: - list_item.append(metadata[X_TIMESTAMP]) - list_item.append(int(metadata[X_CONTENT_LENGTH])) - list_item.append(metadata[X_CONTENT_TYPE]) - list_item.append(metadata[X_ETAG]) - container_list.append(list_item) - - return container_list - - def update_object_count(self): - if not self.object_info: - self.object_info = get_container_details(self.datadir) - - objects, object_count, bytes_used = self.object_info - - if X_OBJECTS_COUNT not in self.metadata \ - or int(self.metadata[X_OBJECTS_COUNT][0]) != object_count \ - or X_BYTES_USED not in self.metadata \ - or int(self.metadata[X_BYTES_USED][0]) != bytes_used: - self.metadata[X_OBJECTS_COUNT] = (object_count, 0) - self.metadata[X_BYTES_USED] = (bytes_used, 0) - write_metadata(self.datadir, self.metadata) - - def update_container_count(self): - if not self.container_info: - self.container_info = get_account_details(self.datadir) - - containers, container_count = self.container_info - - if X_CONTAINER_COUNT not in self.metadata \ - or int(self.metadata[X_CONTAINER_COUNT][0]) != container_count: - self.metadata[X_CONTAINER_COUNT] = (container_count, 0) - write_metadata(self.datadir, self.metadata) - - def get_info(self, include_metadata=False): - """ - Get global data for the container. - :returns: dict with keys: account, container, object_count, bytes_used, - hash, id, created_at, put_timestamp, delete_timestamp, - reported_put_timestamp, reported_delete_timestamp, - reported_object_count, and reported_bytes_used. - If include_metadata is set, metadata is included as a key - pointing to a dict of tuples of the metadata - """ - # TODO: delete_timestamp, reported_put_timestamp - # reported_delete_timestamp, reported_object_count, - # reported_bytes_used, created_at - if not Glusterfs.OBJECT_ONLY: - # If we are not configured for object only environments, we should - # update the object counts in case they changed behind our back. - self.update_object_count() - - data = {'account' : self.account, 'container' : self.container, - 'object_count' : self.metadata.get(X_OBJECTS_COUNT, ('0', 0))[0], - 'bytes_used' : self.metadata.get(X_BYTES_USED, ('0',0))[0], - 'hash': '', 'id' : '', 'created_at' : '1', - 'put_timestamp' : self.metadata.get(X_PUT_TIMESTAMP, ('0',0))[0], - 'delete_timestamp' : '1', - 'reported_put_timestamp' : '1', 'reported_delete_timestamp' : '1', - 'reported_object_count' : '1', 'reported_bytes_used' : '1'} - if include_metadata: - data['metadata'] = self.metadata - return data - - def put_object(self, name, timestamp, size, content_type, - etag, deleted=0): - # TODO: Implement the specifics of this func. - pass - - def initialize(self, timestamp): - pass - - def update_put_timestamp(self, timestamp): - """ - Create the container if it doesn't exist and update the timestamp - """ - if not os.path.exists(self.datadir): - self.put(self.metadata) - - def delete_object(self, name, timestamp): - # TODO: Implement the delete object - pass - - def delete_db(self, timestamp): - """ - Delete the container - """ - self.unlink() - - def update_metadata(self, metadata): - assert self.metadata, "Valid container/account metadata should have been created by now" - if metadata: - new_metadata = self.metadata.copy() - new_metadata.update(metadata) - if new_metadata != self.metadata: - write_metadata(self.datadir, new_metadata) - self.metadata = new_metadata - - -class DiskAccount(DiskDir): - def __init__(self, root, account, logger): - super(DiskAccount, self).__init__(root, account, None, logger) - assert self.dir_exists - - def list_containers_iter(self, limit, marker, end_marker, - prefix, delimiter): - """ - Return tuple of name, object_count, bytes_used, 0(is_subdir). - Used by account server. - """ - if delimiter and not prefix: - prefix = '' - - self.update_container_count() - - containers, container_count = self.container_info - - if containers: - containers.sort() - - if containers and prefix: - containers = self.filter_prefix(containers, prefix) - - if containers and delimiter: - containers = self.filter_delimiter(containers, delimiter, prefix) - - if containers and marker: - containers = self.filter_marker(containers, marker) - - if containers and end_marker: - containers = self.filter_end_marker(containers, end_marker) - - if containers and limit: - if len(containers) > limit: - containers = self.filter_limit(containers, limit) - - account_list = [] - if containers: - for cont in containers: - list_item = [] - metadata = None - list_item.append(cont) - cont_path = os.path.join(self.datadir, cont) - metadata = _read_metadata(cont_path) - if not metadata or not validate_container(metadata): - metadata = create_container_metadata(cont_path) - - if metadata: - list_item.append(metadata[X_OBJECTS_COUNT][0]) - list_item.append(metadata[X_BYTES_USED][0]) - list_item.append(0) - account_list.append(list_item) - - return account_list - - def get_info(self, include_metadata=False): - """ - Get global data for the account. - :returns: dict with keys: account, created_at, put_timestamp, - delete_timestamp, container_count, object_count, - bytes_used, hash, id - """ - if not Glusterfs.OBJECT_ONLY: - # If we are not configured for object only environments, we should - # update the container counts in case they changed behind our back. - self.update_container_count() - - data = {'account' : self.account, 'created_at' : '1', - 'put_timestamp' : '1', 'delete_timestamp' : '1', - 'container_count' : self.metadata.get(X_CONTAINER_COUNT, (0,0))[0], - 'object_count' : self.metadata.get(X_OBJECTS_COUNT, (0,0))[0], - 'bytes_used' : self.metadata.get(X_BYTES_USED, (0,0))[0], - 'hash' : '', 'id' : ''} - - if include_metadata: - data['metadata'] = self.metadata - return data - - def get_container_timestamp(self, container): - cont_path = os.path.join(self.datadir, container) - metadata = read_metadata(cont_path) - - return int(metadata.get(X_PUT_TIMESTAMP, ('0',0))[0]) or None diff --git a/swift/1.4.8/plugins/DiskFile.py b/swift/1.4.8/plugins/DiskFile.py deleted file mode 100644 index a57b2ece3..000000000 --- a/swift/1.4.8/plugins/DiskFile.py +++ /dev/null @@ -1,323 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -from eventlet import tpool -from tempfile import mkstemp -from contextlib import contextmanager -from swift.common.utils import normalize_timestamp, renamer -from swift.plugins.utils import mkdirs, rmdirs, validate_object, \ - create_object_metadata, do_open, do_close, do_unlink, do_chown, \ - do_stat, do_listdir, read_metadata, write_metadata -from swift.plugins.utils import X_CONTENT_TYPE, X_CONTENT_LENGTH, X_TIMESTAMP, \ - X_PUT_TIMESTAMP, X_TYPE, X_ETAG, X_OBJECTS_COUNT, X_BYTES_USED, \ - X_OBJECT_TYPE, FILE, DIR, MARKER_DIR, OBJECT, DIR_TYPE, FILE_TYPE, \ - DEFAULT_UID, DEFAULT_GID - -import logging -from swift.obj.server import DiskFile - - -DATADIR = 'objects' -ASYNCDIR = 'async_pending' -KEEP_CACHE_SIZE = (5 * 1024 * 1024) -# keep these lower-case -DISALLOWED_HEADERS = set('content-length content-type deleted etag'.split()) - - -class Gluster_DiskFile(DiskFile): - """ - Manage object files on disk. - - :param path: path to devices on the node/mount path for UFO. - :param device: device name/account_name for UFO. - :param partition: partition on the device the object lives in - :param account: account name for the object - :param container: container name for the object - :param obj: object name for the object - :param keep_data_fp: if True, don't close the fp, otherwise close it - :param disk_chunk_Size: size of chunks on file reads - :param uid: user ID disk object should assume (file or directory) - :param gid: group ID disk object should assume (file or directory) - """ - - def __init__(self, path, device, partition, account, container, obj, - logger, keep_data_fp=False, disk_chunk_size=65536, - uid=DEFAULT_UID, gid=DEFAULT_GID): - self.disk_chunk_size = disk_chunk_size - device = account - #Don't support obj_name ending/begining with '/', like /a, a/, /a/b/ etc - obj = obj.strip('/') - if '/' in obj: - self.obj_path, self.obj = obj.rsplit('/', 1) - else: - self.obj_path = '' - self.obj = obj - - if self.obj_path: - self.name = '/'.join((container, self.obj_path)) - else: - self.name = container - #Absolute path for obj directory. - self.datadir = os.path.join(path, device, self.name) - self.device_path = os.path.join(path, device) - self.container_path = os.path.join(path, device, container) - self.tmpdir = os.path.join(path, device, 'tmp') - self.logger = logger - self.metadata = {} - self.data_file = None - self.fp = None - self.iter_etag = None - self.started_at_0 = False - self.read_to_eof = False - self.quarantined_dir = None - self.keep_cache = False - self.is_dir = False - self.is_valid = True - self.uid = int(uid) - self.gid = int(gid) - if not os.path.exists(self.datadir + '/' + self.obj): - return - - self.data_file = os.path.join(self.datadir, self.obj) - self.metadata = read_metadata(self.datadir + '/' + self.obj) - if not self.metadata: - create_object_metadata(self.datadir + '/' + self.obj) - self.metadata = read_metadata(self.datadir + '/' + self.obj) - - if not validate_object(self.metadata): - create_object_metadata(self.datadir + '/' + self.obj) - self.metadata = read_metadata(self.datadir + '/' + - self.obj) - - self.filter_metadata() - - if os.path.isdir(self.datadir + '/' + self.obj): - self.is_dir = True - else: - self.fp = do_open(self.data_file, 'rb') - if not keep_data_fp: - self.close(verify_file=False) - - def close(self, verify_file=True): - """ - Close the file. Will handle quarantining file if necessary. - - :param verify_file: Defaults to True. If false, will not check - file to see if it needs quarantining. - """ - #Marker directory - if self.is_dir: - return - if self.fp: - do_close(self.fp) - self.fp = None - - def is_deleted(self): - """ - Check if the file is deleted. - - :returns: True if the file doesn't exist or has been flagged as - deleted. - """ - return not self.data_file - - def create_dir_object(self, dir_path): - #TODO: if object already exists??? - if os.path.exists(dir_path) and not os.path.isdir(dir_path): - self.logger.error("Deleting file %s", dir_path) - do_unlink(dir_path) - #If dir aleady exist just override metadata. - mkdirs(dir_path) - do_chown(dir_path, self.uid, self.gid) - create_object_metadata(dir_path) - return True - - def put_metadata(self, metadata): - obj_path = self.datadir + '/' + self.obj - write_metadata(obj_path, metadata) - self.metadata = metadata - - def put(self, fd, tmppath, metadata, extension=''): - """ - Finalize writing the file on disk, and renames it from the temp file to - the real location. This should be called after the data has been - written to the temp file. - - :params fd: file descriptor of the temp file - :param tmppath: path to the temporary file being used - :param metadata: dictionary of metadata to be written - :param extention: extension to be used when making the file - """ - if extension == '.ts': - # TombStone marker (deleted) - return True - - # Fix up the metadata to ensure it has a proper value for the - # Content-Type metadata, as well as an X_TYPE and X_OBJECT_TYPE - # metadata values. - - content_type = metadata['Content-Type'] - if not content_type: - metadata['Content-Type'] = FILE_TYPE - x_object_type = FILE - else: - x_object_type = MARKER_DIR if content_type.lower() == DIR_TYPE else FILE - metadata[X_TYPE] = OBJECT - metadata[X_OBJECT_TYPE] = x_object_type - - if extension == '.meta': - # Metadata recorded separately from the file - self.put_metadata(metadata) - return True - - extension = '' - - if metadata[X_OBJECT_TYPE] == MARKER_DIR: - self.create_dir_object(os.path.join(self.datadir, self.obj)) - self.put_metadata(metadata) - self.data_file = self.datadir + '/' + self.obj - return True - - # Check if directory already exists. - if self.is_dir: - self.logger.error('Directory already exists %s/%s' % \ - (self.datadir , self.obj)) - return False - - timestamp = normalize_timestamp(metadata[X_TIMESTAMP]) - write_metadata(tmppath, metadata) - if X_CONTENT_LENGTH in metadata: - self.drop_cache(fd, 0, int(metadata[X_CONTENT_LENGTH])) - tpool.execute(os.fsync, fd) - if self.obj_path: - dir_objs = self.obj_path.split('/') - tmp_path = '' - if len(dir_objs): - for dir_name in dir_objs: - if tmp_path: - tmp_path = tmp_path + '/' + dir_name - else: - tmp_path = dir_name - if not self.create_dir_object(os.path.join(self.container_path, - tmp_path)): - self.logger.error("Failed in subdir %s",\ - os.path.join(self.container_path,tmp_path)) - return False - - renamer(tmppath, os.path.join(self.datadir, - self.obj + extension)) - do_chown(os.path.join(self.datadir, self.obj + extension), \ - self.uid, self.gid) - self.metadata = metadata - self.data_file = self.datadir + '/' + self.obj + extension - return True - - def unlinkold(self, timestamp): - """ - Remove any older versions of the object file. Any file that has an - older timestamp than timestamp will be deleted. - - :param timestamp: timestamp to compare with each file - """ - if self.metadata and self.metadata['X-Timestamp'] != timestamp: - self.unlink() - - def unlink(self): - """ - Remove the file. - """ - #Marker dir. - if self.is_dir: - rmdirs(os.path.join(self.datadir, self.obj)) - if not os.path.isdir(os.path.join(self.datadir, self.obj)): - self.metadata = {} - self.data_file = None - else: - logging.error('Unable to delete dir %s' % os.path.join(self.datadir, self.obj)) - return - - for fname in do_listdir(self.datadir): - if fname == self.obj: - try: - do_unlink(os.path.join(self.datadir, fname)) - except OSError, err: - if err.errno != errno.ENOENT: - raise - - #Remove entire path for object. - #remove_dir_path(self.obj_path, self.container_path) - - self.metadata = {} - self.data_file = None - - def get_data_file_size(self): - """ - Returns the os.path.getsize for the file. Raises an exception if this - file does not match the Content-Length stored in the metadata. Or if - self.data_file does not exist. - - :returns: file size as an int - :raises DiskFileError: on file size mismatch. - :raises DiskFileNotExist: on file not existing (including deleted) - """ - #Marker directory. - if self.is_dir: - return 0 - try: - file_size = 0 - if self.data_file: - file_size = os.path.getsize(self.data_file) - if X_CONTENT_LENGTH in self.metadata: - metadata_size = int(self.metadata[X_CONTENT_LENGTH]) - if file_size != metadata_size: - self.metadata[X_CONTENT_LENGTH] = file_size - self.update_object(self.metadata) - - return file_size - except OSError, err: - if err.errno != errno.ENOENT: - raise - raise DiskFileNotExist('Data File does not exist.') - - def update_object(self, metadata): - obj_path = self.datadir + '/' + self.obj - write_metadata(obj_path, metadata) - self.metadata = metadata - - def filter_metadata(self): - if X_TYPE in self.metadata: - self.metadata.pop(X_TYPE) - if X_OBJECT_TYPE in self.metadata: - self.metadata.pop(X_OBJECT_TYPE) - - @contextmanager - def mkstemp(self): - """Contextmanager to make a temporary file.""" - - if not os.path.exists(self.tmpdir): - mkdirs(self.tmpdir) - fd, tmppath = mkstemp(dir=self.tmpdir) - try: - yield fd, tmppath - finally: - try: - os.close(fd) - except OSError: - pass - try: - os.unlink(tmppath) - except OSError: - pass diff --git a/swift/1.4.8/plugins/Glusterfs.py b/swift/1.4.8/plugins/Glusterfs.py deleted file mode 100644 index c176a247b..000000000 --- a/swift/1.4.8/plugins/Glusterfs.py +++ /dev/null @@ -1,131 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging -import os, fcntl, time -from ConfigParser import ConfigParser -from swift.common.utils import TRUE_VALUES -from swift.plugins.fs_utils import mkdirs - - -# -# Read the fs.conf file once at startup (module load) -# -_fs_conf = ConfigParser() -AUTH_ACCOUNT = 'auth' -MOUNT_IP = 'localhost' -REMOTE_CLUSTER = False -OBJECT_ONLY = False -if _fs_conf.read(os.path.join('/etc/swift', 'fs.conf')): - try: - AUTH_ACCOUNT = _fs_conf.get('DEFAULT', 'auth_account', 'auth') - except (NoSectionError, NoOptionError): - pass - try: - MOUNT_IP = _fs_conf.get('DEFAULT', 'mount_ip', 'localhost') - except (NoSectionError, NoOptionError): - pass - try: - REMOTE_CLUSTER = _fs_conf.get('DEFAULT', 'remote_cluster', False) in TRUE_VALUES - except (NoSectionError, NoOptionError): - pass - try: - OBJECT_ONLY = _fs_conf.get('DEFAULT', 'object_only', "no") in TRUE_VALUES - except (NoSectionError, NoOptionError): - pass -NAME = 'glusterfs' - - -def _busy_wait(full_mount_path): - # Iterate for definite number of time over a given - # interval for successful mount - for i in range(0, 5): - if os.path.ismount(os.path.join(full_mount_path)): - return True - time.sleep(2) - logging.error('Busy wait for mount timed out for mount %s', full_mount_path) - return False - -def mount(root, drive): - # FIXME: Possible thundering herd problem here - - el = _get_export_list() - for export in el: - if drive == export: - break - else: - logging.error('No export found in %r matching drive %s', el, drive) - return False - - # NOTE: root is typically the default value of /mnt/gluster-object - full_mount_path = os.path.join(root, drive) - if not os.path.isdir(full_mount_path): - mkdirs(full_mount_path) - - pid_dir = "/var/lib/glusterd/vols/%s/run/" % drive - pid_file = os.path.join(pid_dir, 'swift.pid'); - - if not os.path.exists(pid_dir): - mkdirs(pid_dir) - - fd = os.open(pid_file, os.O_CREAT|os.O_RDWR) - with os.fdopen(fd, 'r+b') as f: - try: - fcntl.lockf(f, fcntl.LOCK_EX|fcntl.LOCK_NB) - except: - ex = sys.exc_info()[1] - if isinstance(ex, IOError) and ex.errno in (EACCES, EAGAIN): - # This means that some other process is mounting the - # filesystem, so wait for the mount process to complete - return _busy_wait(full_mount_path) - - mnt_cmd = 'mount -t glusterfs %s:%s %s' % (MOUNT_IP, export, \ - full_mount_path) - if os.system(mnt_cmd) or not _busy_wait(full_mount_path): - logging.error('Mount failed %s: %s', NAME, mnt_cmd) - return False - return True - -def unmount(full_mount_path): - # FIXME: Possible thundering herd problem here - - umnt_cmd = 'umount %s 2>> /dev/null' % full_mount_path - if os.system(umnt_cmd): - logging.error('Unable to unmount %s %s' % (full_mount_path, NAME)) - -def _get_export_list(): - if REMOTE_CLUSTER: - cmnd = 'ssh %s gluster volume info' % MOUNT_IP - else: - cmnd = 'gluster volume info' - - export_list = [] - - if os.system(cmnd + ' >> /dev/null'): - if REMOTE_CLUSTER: - logging.error('Getting volume info failed %s, make sure to have '\ - 'passwordless ssh on %s', NAME, MOUNT_IP) - else: - logging.error('Getting volume failed %s', NAME) - else: - fp = os.popen(cmnd) - while True: - item = fp.readline() - if not item: - break - item = item.strip('\n').strip(' ') - if item.lower().startswith('volume name:'): - export_list.append(item.split(':')[1].strip(' ')) - - return export_list diff --git a/swift/1.4.8/plugins/__init__.py b/swift/1.4.8/plugins/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/swift/1.4.8/plugins/bin/gluster-swift-gen-builders b/swift/1.4.8/plugins/bin/gluster-swift-gen-builders deleted file mode 100755 index b89cd15fb..000000000 --- a/swift/1.4.8/plugins/bin/gluster-swift-gen-builders +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -function create { - swift-ring-builder $1 create 0 1 1 - swift-ring-builder $1 add z1-127.0.0.1:$2/$3_ 100.0 - swift-ring-builder $1 rebalance - swift-ring-builder $1 -} - -if [ "$1x" = "x" ]; then - echo "Please specify the gluster volume name to use." - exit 1 -fi - -# Note that these port numbers must match the configured values for the -# various servers in their configuration files. -create account.builder 6012 $1 -create container.builder 6011 $1 -create object.builder 6010 $1 diff --git a/swift/1.4.8/plugins/conf/account-server/1.conf-gluster b/swift/1.4.8/plugins/conf/account-server/1.conf-gluster deleted file mode 100644 index c7a4c10a4..000000000 --- a/swift/1.4.8/plugins/conf/account-server/1.conf-gluster +++ /dev/null @@ -1,19 +0,0 @@ -[DEFAULT] -devices = /mnt/gluster-object -mount_check = true -bind_port = 6012 # Be sure to keep in sync with /usr/bin/gluster-swift-gen-builders -user = root -log_facility = LOG_LOCAL2 - -[pipeline:main] -pipeline = account-server - -[app:account-server] -use = egg:swift#account - -[account-replicator] -vm_test_mode = yes - -[account-auditor] - -[account-reaper] diff --git a/swift/1.4.8/plugins/conf/container-server/1.conf-gluster b/swift/1.4.8/plugins/conf/container-server/1.conf-gluster deleted file mode 100644 index 2b97eef62..000000000 --- a/swift/1.4.8/plugins/conf/container-server/1.conf-gluster +++ /dev/null @@ -1,21 +0,0 @@ -[DEFAULT] -devices = /mnt/gluster-object -mount_check = true -bind_port = 6011 # Be sure to keep in sync with /usr/bin/gluster-swift-gen-builders -user = root -log_facility = LOG_LOCAL2 - -[pipeline:main] -pipeline = container-server - -[app:container-server] -use = egg:swift#container - -[container-replicator] -vm_test_mode = yes - -[container-updater] - -[container-auditor] - -[container-sync] diff --git a/swift/1.4.8/plugins/conf/fs.conf-gluster b/swift/1.4.8/plugins/conf/fs.conf-gluster deleted file mode 100644 index 0bd21babf..000000000 --- a/swift/1.4.8/plugins/conf/fs.conf-gluster +++ /dev/null @@ -1,8 +0,0 @@ -[DEFAULT] -auth_account = auth -#ip of the fs server. -mount_ip = localhost -#fs server need not be local, remote server can also be used, -#set remote_cluster=yes for using remote server. -remote_cluster = no -object_only = no diff --git a/swift/1.4.8/plugins/conf/object-server/1.conf-gluster b/swift/1.4.8/plugins/conf/object-server/1.conf-gluster deleted file mode 100644 index 62e2fb496..000000000 --- a/swift/1.4.8/plugins/conf/object-server/1.conf-gluster +++ /dev/null @@ -1,19 +0,0 @@ -[DEFAULT] -devices = /mnt/gluster-object -mount_check = true -bind_port = 6010 # Be sure to keep in sync with /usr/bin/gluster-swift-gen-builders -user = root -log_facility = LOG_LOCAL2 - -[pipeline:main] -pipeline = object-server - -[app:object-server] -use = egg:swift#object - -[object-replicator] -vm_test_mode = yes - -[object-updater] - -[object-auditor] diff --git a/swift/1.4.8/plugins/conf/proxy-server.conf-gluster b/swift/1.4.8/plugins/conf/proxy-server.conf-gluster deleted file mode 100644 index b5e1f6f35..000000000 --- a/swift/1.4.8/plugins/conf/proxy-server.conf-gluster +++ /dev/null @@ -1,36 +0,0 @@ -[DEFAULT] -bind_port = 8080 -user = root -log_facility = LOG_LOCAL1 - -[pipeline:main] -pipeline = healthcheck cache tempauth gluster proxy-server - -[app:proxy-server] -use = egg:swift#proxy -allow_account_management = true -account_autocreate = true - -[filter:tempauth] -use = egg:swift#tempauth -# Here you need to add users explicitly. See the OpenStack Swift Deployment -# Guide for more information. The user and user64 directives take the -# following form: -# user__ = [group] [group] [...] [storage_url] -# user64__ = [group] [group] [...] [storage_url] -# Where you use user64 for accounts and/or usernames that include underscores. -# -# NOTE (and WARNING): The account name must match the device name specified -# when generating the account, container, and object build rings. -# -# E.g. -# user_ufo0_admin = abc123 .admin - -[filter:healthcheck] -use = egg:swift#healthcheck - -[filter:cache] -use = egg:swift#memcache - -[filter:gluster] -use = egg:swift#gluster diff --git a/swift/1.4.8/plugins/conf/swift.conf-gluster b/swift/1.4.8/plugins/conf/swift.conf-gluster deleted file mode 100644 index e506b6f54..000000000 --- a/swift/1.4.8/plugins/conf/swift.conf-gluster +++ /dev/null @@ -1,92 +0,0 @@ -[DEFAULT] -Enable_plugin = yes - - -[swift-hash] -# random unique string that can never change (DO NOT LOSE) -swift_hash_path_suffix = gluster - - -# The swift-constraints section sets the basic constraints on data -# saved in the swift cluster. - -[swift-constraints] - -# max_file_size is the largest "normal" object that can be saved in -# the cluster. This is also the limit on the size of each segment of -# a "large" object when using the large object manifest support. -# This value is set in bytes. Setting it to lower than 1MiB will cause -# some tests to fail. It is STRONGLY recommended to leave this value at -# the default (5 * 2**30 + 2). - -# FIXME: Really? Gluster can handle a 2^64 sized file? And can the fronting -# web service handle such a size? I think with UFO, we need to keep with the -# default size from Swift and encourage users to research what size their web -# services infrastructure can handle. - -max_file_size = 18446744073709551616 - - -# max_meta_name_length is the max number of bytes in the utf8 encoding -# of the name portion of a metadata header. - -#max_meta_name_length = 128 - - -# max_meta_value_length is the max number of bytes in the utf8 encoding -# of a metadata value - -#max_meta_value_length = 256 - - -# max_meta_count is the max number of metadata keys that can be stored -# on a single account, container, or object - -#max_meta_count = 90 - - -# max_meta_overall_size is the max number of bytes in the utf8 encoding -# of the metadata (keys + values) - -#max_meta_overall_size = 4096 - - -# max_object_name_length is the max number of bytes in the utf8 encoding of an -# object name: Gluster FS can handle much longer file names, but the length -# between the slashes of the URL is handled below. Remember that most web -# clients can't handle anything greater than 2048, and those that do are -# rather clumsy. - -max_object_name_length = 2048 - -# max_object_name_component_length (GlusterFS) is the max number of bytes in -# the utf8 encoding of an object name component (the part between the -# slashes); this is a limit imposed by the underlying file system (for XFS it -# is 255 bytes). - -max_object_name_component_length = 255 - -# container_listing_limit is the default (and max) number of items -# returned for a container listing request - -#container_listing_limit = 10000 - - -# account_listing_limit is the default (and max) number of items returned -# for an account listing request - -#account_listing_limit = 10000 - - -# max_account_name_length is the max number of bytes in the utf8 encoding of -# an account name: Gluster FS Filename limit (XFS limit?), must be the same -# size as max_object_name_component_length above. - -max_account_name_length = 255 - - -# max_container_name_length is the max number of bytes in the utf8 encoding -# of a container name: Gluster FS Filename limit (XFS limit?), must be the same -# size as max_object_name_component_length above. - -max_container_name_length = 255 diff --git a/swift/1.4.8/plugins/constraints.py b/swift/1.4.8/plugins/constraints.py deleted file mode 100644 index 6d003b94c..000000000 --- a/swift/1.4.8/plugins/constraints.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from webob.exc import HTTPBadRequest - -import swift.common.constraints -from swift.plugins import Glusterfs - - -MAX_OBJECT_NAME_COMPONENT_LENGTH = swift.common.constraints.constraints_conf_int( - 'max_object_name_component_length', 255) - -def validate_obj_name_component(obj): - if len(obj) > MAX_OBJECT_NAME_COMPONENT_LENGTH: - return 'too long (%d)' % len(obj) - if obj == '.' or obj == '..': - return 'cannot be . or ..' - return '' - -# Save the original check object creation -__check_object_creation = swift.common.constraints.check_object_creation - -# Define our new one which invokes the original -def gluster_check_object_creation(req, object_name): - """ - Check to ensure that everything is alright about an object to be created. - Monkey patches swift.common.constraints.check_object_creation, invoking - the original, and then adding an additional check for individual object - name components. - - :param req: HTTP request object - :param object_name: name of object to be created - :raises HTTPRequestEntityTooLarge: the object is too large - :raises HTTPLengthRequered: missing content-length header and not - a chunked request - :raises HTTPBadRequest: missing or bad content-type header, or - bad metadata - """ - ret = __check_object_creation(req, object_name) - - if ret is None: - for obj in object_name.split('/'): - reason = validate_obj_name_component(obj) - if reason: - bdy = 'Invalid object name "%s", component "%s" %s' \ - % (object_name, obj, reason) - ret = HTTPBadRequest(body=bdy, - request=req, - content_type='text/plain') - - return ret - -# Replace the original check object creation with ours -swift.common.constraints.check_object_creation = gluster_check_object_creation - -# Save the original check mount -__check_mount = swift.common.constraints.check_mount - -# Define our new one which invokes the original -def gluster_check_mount(root, drive): - # FIXME: Potential performance optimization here to not call the original - # check mount which makes two stat calls. We could do what they do with - # just one. - if __check_mount(root, drive): - return True - - return Glusterfs.mount(root, drive) - -# Replace the original check mount with ours -swift.common.constraints.check_mount = gluster_check_mount diff --git a/swift/1.4.8/plugins/fs_utils.py b/swift/1.4.8/plugins/fs_utils.py deleted file mode 100644 index 7f5292c2b..000000000 --- a/swift/1.4.8/plugins/fs_utils.py +++ /dev/null @@ -1,156 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import os -import errno - -def do_mkdir(path): - try: - os.mkdir(path) - except Exception, err: - logging.exception("Mkdir failed on %s err: %s", path, str(err)) - if err.errno != errno.EEXIST: - raise - return True - -def do_makedirs(path): - try: - os.makedirs(path) - except Exception, err: - logging.exception("Makedirs failed on %s err: %s", path, str(err)) - if err.errno != errno.EEXIST: - raise - return True - -def do_listdir(path): - try: - buf = os.listdir(path) - except Exception, err: - logging.exception("Listdir failed on %s err: %s", path, str(err)) - raise - return buf - -def do_chown(path, uid, gid): - try: - os.chown(path, uid, gid) - except Exception, err: - logging.exception("Chown failed on %s err: %s", path, str(err)) - raise - return True - -def do_stat(path): - try: - #Check for fd. - if isinstance(path, int): - buf = os.fstat(path) - else: - buf = os.stat(path) - except Exception, err: - logging.exception("Stat failed on %s err: %s", path, str(err)) - raise - - return buf - -def do_open(path, mode): - try: - fd = open(path, mode) - except Exception, err: - logging.exception("Open failed on %s err: %s", path, str(err)) - raise - return fd - -def do_close(fd): - #fd could be file or int type. - try: - if isinstance(fd, int): - os.close(fd) - else: - fd.close() - except Exception, err: - logging.exception("Close failed on %s err: %s", fd, str(err)) - raise - return True - -def do_unlink(path, log = True): - try: - os.unlink(path) - except Exception, err: - if log: - logging.exception("Unlink failed on %s err: %s", path, str(err)) - if err.errno != errno.ENOENT: - raise - return True - -def do_rmdir(path): - try: - os.rmdir(path) - except Exception, err: - logging.exception("Rmdir failed on %s err: %s", path, str(err)) - if err.errno != errno.ENOENT: - raise - return True - -def do_rename(old_path, new_path): - try: - os.rename(old_path, new_path) - except Exception, err: - logging.exception("Rename failed on %s to %s err: %s", old_path, new_path, \ - str(err)) - raise - return True - -def mkdirs(path): - """ - Ensures the path is a directory or makes it if not. Errors if the path - exists but is a file or on permissions failure. - - :param path: path to create - """ - if not os.path.isdir(path): - try: - do_makedirs(path) - except OSError, err: - #TODO: check, isdir will fail if mounted and volume stopped. - #if err.errno != errno.EEXIST or not os.path.isdir(path) - if err.errno != errno.EEXIST: - raise - -def dir_empty(path): - """ - Return true if directory/container is empty. - :param path: Directory path. - :returns: True/False. - """ - if os.path.isdir(path): - try: - files = do_listdir(path) - except Exception, err: - logging.exception("listdir failed on %s err: %s", path, str(err)) - raise - if not files: - return True - else: - return False - else: - if not os.path.exists(path): - return True - -def rmdirs(path): - if os.path.isdir(path) and dir_empty(path): - do_rmdir(path) - else: - logging.error("rmdirs failed dir may not be empty or not valid dir") - return False diff --git a/swift/1.4.8/plugins/middleware/__init__.py b/swift/1.4.8/plugins/middleware/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/swift/1.4.8/plugins/middleware/gluster.py b/swift/1.4.8/plugins/middleware/gluster.py deleted file mode 100644 index e0e65e353..000000000 --- a/swift/1.4.8/plugins/middleware/gluster.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Noop Middleware that simply allows us to monkey patch the constraints -import swift.plugins.constraints - -class Gluster(object): - """ - Noop middleware for use with the proxy server to get paste.deploy to load - this middleware such that the plugin constraints monkey patch the common - constraints ahead of their use. - """ - def __init__(self, app, conf): - self.app = app - self.conf = conf - - def __call__(self, env, start_response): - return self.app(env, start_response) - - -def filter_factory(global_conf, **local_conf): - """Returns a WSGI filter app for use with paste.deploy.""" - conf = global_conf.copy() - conf.update(local_conf) - - def gluster_filter(app): - return Gluster(app, conf) - return gluster_filter diff --git a/swift/1.4.8/plugins/utils.py b/swift/1.4.8/plugins/utils.py deleted file mode 100644 index 45990d1da..000000000 --- a/swift/1.4.8/plugins/utils.py +++ /dev/null @@ -1,475 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import os -import errno -import xattr -from hashlib import md5 -import cPickle as pickle -from ConfigParser import ConfigParser, NoSectionError, NoOptionError -from swift.common.utils import normalize_timestamp, TRUE_VALUES -from swift.plugins.fs_utils import * -from swift.plugins import Glusterfs - -X_CONTENT_TYPE = 'Content-Type' -X_CONTENT_LENGTH = 'Content-Length' -X_TIMESTAMP = 'X-Timestamp' -X_PUT_TIMESTAMP = 'X-PUT-Timestamp' -X_TYPE = 'X-Type' -X_ETAG = 'ETag' -X_OBJECTS_COUNT = 'X-Object-Count' -X_BYTES_USED = 'X-Bytes-Used' -X_CONTAINER_COUNT = 'X-Container-Count' -X_OBJECT_TYPE = 'X-Object-Type' -DIR_TYPE = 'application/directory' -ACCOUNT = 'Account' -METADATA_KEY = 'user.swift.metadata' -MAX_XATTR_SIZE = 65536 -CONTAINER = 'container' -DIR = 'dir' -MARKER_DIR = 'marker_dir' -TEMP_DIR = 'tmp' -ASYNCDIR = 'async_pending' # Keep in sync with swift.obj.server.ASYNCDIR -FILE = 'file' -FILE_TYPE = 'application/octet-stream' -OBJECT = 'Object' -OBJECT_TYPE = 'application/octet-stream' -DEFAULT_UID = -1 -DEFAULT_GID = -1 -PICKLE_PROTOCOL = 2 -CHUNK_SIZE = 65536 -MEMCACHE_KEY_PREFIX = 'gluster.swift.' -MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX = MEMCACHE_KEY_PREFIX + 'account.details.' -MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX = MEMCACHE_KEY_PREFIX + 'container.details.' - - -def read_metadata(path): - """ - Helper function to read the pickled metadata from a File/Directory. - - :param path: File/Directory to read metadata from. - - :returns: dictionary of metadata - """ - metadata = None - metadata_s = '' - key = 0 - while metadata is None: - try: - metadata_s += xattr.get(path, '%s%s' % (METADATA_KEY, (key or ''))) - except IOError as err: - if err.errno == errno.ENODATA: - if key > 0: - # No errors reading the xattr keys, but since we have not - # been able to find enough chunks to get a successful - # unpickle operation, we consider the metadata lost, and - # drop the existing data so that the internal state can be - # recreated. - clean_metadata(path) - # We either could not find any metadata key, or we could find - # some keys, but were not successful in performing the - # unpickling (missing keys perhaps)? Either way, just report - # to the caller we have no metadata. - metadata = {} - else: - logging.exception("xattr.get failed on %s key %s err: %s", - path, key, str(err)) - # Note that we don't touch the keys on errors fetching the - # data since it could be a transient state. - raise - else: - try: - # If this key provides all or the remaining part of the pickle - # data, we don't need to keep searching for more keys. This - # means if we only need to store data in N xattr key/value - # pair, we only need to invoke xattr get N times. With large - # keys sizes we are shooting for N = 1. - metadata = pickle.loads(metadata_s) - assert isinstance(metadata, dict) - except EOFError, pickle.UnpicklingError: - # We still are not able recognize this existing data collected - # as a pickled object. Make sure we loop around to try to get - # more from another xattr key. - metadata = None - key += 1 - return metadata - -def write_metadata(path, metadata): - """ - Helper function to write pickled metadata for a File/Directory. - - :param path: File/Directory path to write the metadata - :param metadata: dictionary to metadata write - """ - assert isinstance(metadata, dict) - metastr = pickle.dumps(metadata, PICKLE_PROTOCOL) - key = 0 - while metastr: - try: - xattr.set(path, '%s%s' % (METADATA_KEY, key or ''), metastr[:MAX_XATTR_SIZE]) - except IOError as err: - logging.exception("xattr.set failed on %s key %s err: %s", path, key, str(err)) - raise - metastr = metastr[MAX_XATTR_SIZE:] - key += 1 - -def clean_metadata(path): - key = 0 - while True: - try: - xattr.remove(path, '%s%s' % (METADATA_KEY, (key or ''))) - except IOError as err: - if err.errno == errno.ENODATA: - break - raise - key += 1 - -def check_user_xattr(path): - if not os.path.exists(path): - return False - try: - xattr.set(path, 'user.test.key1', 'value1') - except IOError as err: - logging.exception("check_user_xattr: set failed on %s err: %s", path, str(err)) - raise - try: - xattr.remove(path, 'user.test.key1') - except IOError as err: - logging.exception("check_user_xattr: remove failed on %s err: %s", path, str(err)) - #Remove xattr may fail in case of concurrent remove. - return True - -def validate_container(metadata): - if not metadata: - logging.warn('validate_container: No metadata') - return False - - if X_TYPE not in metadata.keys() or \ - X_TIMESTAMP not in metadata.keys() or \ - X_PUT_TIMESTAMP not in metadata.keys() or \ - X_OBJECTS_COUNT not in metadata.keys() or \ - X_BYTES_USED not in metadata.keys(): - #logging.warn('validate_container: Metadata missing entries: %s' % metadata) - return False - - (value, timestamp) = metadata[X_TYPE] - if value == CONTAINER: - return True - - logging.warn('validate_container: metadata type is not CONTAINER (%r)' % (value,)) - return False - -def validate_account(metadata): - if not metadata: - logging.warn('validate_account: No metadata') - return False - - if X_TYPE not in metadata.keys() or \ - X_TIMESTAMP not in metadata.keys() or \ - X_PUT_TIMESTAMP not in metadata.keys() or \ - X_OBJECTS_COUNT not in metadata.keys() or \ - X_BYTES_USED not in metadata.keys() or \ - X_CONTAINER_COUNT not in metadata.keys(): - #logging.warn('validate_account: Metadata missing entries: %s' % metadata) - return False - - (value, timestamp) = metadata[X_TYPE] - if value == ACCOUNT: - return True - - logging.warn('validate_account: metadata type is not ACCOUNT (%r)' % (value,)) - return False - -def validate_object(metadata): - if not metadata: - logging.warn('validate_object: No metadata') - return False - - if X_TIMESTAMP not in metadata.keys() or \ - X_CONTENT_TYPE not in metadata.keys() or \ - X_ETAG not in metadata.keys() or \ - X_CONTENT_LENGTH not in metadata.keys() or \ - X_TYPE not in metadata.keys() or \ - X_OBJECT_TYPE not in metadata.keys(): - #logging.warn('validate_object: Metadata missing entries: %s' % metadata) - return False - - if metadata[X_TYPE] == OBJECT: - return True - - logging.warn('validate_object: metadata type is not OBJECT (%r)' % (metadata[X_TYPE],)) - return False - -def is_marker(metadata): - if not metadata: - logging.warn('is_marker: No metadata') - return False - - if X_OBJECT_TYPE not in metadata.keys(): - logging.warn('is_marker: X_OBJECT_TYPE missing from metadata: %s' % metadata) - return False - - if metadata[X_OBJECT_TYPE] == MARKER_DIR: - return True - else: - return False - -def _update_list(path, cont_path, src_list, reg_file=True, object_count=0, - bytes_used=0, obj_list=[]): - # strip the prefix off, also stripping the leading and trailing slashes - obj_path = path.replace(cont_path, '').strip(os.path.sep) - - for i in src_list: - if obj_path: - obj_list.append(os.path.join(obj_path, i)) - else: - obj_list.append(i) - - object_count += 1 - - if reg_file: - bytes_used += os.path.getsize(path + '/' + i) - - return object_count, bytes_used - -def update_list(path, cont_path, dirs=[], files=[], object_count=0, - bytes_used=0, obj_list=[]): - object_count, bytes_used = _update_list(path, cont_path, files, True, - object_count, bytes_used, - obj_list) - object_count, bytes_used = _update_list(path, cont_path, dirs, False, - object_count, bytes_used, - obj_list) - return object_count, bytes_used - - -class ContainerDetails(object): - def __init__(self, bytes_used, object_count, obj_list, dir_list): - self.bytes_used = bytes_used - self.object_count = object_count - self.obj_list = obj_list - self.dir_list = dir_list - - -def _get_container_details_from_fs(cont_path): - """ - get container details by traversing the filesystem - """ - bytes_used = 0 - object_count = 0 - obj_list = [] - dir_list = [] - - if os.path.isdir(cont_path): - for (path, dirs, files) in os.walk(cont_path): - object_count, bytes_used = update_list(path, cont_path, dirs, files, - object_count, bytes_used, - obj_list) - - dir_list.append((path, do_stat(path).st_mtime)) - - return ContainerDetails(bytes_used, object_count, obj_list, dir_list) - -def get_container_details(cont_path, memcache=None): - """ - Return object_list, object_count and bytes_used. - """ - mkey = '' - if memcache: - mkey = MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + cont_path - cd = memcache.get(mkey) - if cd: - if not cd.dir_list: - cd = None - else: - for (path, mtime) in cd.dir_list: - if mtime != do_stat(path).st_mtime: - cd = None - else: - cd = None - if not cd: - cd = _get_container_details_from_fs(cont_path) - if memcache: - memcache.set(mkey, cd) - return cd.obj_list, cd.object_count, cd.bytes_used - - -class AccountDetails(object): - """ A simple class to store the three pieces of information associated - with an account: - - 1. The last known modification time - 2. The count of containers in the following list - 3. The list of containers - """ - def __init__(self, mtime, container_count, container_list): - self.mtime = mtime - self.container_count = container_count - self.container_list = container_list - - -def _get_account_details_from_fs(acc_path, acc_stats): - container_list = [] - container_count = 0 - - if not acc_stats: - acc_stats = do_stat(acc_path) - is_dir = (acc_stats.st_mode & 0040000) != 0 - if is_dir: - for name in do_listdir(acc_path): - if name.lower() == TEMP_DIR \ - or name.lower() == ASYNCDIR \ - or not os.path.isdir(os.path.join(acc_path, name)): - continue - container_count += 1 - container_list.append(name) - - return AccountDetails(acc_stats.st_mtime, container_count, container_list) - -def get_account_details(acc_path, memcache=None): - """ - Return container_list and container_count. - """ - acc_stats = None - mkey = '' - if memcache: - mkey = MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX + acc_path - ad = memcache.get(mkey) - if ad: - # FIXME: Do we really need to stat the file? If we are object - # only, then we can track the other Swift HTTP APIs that would - # modify the account and invalidate the cached entry there. If we - # are not object only, are we even called on this path? - acc_stats = do_stat(acc_path) - if ad.mtime != acc_stats.st_mtime: - ad = None - else: - ad = None - if not ad: - ad = _get_account_details_from_fs(acc_path, acc_stats) - if memcache: - memcache.set(mkey, ad) - return ad.container_list, ad.container_count - -def _get_etag(path): - etag = md5() - with open(path, 'rb') as fp: - while True: - chunk = fp.read(CHUNK_SIZE) - if chunk: - etag.update(chunk) - else: - break - return etag.hexdigest() - -def get_object_metadata(obj_path): - """ - Return metadata of object. - """ - try: - stats = os.stat(obj_path) - except OSError as e: - if e.errno != errno.ENOENT: - raise - metadata = {} - else: - is_dir = (stats.st_mode & 0040000) != 0 - metadata = { - X_TYPE: OBJECT, - X_TIMESTAMP: normalize_timestamp(stats.st_ctime), - X_CONTENT_TYPE: DIR_TYPE if is_dir else FILE_TYPE, - X_OBJECT_TYPE: DIR if is_dir else FILE, - X_CONTENT_LENGTH: 0 if is_dir else stats.st_size, - X_ETAG: md5().hexdigest() if is_dir else _get_etag(obj_path), - } - return metadata - -def _add_timestamp(metadata_i): - # At this point we have a simple key/value dictionary, turn it into - # key/(value,timestamp) pairs. - timestamp = 0 - metadata = {} - for key, value_i in metadata_i.iteritems(): - if not isinstance(value_i, tuple): - metadata[key] = (value_i, timestamp) - else: - metadata[key] = value_i - return metadata - -def get_container_metadata(cont_path, memcache=None): - objects = [] - object_count = 0 - bytes_used = 0 - objects, object_count, bytes_used = get_container_details(cont_path, memcache) - metadata = {X_TYPE: CONTAINER, - X_TIMESTAMP: normalize_timestamp(os.path.getctime(cont_path)), - X_PUT_TIMESTAMP: normalize_timestamp(os.path.getmtime(cont_path)), - X_OBJECTS_COUNT: object_count, - X_BYTES_USED: bytes_used} - return _add_timestamp(metadata) - -def get_account_metadata(acc_path, memcache=None): - containers = [] - container_count = 0 - containers, container_count = get_account_details(acc_path, memcache) - metadata = {X_TYPE: ACCOUNT, - X_TIMESTAMP: normalize_timestamp(os.path.getctime(acc_path)), - X_PUT_TIMESTAMP: normalize_timestamp(os.path.getmtime(acc_path)), - X_OBJECTS_COUNT: 0, - X_BYTES_USED: 0, - X_CONTAINER_COUNT: container_count} - return _add_timestamp(metadata) - -def restore_metadata(path, metadata): - meta_orig = read_metadata(path) - if meta_orig: - meta_new = meta_orig.copy() - meta_new.update(metadata) - else: - meta_new = metadata - if meta_orig != meta_new: - write_metadata(path, meta_new) - return meta_new - -def create_object_metadata(obj_path): - metadata = get_object_metadata(obj_path) - return restore_metadata(obj_path, metadata) - -def create_container_metadata(cont_path, memcache=None): - metadata = get_container_metadata(cont_path, memcache) - return restore_metadata(cont_path, metadata) - -def create_account_metadata(acc_path, memcache=None): - metadata = get_account_metadata(acc_path, memcache) - return restore_metadata(acc_path, metadata) - - -_DEFAULT_GLUSTER_ENABLED = os.getenv('GLUSTER_UNIT_TEST_ENABLED', 'no') -__swift_conf = ConfigParser() -__swift_conf.read(os.path.join('/etc/swift', 'swift.conf')) -try: - _gluster_enabled_val = __swift_conf.get('DEFAULT', 'Enable_plugin', _DEFAULT_GLUSTER_ENABLED) -except NoOptionError, NoSectionError: - _gluster_enabled_val = _DEFAULT_GLUSTER_ENABLED -del __swift_conf -_gluster_enabled = _gluster_enabled_val in TRUE_VALUES - -def Gluster_enabled(): - return _gluster_enabled - -if _gluster_enabled: - # Monkey patch only when Gluster enabled - import swift.plugins.constraints diff --git a/swift/1.4.8/setup.py b/swift/1.4.8/setup.py new file mode 100644 index 000000000..a48310282 --- /dev/null +++ b/swift/1.4.8/setup.py @@ -0,0 +1,57 @@ +#!/usr/bin/python +# Copyright (c) 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from setuptools import setup, find_packages + +from gluster.swift import __canonical_version__ as version + + +name = 'gluster_swift_ufo' + + +setup( + name=name, + version=version, + description='Gluster Swift/UFO', + license='Apache License (2.0)', + author='Red Hat, Inc.', + author_email='gluster-users@gluster.org', + url='https://gluster.org/', + packages=find_packages(exclude=['test', 'bin']), + test_suite='nose.collector', + classifiers=[ + 'Development Status :: 4 - Beta', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python :: 2.6', + 'Environment :: No Input/Output (Daemon)', + ], + install_requires=[], # removed for better compat + scripts=[ + 'bin/gluster-swift-gen-builders', + ], + entry_points={ + 'paste.app_factory': [ + 'proxy=gluster.swift.proxy.server:app_factory', + 'object=gluster.swift.obj.server:app_factory', + 'container=gluster.swift.container.server:app_factory', + 'account=gluster.swift.account.server:app_factory', + ], + 'paste.filter_factory': [ + 'gluster=gluster.swift.common.middleware.gluster:filter_factory', + ], + }, + ) diff --git a/swift/1.4.8/swift.diff b/swift/1.4.8/swift.diff deleted file mode 100644 index 59bbbe262..000000000 --- a/swift/1.4.8/swift.diff +++ /dev/null @@ -1,174 +0,0 @@ -diff --git a/setup.py b/setup.py -index d195d34..ef625ff 100644 ---- a/setup.py -+++ b/setup.py -@@ -1,5 +1,6 @@ - #!/usr/bin/python - # Copyright (c) 2010-2012 OpenStack, LLC. -+# Copyright (c) 2012 Red Hat, Inc. - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. -@@ -94,6 +95,7 @@ setup( - 'tempurl=swift.common.middleware.tempurl:filter_factory', - 'formpost=swift.common.middleware.formpost:filter_factory', - 'name_check=swift.common.middleware.name_check:filter_factory', -+ 'gluster=swift.plugins.middleware.gluster:filter_factory', - ], - }, - ) -diff --git a/swift/account/server.py b/swift/account/server.py -index 800b3c0..eaf9e0d 100644 ---- a/swift/account/server.py -+++ b/swift/account/server.py -@@ -1,4 +1,5 @@ - # Copyright (c) 2010-2012 OpenStack, LLC. -+# Copyright (c) 2012 Red Hat, Inc. - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. -@@ -29,6 +30,10 @@ from webob.exc import HTTPAccepted, HTTPBadRequest, \ - HTTPPreconditionFailed, HTTPConflict - import simplejson - -+from swift.plugins.utils import Gluster_enabled -+if Gluster_enabled(): -+ from swift.plugins.DiskDir import DiskAccount -+ - from swift.common.db import AccountBroker - from swift.common.utils import get_logger, get_param, hash_path, \ - normalize_timestamp, split_path, storage_directory -@@ -54,6 +59,8 @@ class AccountController(object): - conf.get('auto_create_account_prefix') or '.' - - def _get_account_broker(self, drive, part, account): -+ if Gluster_enabled(): -+ return DiskAccount(self.root, account, self.logger) - hsh = hash_path(account) - db_dir = storage_directory(DATADIR, part, hsh) - db_path = os.path.join(self.root, drive, db_dir, hsh + '.db') -diff --git a/swift/container/server.py b/swift/container/server.py -index 8a18cfd..3da0f95 100644 ---- a/swift/container/server.py -+++ b/swift/container/server.py -@@ -1,4 +1,5 @@ - # Copyright (c) 2010-2012 OpenStack, LLC. -+# Copyright (c) 2012 Red Hat, Inc. - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. -@@ -29,6 +30,10 @@ from webob.exc import HTTPAccepted, HTTPBadRequest, HTTPConflict, \ - HTTPCreated, HTTPInternalServerError, HTTPNoContent, \ - HTTPNotFound, HTTPPreconditionFailed, HTTPMethodNotAllowed - -+from swift.plugins.utils import Gluster_enabled -+if Gluster_enabled(): -+ from swift.plugins.DiskDir import DiskDir -+ - from swift.common.db import ContainerBroker - from swift.common.utils import get_logger, get_param, hash_path, \ - normalize_timestamp, storage_directory, split_path, validate_sync_to -@@ -73,6 +78,8 @@ class ContainerController(object): - :param container: container name - :returns: ContainerBroker object - """ -+ if Gluster_enabled(): -+ return DiskDir(self.root, account, container, self.logger) - hsh = hash_path(account, container) - db_dir = storage_directory(DATADIR, part, hsh) - db_path = os.path.join(self.root, drive, db_dir, hsh + '.db') -diff --git a/swift/obj/server.py b/swift/obj/server.py -index 9cca16b..448ea5c 100644 ---- a/swift/obj/server.py -+++ b/swift/obj/server.py -@@ -1,4 +1,5 @@ - # Copyright (c) 2010-2012 OpenStack, LLC. -+# Copyright (c) 2012 Red Hat, Inc. - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. -@@ -35,6 +36,8 @@ from webob.exc import HTTPAccepted, HTTPBadRequest, HTTPCreated, \ - from xattr import getxattr, setxattr - from eventlet import sleep, Timeout, tpool - -+from swift.plugins.utils import Gluster_enabled -+ - from swift.common.utils import mkdirs, normalize_timestamp, \ - storage_directory, hash_path, renamer, fallocate, \ - split_path, drop_buffer_cache, get_logger, write_pickle -@@ -340,6 +343,10 @@ class DiskFile(object): - raise DiskFileNotExist('Data File does not exist.') - - -+if Gluster_enabled(): -+ from swift.plugins.DiskFile import Gluster_DiskFile -+ -+ - class ObjectController(object): - """Implements the WSGI application for the Swift Object Server.""" - -@@ -378,6 +385,15 @@ class ObjectController(object): - self.expiring_objects_container_divisor = \ - int(conf.get('expiring_objects_container_divisor') or 86400) - -+ def get_DiskFile_obj(self, path, device, partition, account, container, obj, -+ logger, keep_data_fp=False, disk_chunk_size=65536): -+ if Gluster_enabled(): -+ return Gluster_DiskFile(path, device, partition, account, container, -+ obj, logger, keep_data_fp, disk_chunk_size) -+ else: -+ return DiskFile(path, device, partition, account, container, -+ obj, logger, keep_data_fp, disk_chunk_size) -+ - def async_update(self, op, account, container, obj, host, partition, - contdevice, headers_out, objdevice): - """ -@@ -493,7 +509,7 @@ class ObjectController(object): - content_type='text/plain') - if self.mount_check and not check_mount(self.devices, device): - return Response(status='507 %s is not mounted' % device) -- file = DiskFile(self.devices, device, partition, account, container, -+ file = self.get_DiskFile_obj(self.devices, device, partition, account, container, - obj, self.logger, disk_chunk_size=self.disk_chunk_size) - - if 'X-Delete-At' in file.metadata and \ -@@ -548,7 +564,7 @@ class ObjectController(object): - if new_delete_at and new_delete_at < time.time(): - return HTTPBadRequest(body='X-Delete-At in past', request=request, - content_type='text/plain') -- file = DiskFile(self.devices, device, partition, account, container, -+ file = self.get_DiskFile_obj(self.devices, device, partition, account, container, - obj, self.logger, disk_chunk_size=self.disk_chunk_size) - orig_timestamp = file.metadata.get('X-Timestamp') - upload_expiration = time.time() + self.max_upload_time -@@ -626,9 +642,9 @@ class ObjectController(object): - content_type='text/plain') - if self.mount_check and not check_mount(self.devices, device): - return Response(status='507 %s is not mounted' % device) -- file = DiskFile(self.devices, device, partition, account, container, -- obj, self.logger, keep_data_fp=True, -- disk_chunk_size=self.disk_chunk_size) -+ file = self.get_DiskFile_obj(self.devices, device, partition, account, container, -+ obj, self.logger, keep_data_fp=True, -+ disk_chunk_size=self.disk_chunk_size) - if file.is_deleted() or ('X-Delete-At' in file.metadata and - int(file.metadata['X-Delete-At']) <= time.time()): - if request.headers.get('if-match') == '*': -@@ -702,7 +718,7 @@ class ObjectController(object): - return resp - if self.mount_check and not check_mount(self.devices, device): - return Response(status='507 %s is not mounted' % device) -- file = DiskFile(self.devices, device, partition, account, container, -+ file = self.get_DiskFile_obj(self.devices, device, partition, account, container, - obj, self.logger, disk_chunk_size=self.disk_chunk_size) - if file.is_deleted() or ('X-Delete-At' in file.metadata and - int(file.metadata['X-Delete-At']) <= time.time()): -@@ -744,7 +760,7 @@ class ObjectController(object): - if self.mount_check and not check_mount(self.devices, device): - return Response(status='507 %s is not mounted' % device) - response_class = HTTPNoContent -- file = DiskFile(self.devices, device, partition, account, container, -+ file = self.get_DiskFile_obj(self.devices, device, partition, account, container, - obj, self.logger, disk_chunk_size=self.disk_chunk_size) - if 'x-if-delete-at' in request.headers and \ - int(request.headers['x-if-delete-at']) != \ diff --git a/swift/1.4.8/test/unit/common/__init__.py b/swift/1.4.8/test/unit/common/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/swift/1.4.8/test/unit/common/data/account_tree.tar.bz2 b/swift/1.4.8/test/unit/common/data/account_tree.tar.bz2 new file mode 100644 index 000000000..cb23e4dd7 Binary files /dev/null and b/swift/1.4.8/test/unit/common/data/account_tree.tar.bz2 differ diff --git a/swift/1.4.8/test/unit/common/data/container_tree.tar.bz2 b/swift/1.4.8/test/unit/common/data/container_tree.tar.bz2 new file mode 100644 index 000000000..b4a149285 Binary files /dev/null and b/swift/1.4.8/test/unit/common/data/container_tree.tar.bz2 differ diff --git a/swift/1.4.8/test/unit/common/test_utils.py b/swift/1.4.8/test/unit/common/test_utils.py new file mode 100644 index 000000000..ec66324a0 --- /dev/null +++ b/swift/1.4.8/test/unit/common/test_utils.py @@ -0,0 +1,818 @@ +# Copyright (c) 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Tests for common.utils """ + +import os +import unittest +import errno +import xattr +import cPickle as pickle +import tempfile +import hashlib +import tarfile +import shutil +from collections import defaultdict +from swift.common.utils import normalize_timestamp +from gluster.swift.common import utils + +# +# Somewhat hacky way of emulating the operation of xattr calls. They are made +# against a dictionary that stores the xattr key/value pairs. +# +_xattrs = {} +_xattr_op_cnt = defaultdict(int) +_xattr_err = {} + +def _xkey(path, key): + return "%s:%s" % (path, key) + +def _setxattr(path, key, value): + _xattr_op_cnt['set'] += 1 + xkey = _xkey(path, key) + if xkey in _xattr_err: + e = IOError() + e.errno = _xattr_err[xkey] + raise e + global _xattrs + _xattrs[xkey] = value + +def _getxattr(path, key): + _xattr_op_cnt['get'] += 1 + xkey = _xkey(path, key) + if xkey in _xattr_err: + e = IOError() + e.errno = _xattr_err[xkey] + raise e + global _xattrs + if xkey in _xattrs: + ret_val = _xattrs[xkey] + else: + e = IOError("Fake IOError") + e.errno = errno.ENODATA + raise e + return ret_val + +def _removexattr(path, key): + _xattr_op_cnt['remove'] += 1 + xkey = _xkey(path, key) + if xkey in _xattr_err: + e = IOError() + e.errno = _xattr_err[xkey] + raise e + global _xattrs + if xkey in _xattrs: + del _xattrs[xkey] + else: + e = IOError("Fake IOError") + e.errno = errno.ENODATA + raise e + +def _initxattr(): + global _xattrs + _xattrs = {} + global _xattr_op_cnt + _xattr_op_cnt = defaultdict(int) + global _xattr_err + _xattr_err = {} + + # Save the current methods + global _xattr_set; _xattr_set = xattr.set + global _xattr_get; _xattr_get = xattr.get + global _xattr_remove; _xattr_remove = xattr.remove + + # Monkey patch the calls we use with our internal unit test versions + xattr.set = _setxattr + xattr.get = _getxattr + xattr.remove = _removexattr + +def _destroyxattr(): + # Restore the current methods just in case + global _xattr_set; xattr.set = _xattr_set + global _xattr_get; xattr.get = _xattr_get + global _xattr_remove; xattr.remove = _xattr_remove + # Destroy the stored values and + global _xattrs; _xattrs = None + + +class SimMemcache(object): + def __init__(self): + self._d = {} + + def get(self, key): + return self._d.get(key, None) + + def set(self, key, value): + self._d[key] = value + + +class TestUtils(unittest.TestCase): + """ Tests for common.utils """ + + def setUp(self): + _initxattr() + + def tearDown(self): + _destroyxattr() + + def test_write_metadata(self): + path = "/tmp/foo/w" + orig_d = { 'bar' : 'foo' } + utils.write_metadata(path, orig_d) + xkey = _xkey(path, utils.METADATA_KEY) + assert len(_xattrs.keys()) == 1 + assert xkey in _xattrs + assert orig_d == pickle.loads(_xattrs[xkey]) + assert _xattr_op_cnt['set'] == 1 + + def test_write_metadata_err(self): + path = "/tmp/foo/w" + orig_d = { 'bar' : 'foo' } + xkey = _xkey(path, utils.METADATA_KEY) + _xattr_err[xkey] = errno.EOPNOTSUPP + try: + utils.write_metadata(path, orig_d) + except IOError as e: + assert e.errno == errno.EOPNOTSUPP + assert len(_xattrs.keys()) == 0 + assert _xattr_op_cnt['set'] == 1 + else: + self.fail("Expected an IOError exception on write") + + def test_write_metadata_multiple(self): + # At 64 KB an xattr key/value pair, this should generate three keys. + path = "/tmp/foo/w" + orig_d = { 'bar' : 'x' * 150000 } + utils.write_metadata(path, orig_d) + assert len(_xattrs.keys()) == 3, "Expected 3 keys, found %d" % len(_xattrs.keys()) + payload = '' + for i in range(0,3): + xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or '')) + assert xkey in _xattrs + assert len(_xattrs[xkey]) <= utils.MAX_XATTR_SIZE + payload += _xattrs[xkey] + assert orig_d == pickle.loads(payload) + assert _xattr_op_cnt['set'] == 3, "%r" % _xattr_op_cnt + + def test_clean_metadata(self): + path = "/tmp/foo/c" + expected_d = { 'a': 'y' * 150000 } + expected_p = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL) + for i in range(0,3): + xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or '')) + _xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE] + expected_p = expected_p[utils.MAX_XATTR_SIZE:] + assert not expected_p + utils.clean_metadata(path) + assert _xattr_op_cnt['remove'] == 4, "%r" % _xattr_op_cnt + + def test_clean_metadata_err(self): + path = "/tmp/foo/c" + xkey = _xkey(path, utils.METADATA_KEY) + _xattrs[xkey] = pickle.dumps({ 'a': 'y' }, utils.PICKLE_PROTOCOL) + _xattr_err[xkey] = errno.EOPNOTSUPP + try: + utils.clean_metadata(path) + except IOError as e: + assert e.errno == errno.EOPNOTSUPP + assert _xattr_op_cnt['remove'] == 1, "%r" % _xattr_op_cnt + else: + self.fail("Expected an IOError exception on remove") + + def test_read_metadata(self): + path = "/tmp/foo/r" + expected_d = { 'a': 'y' } + xkey = _xkey(path, utils.METADATA_KEY) + _xattrs[xkey] = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL) + res_d = utils.read_metadata(path) + assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d) + assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt + + def test_read_metadata_notfound(self): + path = "/tmp/foo/r" + res_d = utils.read_metadata(path) + assert res_d == {} + assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt + + def test_read_metadata_err(self): + path = "/tmp/foo/r" + expected_d = { 'a': 'y' } + xkey = _xkey(path, utils.METADATA_KEY) + _xattrs[xkey] = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL) + _xattr_err[xkey] = errno.EOPNOTSUPP + try: + res_d = utils.read_metadata(path) + except IOError as e: + assert e.errno == errno.EOPNOTSUPP + assert (_xattr_op_cnt['get'] == 1), "%r" % _xattr_op_cnt + else: + self.fail("Expected an IOError exception on get") + + def test_read_metadata_multiple(self): + path = "/tmp/foo/r" + expected_d = { 'a': 'y' * 150000 } + expected_p = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL) + for i in range(0,3): + xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or '')) + _xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE] + expected_p = expected_p[utils.MAX_XATTR_SIZE:] + assert not expected_p + res_d = utils.read_metadata(path) + assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d) + assert _xattr_op_cnt['get'] == 3, "%r" % _xattr_op_cnt + + def test_read_metadata_multiple_one_missing(self): + path = "/tmp/foo/r" + expected_d = { 'a': 'y' * 150000 } + expected_p = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL) + for i in range(0,2): + xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or '')) + _xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE] + expected_p = expected_p[utils.MAX_XATTR_SIZE:] + assert len(expected_p) <= utils.MAX_XATTR_SIZE + res_d = utils.read_metadata(path) + assert res_d == {} + assert _xattr_op_cnt['get'] == 3, "%r" % _xattr_op_cnt + assert len(_xattrs.keys()) == 0, "Expected 0 keys, found %d" % len(_xattrs.keys()) + + def test_restore_metadata_none(self): + # No initial metadata + path = "/tmp/foo/i" + res_d = utils.restore_metadata(path, { 'b': 'y' }) + expected_d = { 'b': 'y' } + assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d) + assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt + assert _xattr_op_cnt['set'] == 1, "%r" % _xattr_op_cnt + + def test_restore_metadata(self): + # Initial metadata + path = "/tmp/foo/i" + initial_d = { 'a': 'z' } + xkey = _xkey(path, utils.METADATA_KEY) + _xattrs[xkey] = pickle.dumps(initial_d, utils.PICKLE_PROTOCOL) + res_d = utils.restore_metadata(path, { 'b': 'y' }) + expected_d = { 'a': 'z', 'b': 'y' } + assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d) + assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt + assert _xattr_op_cnt['set'] == 1, "%r" % _xattr_op_cnt + + def test_restore_metadata_nochange(self): + # Initial metadata but no changes + path = "/tmp/foo/i" + initial_d = { 'a': 'z' } + xkey = _xkey(path, utils.METADATA_KEY) + _xattrs[xkey] = pickle.dumps(initial_d, utils.PICKLE_PROTOCOL) + res_d = utils.restore_metadata(path, {}) + expected_d = { 'a': 'z' } + assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d) + assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt + assert _xattr_op_cnt['set'] == 0, "%r" % _xattr_op_cnt + + def test_add_timestamp_empty(self): + orig = {} + res = utils._add_timestamp(orig) + assert res == {} + + def test_add_timestamp_none(self): + orig = { 'a': 1, 'b': 2, 'c': 3 } + exp = { 'a': (1, 0), 'b': (2, 0), 'c': (3, 0) } + res = utils._add_timestamp(orig) + assert res == exp + + def test_add_timestamp_mixed(self): + orig = { 'a': 1, 'b': (2, 1), 'c': 3 } + exp = { 'a': (1, 0), 'b': (2, 1), 'c': (3, 0) } + res = utils._add_timestamp(orig) + assert res == exp + + def test_add_timestamp_all(self): + orig = { 'a': (1, 0), 'b': (2, 1), 'c': (3, 0) } + res = utils._add_timestamp(orig) + assert res == orig + + def test_get_etag_empty(self): + tf = tempfile.NamedTemporaryFile() + hd = utils._get_etag(tf.name) + assert hd == hashlib.md5().hexdigest() + + def test_get_etag(self): + tf = tempfile.NamedTemporaryFile() + tf.file.write('123' * utils.CHUNK_SIZE) + tf.file.flush() + hd = utils._get_etag(tf.name) + tf.file.seek(0) + md5 = hashlib.md5() + while True: + chunk = tf.file.read(utils.CHUNK_SIZE) + if not chunk: + break + md5.update(chunk) + assert hd == md5.hexdigest() + + def test_get_object_metadata_dne(self): + md = utils.get_object_metadata("/tmp/doesNotEx1st") + assert md == {} + + def test_get_object_metadata_err(self): + tf = tempfile.NamedTemporaryFile() + try: + md = utils.get_object_metadata(os.path.join(tf.name,"doesNotEx1st")) + except OSError as e: + assert e.errno != errno.ENOENT + else: + self.fail("Expected exception") + + obj_keys = (utils.X_TIMESTAMP, utils.X_CONTENT_TYPE, utils.X_ETAG, + utils.X_CONTENT_LENGTH, utils.X_TYPE, utils.X_OBJECT_TYPE) + + def test_get_object_metadata_file(self): + tf = tempfile.NamedTemporaryFile() + tf.file.write('123'); tf.file.flush() + md = utils.get_object_metadata(tf.name) + for key in self.obj_keys: + assert key in md, "Expected key %s in %r" % (key, md) + assert md[utils.X_TYPE] == utils.OBJECT + assert md[utils.X_OBJECT_TYPE] == utils.FILE + assert md[utils.X_CONTENT_TYPE] == utils.FILE_TYPE + assert md[utils.X_CONTENT_LENGTH] == os.path.getsize(tf.name) + assert md[utils.X_TIMESTAMP] == normalize_timestamp(os.path.getctime(tf.name)) + assert md[utils.X_ETAG] == utils._get_etag(tf.name) + + def test_get_object_metadata_dir(self): + td = tempfile.mkdtemp() + try: + md = utils.get_object_metadata(td) + for key in self.obj_keys: + assert key in md, "Expected key %s in %r" % (key, md) + assert md[utils.X_TYPE] == utils.OBJECT + assert md[utils.X_OBJECT_TYPE] == utils.DIR + assert md[utils.X_CONTENT_TYPE] == utils.DIR_TYPE + assert md[utils.X_CONTENT_LENGTH] == 0 + assert md[utils.X_TIMESTAMP] == normalize_timestamp(os.path.getctime(td)) + assert md[utils.X_ETAG] == hashlib.md5().hexdigest() + finally: + os.rmdir(td) + + def test_create_object_metadata_file(self): + tf = tempfile.NamedTemporaryFile() + tf.file.write('4567'); tf.file.flush() + r_md = utils.create_object_metadata(tf.name) + + xkey = _xkey(tf.name, utils.METADATA_KEY) + assert len(_xattrs.keys()) == 1 + assert xkey in _xattrs + assert _xattr_op_cnt['get'] == 1 + assert _xattr_op_cnt['set'] == 1 + md = pickle.loads(_xattrs[xkey]) + assert r_md == md + + for key in self.obj_keys: + assert key in md, "Expected key %s in %r" % (key, md) + assert md[utils.X_TYPE] == utils.OBJECT + assert md[utils.X_OBJECT_TYPE] == utils.FILE + assert md[utils.X_CONTENT_TYPE] == utils.FILE_TYPE + assert md[utils.X_CONTENT_LENGTH] == os.path.getsize(tf.name) + assert md[utils.X_TIMESTAMP] == normalize_timestamp(os.path.getctime(tf.name)) + assert md[utils.X_ETAG] == utils._get_etag(tf.name) + + def test_create_object_metadata_dir(self): + td = tempfile.mkdtemp() + try: + r_md = utils.create_object_metadata(td) + + xkey = _xkey(td, utils.METADATA_KEY) + assert len(_xattrs.keys()) == 1 + assert xkey in _xattrs + assert _xattr_op_cnt['get'] == 1 + assert _xattr_op_cnt['set'] == 1 + md = pickle.loads(_xattrs[xkey]) + assert r_md == md + + for key in self.obj_keys: + assert key in md, "Expected key %s in %r" % (key, md) + assert md[utils.X_TYPE] == utils.OBJECT + assert md[utils.X_OBJECT_TYPE] == utils.DIR + assert md[utils.X_CONTENT_TYPE] == utils.DIR_TYPE + assert md[utils.X_CONTENT_LENGTH] == 0 + assert md[utils.X_TIMESTAMP] == normalize_timestamp(os.path.getctime(td)) + assert md[utils.X_ETAG] == hashlib.md5().hexdigest() + finally: + os.rmdir(td) + + def test_get_container_metadata(self): + def _mock_get_container_details(path, memcache=None): + o_list = [ 'a', 'b', 'c' ] + o_count = 3 + b_used = 47 + return o_list, o_count, b_used + td = tempfile.mkdtemp() + orig_gcd = utils.get_container_details + utils.get_container_details = _mock_get_container_details + try: + exp_md = { + utils.X_TYPE: (utils.CONTAINER, 0), + utils.X_TIMESTAMP: (normalize_timestamp(os.path.getctime(td)), 0), + utils.X_PUT_TIMESTAMP: (normalize_timestamp(os.path.getmtime(td)), 0), + utils.X_OBJECTS_COUNT: (3, 0), + utils.X_BYTES_USED: (47, 0), + } + md = utils.get_container_metadata(td) + assert md == exp_md + finally: + utils.get_container_details = orig_gcd + os.rmdir(td) + + def test_get_account_metadata(self): + def _mock_get_account_details(path, memcache=None): + c_list = [ '123', 'abc' ] + c_count = 2 + return c_list, c_count + td = tempfile.mkdtemp() + orig_gad = utils.get_account_details + utils.get_account_details = _mock_get_account_details + try: + exp_md = { + utils.X_TYPE: (utils.ACCOUNT, 0), + utils.X_TIMESTAMP: (normalize_timestamp(os.path.getctime(td)), 0), + utils.X_PUT_TIMESTAMP: (normalize_timestamp(os.path.getmtime(td)), 0), + utils.X_OBJECTS_COUNT: (0, 0), + utils.X_BYTES_USED: (0, 0), + utils.X_CONTAINER_COUNT: (2, 0), + } + md = utils.get_account_metadata(td) + assert md == exp_md + finally: + utils.get_account_details = orig_gad + os.rmdir(td) + + cont_keys = [utils.X_TYPE, utils.X_TIMESTAMP, utils.X_PUT_TIMESTAMP, + utils.X_OBJECTS_COUNT, utils.X_BYTES_USED] + + def test_create_container_metadata(self): + td = tempfile.mkdtemp() + try: + r_md = utils.create_container_metadata(td) + + xkey = _xkey(td, utils.METADATA_KEY) + assert len(_xattrs.keys()) == 1 + assert xkey in _xattrs + assert _xattr_op_cnt['get'] == 1 + assert _xattr_op_cnt['set'] == 1 + md = pickle.loads(_xattrs[xkey]) + assert r_md == md + + for key in self.cont_keys: + assert key in md, "Expected key %s in %r" % (key, md) + assert md[utils.X_TYPE] == (utils.CONTAINER, 0) + assert md[utils.X_TIMESTAMP] == (normalize_timestamp(os.path.getctime(td)), 0) + assert md[utils.X_PUT_TIMESTAMP] == (normalize_timestamp(os.path.getmtime(td)), 0) + assert md[utils.X_OBJECTS_COUNT] == (0, 0) + assert md[utils.X_BYTES_USED] == (0, 0) + finally: + os.rmdir(td) + + acct_keys = [val for val in cont_keys] + acct_keys.append(utils.X_CONTAINER_COUNT) + + def test_create_account_metadata(self): + td = tempfile.mkdtemp() + try: + r_md = utils.create_account_metadata(td) + + xkey = _xkey(td, utils.METADATA_KEY) + assert len(_xattrs.keys()) == 1 + assert xkey in _xattrs + assert _xattr_op_cnt['get'] == 1 + assert _xattr_op_cnt['set'] == 1 + md = pickle.loads(_xattrs[xkey]) + assert r_md == md + + for key in self.acct_keys: + assert key in md, "Expected key %s in %r" % (key, md) + assert md[utils.X_TYPE] == (utils.ACCOUNT, 0) + assert md[utils.X_TIMESTAMP] == (normalize_timestamp(os.path.getctime(td)), 0) + assert md[utils.X_PUT_TIMESTAMP] == (normalize_timestamp(os.path.getmtime(td)), 0) + assert md[utils.X_OBJECTS_COUNT] == (0, 0) + assert md[utils.X_BYTES_USED] == (0, 0) + assert md[utils.X_CONTAINER_COUNT] == (0, 0) + finally: + os.rmdir(td) + + def test_container_details_uncached(self): + the_path = "/tmp/bar" + def mock_get_container_details_from_fs(cont_path): + bu = 5 + oc = 1 + ol = ['foo',] + dl = [('a',100),] + return utils.ContainerDetails(bu, oc, ol, dl) + orig_gcdff = utils._get_container_details_from_fs + utils._get_container_details_from_fs = mock_get_container_details_from_fs + try: + retval = utils.get_container_details(the_path) + cd = mock_get_container_details_from_fs(the_path) + assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) + finally: + utils._get_container_details_from_fs = orig_gcdff + + def test_container_details_cached_hit(self): + mc = SimMemcache() + the_path = "/tmp/bar" + def mock_get_container_details_from_fs(cont_path, bu_p=5): + bu = bu_p + oc = 1 + ol = ['foo',] + dl = [('a',100),] + return utils.ContainerDetails(bu, oc, ol, dl) + def mock_do_stat(path): + class MockStat(object): + def __init__(self, mtime): + self.st_mtime = mtime + return MockStat(100) + cd = mock_get_container_details_from_fs(the_path, bu_p=6) + mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path, cd) + orig_gcdff = utils._get_container_details_from_fs + utils._get_container_details_from_fs = mock_get_container_details_from_fs + orig_ds = utils.do_stat + utils.do_stat = mock_do_stat + try: + retval = utils.get_container_details(the_path, memcache=mc) + # If it did not properly use memcache, the default mocked version + # of get details from fs would return 5 bytes used instead of the + # 6 we specified above. + cd = mock_get_container_details_from_fs(the_path, bu_p=6) + assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) + finally: + utils._get_container_details_from_fs = orig_gcdff + utils.do_stat = orig_ds + + def test_container_details_cached_miss_key(self): + mc = SimMemcache() + the_path = "/tmp/bar" + def mock_get_container_details_from_fs(cont_path, bu_p=5): + bu = bu_p + oc = 1 + ol = ['foo',] + dl = [('a',100),] + return utils.ContainerDetails(bu, oc, ol, dl) + def mock_do_stat(path): + # Be sure we don't miss due to mtimes not matching + self.fail("do_stat should not have been called") + cd = mock_get_container_details_from_fs(the_path + "u", bu_p=6) + mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path + "u", cd) + orig_gcdff = utils._get_container_details_from_fs + utils._get_container_details_from_fs = mock_get_container_details_from_fs + orig_ds = utils.do_stat + utils.do_stat = mock_do_stat + try: + retval = utils.get_container_details(the_path, memcache=mc) + cd = mock_get_container_details_from_fs(the_path) + assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) + mkey = utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path + assert mkey in mc._d + finally: + utils._get_container_details_from_fs = orig_gcdff + utils.do_stat = orig_ds + + def test_container_details_cached_miss_dir_list(self): + mc = SimMemcache() + the_path = "/tmp/bar" + def mock_get_container_details_from_fs(cont_path, bu_p=5): + bu = bu_p + oc = 1 + ol = ['foo',] + dl = [] + return utils.ContainerDetails(bu, oc, ol, dl) + def mock_do_stat(path): + # Be sure we don't miss due to mtimes not matching + self.fail("do_stat should not have been called") + cd = mock_get_container_details_from_fs(the_path, bu_p=6) + mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path, cd) + orig_gcdff = utils._get_container_details_from_fs + utils._get_container_details_from_fs = mock_get_container_details_from_fs + orig_ds = utils.do_stat + utils.do_stat = mock_do_stat + try: + retval = utils.get_container_details(the_path, memcache=mc) + cd = mock_get_container_details_from_fs(the_path) + assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) + mkey = utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path + assert mkey in mc._d + assert 5 == mc._d[mkey].bytes_used + finally: + utils._get_container_details_from_fs = orig_gcdff + utils.do_stat = orig_ds + + def test_container_details_cached_miss_mtime(self): + mc = SimMemcache() + the_path = "/tmp/bar" + def mock_get_container_details_from_fs(cont_path, bu_p=5): + bu = bu_p + oc = 1 + ol = ['foo',] + dl = [('a',100),] + return utils.ContainerDetails(bu, oc, ol, dl) + def mock_do_stat(path): + # Be sure we miss due to mtimes not matching + class MockStat(object): + def __init__(self, mtime): + self.st_mtime = mtime + return MockStat(200) + cd = mock_get_container_details_from_fs(the_path, bu_p=6) + mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path, cd) + orig_gcdff = utils._get_container_details_from_fs + utils._get_container_details_from_fs = mock_get_container_details_from_fs + orig_ds = utils.do_stat + utils.do_stat = mock_do_stat + try: + retval = utils.get_container_details(the_path, memcache=mc) + cd = mock_get_container_details_from_fs(the_path) + assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) + mkey = utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path + assert mkey in mc._d + assert 5 == mc._d[mkey].bytes_used + finally: + utils._get_container_details_from_fs = orig_gcdff + utils.do_stat = orig_ds + + def test_account_details_uncached(self): + the_path = "/tmp/bar" + def mock_get_account_details_from_fs(acc_path, acc_stats): + mt = 100 + cc = 2 + cl = ['a', 'b'] + return utils.AccountDetails(mt, cc, cl) + orig_gcdff = utils._get_account_details_from_fs + utils._get_account_details_from_fs = mock_get_account_details_from_fs + try: + retval = utils.get_account_details(the_path) + ad = mock_get_account_details_from_fs(the_path, None) + assert retval == (ad.container_list, ad.container_count) + finally: + utils._get_account_details_from_fs = orig_gcdff + + def test_account_details_cached_hit(self): + mc = SimMemcache() + the_path = "/tmp/bar" + def mock_get_account_details_from_fs(acc_path, acc_stats): + mt = 100 + cc = 2 + cl = ['a', 'b'] + return utils.AccountDetails(mt, cc, cl) + def mock_do_stat(path): + class MockStat(object): + def __init__(self, mtime): + self.st_mtime = mtime + return MockStat(100) + ad = mock_get_account_details_from_fs(the_path, None) + ad.container_list = ['x', 'y'] + mc.set(utils.MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX + the_path, ad) + orig_gcdff = utils._get_account_details_from_fs + orig_ds = utils.do_stat + utils._get_account_details_from_fs = mock_get_account_details_from_fs + utils.do_stat = mock_do_stat + try: + retval = utils.get_account_details(the_path, memcache=mc) + assert retval == (ad.container_list, ad.container_count) + wrong_ad = mock_get_account_details_from_fs(the_path, None) + assert wrong_ad != ad + finally: + utils._get_account_details_from_fs = orig_gcdff + utils.do_stat = orig_ds + + def test_account_details_cached_miss(self): + mc = SimMemcache() + the_path = "/tmp/bar" + def mock_get_account_details_from_fs(acc_path, acc_stats): + mt = 100 + cc = 2 + cl = ['a', 'b'] + return utils.AccountDetails(mt, cc, cl) + def mock_do_stat(path): + class MockStat(object): + def __init__(self, mtime): + self.st_mtime = mtime + return MockStat(100) + ad = mock_get_account_details_from_fs(the_path, None) + ad.container_list = ['x', 'y'] + mc.set(utils.MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX + the_path + 'u', ad) + orig_gcdff = utils._get_account_details_from_fs + orig_ds = utils.do_stat + utils._get_account_details_from_fs = mock_get_account_details_from_fs + utils.do_stat = mock_do_stat + try: + retval = utils.get_account_details(the_path, memcache=mc) + correct_ad = mock_get_account_details_from_fs(the_path, None) + assert retval == (correct_ad.container_list, correct_ad.container_count) + assert correct_ad != ad + finally: + utils._get_account_details_from_fs = orig_gcdff + utils.do_stat = orig_ds + + def test_account_details_cached_miss_mtime(self): + mc = SimMemcache() + the_path = "/tmp/bar" + def mock_get_account_details_from_fs(acc_path, acc_stats): + mt = 100 + cc = 2 + cl = ['a', 'b'] + return utils.AccountDetails(mt, cc, cl) + def mock_do_stat(path): + class MockStat(object): + def __init__(self, mtime): + self.st_mtime = mtime + return MockStat(100) + ad = mock_get_account_details_from_fs(the_path, None) + ad.container_list = ['x', 'y'] + ad.mtime = 200 + mc.set(utils.MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX + the_path, ad) + orig_gcdff = utils._get_account_details_from_fs + orig_ds = utils.do_stat + utils._get_account_details_from_fs = mock_get_account_details_from_fs + utils.do_stat = mock_do_stat + try: + retval = utils.get_account_details(the_path, memcache=mc) + correct_ad = mock_get_account_details_from_fs(the_path, None) + assert retval == (correct_ad.container_list, correct_ad.container_count) + assert correct_ad != ad + finally: + utils._get_account_details_from_fs = orig_gcdff + utils.do_stat = orig_ds + + def test_get_container_details_from_fs(self): + td = tempfile.mkdtemp() + tf = tarfile.open("common/data/account_tree.tar.bz2", "r:bz2") + orig_cwd = os.getcwd() + os.chdir(td) + tf.extractall() + try: + ad = utils._get_account_details_from_fs(td, None) + assert ad.mtime == os.path.getmtime(td) + assert ad.container_count == 3 + assert set(ad.container_list) == set(['c1', 'c2', 'c3']) + finally: + os.chdir(orig_cwd) + shutil.rmtree(td) + + def test_get_container_details_from_fs_notadir(self): + tf = tempfile.NamedTemporaryFile() + cd = utils._get_container_details_from_fs(tf.name) + assert cd.bytes_used == 0 + assert cd.object_count == 0 + assert cd.obj_list == [] + assert cd.dir_list == [] + + def test_get_account_details_from_fs(self): + td = tempfile.mkdtemp() + tf = tarfile.open("common/data/container_tree.tar.bz2", "r:bz2") + orig_cwd = os.getcwd() + os.chdir(td) + tf.extractall() + try: + cd = utils._get_container_details_from_fs(td) + assert cd.bytes_used == 30, repr(cd.bytes_used) + assert cd.object_count == 8, repr(cd.object_count) + assert cd.obj_list == ['file1', 'file3', 'file2', + 'dir3', 'dir1', 'dir2', + 'dir1/file1', 'dir1/file2' + ], repr(cd.obj_list) + full_dir1 = os.path.join(td, 'dir1') + full_dir2 = os.path.join(td, 'dir2') + full_dir3 = os.path.join(td, 'dir3') + exp_dir_dict = { td: os.path.getmtime(td), + full_dir1: os.path.getmtime(full_dir1), + full_dir2: os.path.getmtime(full_dir2), + full_dir3: os.path.getmtime(full_dir3), + } + for d,m in cd.dir_list: + assert d in exp_dir_dict + assert exp_dir_dict[d] == m + finally: + os.chdir(orig_cwd) + shutil.rmtree(td) + + def test_get_account_details_from_fs_notadir_w_stats(self): + tf = tempfile.NamedTemporaryFile() + ad = utils._get_account_details_from_fs(tf.name, os.stat(tf.name)) + assert ad.mtime == os.path.getmtime(tf.name) + assert ad.container_count == 0 + assert ad.container_list == [] + + def test_get_account_details_from_fs_notadir(self): + tf = tempfile.NamedTemporaryFile() + ad = utils._get_account_details_from_fs(tf.name, None) + assert ad.mtime == os.path.getmtime(tf.name) + assert ad.container_count == 0 + assert ad.container_list == [] diff --git a/swift/1.4.8/test/unit/plugins/__init__.py b/swift/1.4.8/test/unit/plugins/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/swift/1.4.8/test/unit/plugins/data/account_tree.tar.bz2 b/swift/1.4.8/test/unit/plugins/data/account_tree.tar.bz2 deleted file mode 100644 index cb23e4dd7..000000000 Binary files a/swift/1.4.8/test/unit/plugins/data/account_tree.tar.bz2 and /dev/null differ diff --git a/swift/1.4.8/test/unit/plugins/data/container_tree.tar.bz2 b/swift/1.4.8/test/unit/plugins/data/container_tree.tar.bz2 deleted file mode 100644 index b4a149285..000000000 Binary files a/swift/1.4.8/test/unit/plugins/data/container_tree.tar.bz2 and /dev/null differ diff --git a/swift/1.4.8/test/unit/plugins/test_utils.py b/swift/1.4.8/test/unit/plugins/test_utils.py deleted file mode 100644 index 92c7ff005..000000000 --- a/swift/1.4.8/test/unit/plugins/test_utils.py +++ /dev/null @@ -1,818 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" Tests for plugins.utils """ - -import os -import unittest -import errno -import xattr -import cPickle as pickle -import tempfile -import hashlib -import tarfile -import shutil -from collections import defaultdict -from swift.common.utils import normalize_timestamp -from swift.plugins import utils - -# -# Somewhat hacky way of emulating the operation of xattr calls. They are made -# against a dictionary that stores the xattr key/value pairs. -# -_xattrs = {} -_xattr_op_cnt = defaultdict(int) -_xattr_err = {} - -def _xkey(path, key): - return "%s:%s" % (path, key) - -def _setxattr(path, key, value): - _xattr_op_cnt['set'] += 1 - xkey = _xkey(path, key) - if xkey in _xattr_err: - e = IOError() - e.errno = _xattr_err[xkey] - raise e - global _xattrs - _xattrs[xkey] = value - -def _getxattr(path, key): - _xattr_op_cnt['get'] += 1 - xkey = _xkey(path, key) - if xkey in _xattr_err: - e = IOError() - e.errno = _xattr_err[xkey] - raise e - global _xattrs - if xkey in _xattrs: - ret_val = _xattrs[xkey] - else: - e = IOError("Fake IOError") - e.errno = errno.ENODATA - raise e - return ret_val - -def _removexattr(path, key): - _xattr_op_cnt['remove'] += 1 - xkey = _xkey(path, key) - if xkey in _xattr_err: - e = IOError() - e.errno = _xattr_err[xkey] - raise e - global _xattrs - if xkey in _xattrs: - del _xattrs[xkey] - else: - e = IOError("Fake IOError") - e.errno = errno.ENODATA - raise e - -def _initxattr(): - global _xattrs - _xattrs = {} - global _xattr_op_cnt - _xattr_op_cnt = defaultdict(int) - global _xattr_err - _xattr_err = {} - - # Save the current methods - global _xattr_set; _xattr_set = xattr.set - global _xattr_get; _xattr_get = xattr.get - global _xattr_remove; _xattr_remove = xattr.remove - - # Monkey patch the calls we use with our internal unit test versions - xattr.set = _setxattr - xattr.get = _getxattr - xattr.remove = _removexattr - -def _destroyxattr(): - # Restore the current methods just in case - global _xattr_set; xattr.set = _xattr_set - global _xattr_get; xattr.get = _xattr_get - global _xattr_remove; xattr.remove = _xattr_remove - # Destroy the stored values and - global _xattrs; _xattrs = None - - -class SimMemcache(object): - def __init__(self): - self._d = {} - - def get(self, key): - return self._d.get(key, None) - - def set(self, key, value): - self._d[key] = value - - -class TestUtils(unittest.TestCase): - """ Tests for plugins.utils """ - - def setUp(self): - _initxattr() - - def tearDown(self): - _destroyxattr() - - def test_write_metadata(self): - path = "/tmp/foo/w" - orig_d = { 'bar' : 'foo' } - utils.write_metadata(path, orig_d) - xkey = _xkey(path, utils.METADATA_KEY) - assert len(_xattrs.keys()) == 1 - assert xkey in _xattrs - assert orig_d == pickle.loads(_xattrs[xkey]) - assert _xattr_op_cnt['set'] == 1 - - def test_write_metadata_err(self): - path = "/tmp/foo/w" - orig_d = { 'bar' : 'foo' } - xkey = _xkey(path, utils.METADATA_KEY) - _xattr_err[xkey] = errno.EOPNOTSUPP - try: - utils.write_metadata(path, orig_d) - except IOError as e: - assert e.errno == errno.EOPNOTSUPP - assert len(_xattrs.keys()) == 0 - assert _xattr_op_cnt['set'] == 1 - else: - self.fail("Expected an IOError exception on write") - - def test_write_metadata_multiple(self): - # At 64 KB an xattr key/value pair, this should generate three keys. - path = "/tmp/foo/w" - orig_d = { 'bar' : 'x' * 150000 } - utils.write_metadata(path, orig_d) - assert len(_xattrs.keys()) == 3, "Expected 3 keys, found %d" % len(_xattrs.keys()) - payload = '' - for i in range(0,3): - xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or '')) - assert xkey in _xattrs - assert len(_xattrs[xkey]) <= utils.MAX_XATTR_SIZE - payload += _xattrs[xkey] - assert orig_d == pickle.loads(payload) - assert _xattr_op_cnt['set'] == 3, "%r" % _xattr_op_cnt - - def test_clean_metadata(self): - path = "/tmp/foo/c" - expected_d = { 'a': 'y' * 150000 } - expected_p = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL) - for i in range(0,3): - xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or '')) - _xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE] - expected_p = expected_p[utils.MAX_XATTR_SIZE:] - assert not expected_p - utils.clean_metadata(path) - assert _xattr_op_cnt['remove'] == 4, "%r" % _xattr_op_cnt - - def test_clean_metadata_err(self): - path = "/tmp/foo/c" - xkey = _xkey(path, utils.METADATA_KEY) - _xattrs[xkey] = pickle.dumps({ 'a': 'y' }, utils.PICKLE_PROTOCOL) - _xattr_err[xkey] = errno.EOPNOTSUPP - try: - utils.clean_metadata(path) - except IOError as e: - assert e.errno == errno.EOPNOTSUPP - assert _xattr_op_cnt['remove'] == 1, "%r" % _xattr_op_cnt - else: - self.fail("Expected an IOError exception on remove") - - def test_read_metadata(self): - path = "/tmp/foo/r" - expected_d = { 'a': 'y' } - xkey = _xkey(path, utils.METADATA_KEY) - _xattrs[xkey] = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL) - res_d = utils.read_metadata(path) - assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d) - assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt - - def test_read_metadata_notfound(self): - path = "/tmp/foo/r" - res_d = utils.read_metadata(path) - assert res_d == {} - assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt - - def test_read_metadata_err(self): - path = "/tmp/foo/r" - expected_d = { 'a': 'y' } - xkey = _xkey(path, utils.METADATA_KEY) - _xattrs[xkey] = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL) - _xattr_err[xkey] = errno.EOPNOTSUPP - try: - res_d = utils.read_metadata(path) - except IOError as e: - assert e.errno == errno.EOPNOTSUPP - assert (_xattr_op_cnt['get'] == 1), "%r" % _xattr_op_cnt - else: - self.fail("Expected an IOError exception on get") - - def test_read_metadata_multiple(self): - path = "/tmp/foo/r" - expected_d = { 'a': 'y' * 150000 } - expected_p = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL) - for i in range(0,3): - xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or '')) - _xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE] - expected_p = expected_p[utils.MAX_XATTR_SIZE:] - assert not expected_p - res_d = utils.read_metadata(path) - assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d) - assert _xattr_op_cnt['get'] == 3, "%r" % _xattr_op_cnt - - def test_read_metadata_multiple_one_missing(self): - path = "/tmp/foo/r" - expected_d = { 'a': 'y' * 150000 } - expected_p = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL) - for i in range(0,2): - xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or '')) - _xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE] - expected_p = expected_p[utils.MAX_XATTR_SIZE:] - assert len(expected_p) <= utils.MAX_XATTR_SIZE - res_d = utils.read_metadata(path) - assert res_d == {} - assert _xattr_op_cnt['get'] == 3, "%r" % _xattr_op_cnt - assert len(_xattrs.keys()) == 0, "Expected 0 keys, found %d" % len(_xattrs.keys()) - - def test_restore_metadata_none(self): - # No initial metadata - path = "/tmp/foo/i" - res_d = utils.restore_metadata(path, { 'b': 'y' }) - expected_d = { 'b': 'y' } - assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d) - assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt - assert _xattr_op_cnt['set'] == 1, "%r" % _xattr_op_cnt - - def test_restore_metadata(self): - # Initial metadata - path = "/tmp/foo/i" - initial_d = { 'a': 'z' } - xkey = _xkey(path, utils.METADATA_KEY) - _xattrs[xkey] = pickle.dumps(initial_d, utils.PICKLE_PROTOCOL) - res_d = utils.restore_metadata(path, { 'b': 'y' }) - expected_d = { 'a': 'z', 'b': 'y' } - assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d) - assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt - assert _xattr_op_cnt['set'] == 1, "%r" % _xattr_op_cnt - - def test_restore_metadata_nochange(self): - # Initial metadata but no changes - path = "/tmp/foo/i" - initial_d = { 'a': 'z' } - xkey = _xkey(path, utils.METADATA_KEY) - _xattrs[xkey] = pickle.dumps(initial_d, utils.PICKLE_PROTOCOL) - res_d = utils.restore_metadata(path, {}) - expected_d = { 'a': 'z' } - assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d) - assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt - assert _xattr_op_cnt['set'] == 0, "%r" % _xattr_op_cnt - - def test_add_timestamp_empty(self): - orig = {} - res = utils._add_timestamp(orig) - assert res == {} - - def test_add_timestamp_none(self): - orig = { 'a': 1, 'b': 2, 'c': 3 } - exp = { 'a': (1, 0), 'b': (2, 0), 'c': (3, 0) } - res = utils._add_timestamp(orig) - assert res == exp - - def test_add_timestamp_mixed(self): - orig = { 'a': 1, 'b': (2, 1), 'c': 3 } - exp = { 'a': (1, 0), 'b': (2, 1), 'c': (3, 0) } - res = utils._add_timestamp(orig) - assert res == exp - - def test_add_timestamp_all(self): - orig = { 'a': (1, 0), 'b': (2, 1), 'c': (3, 0) } - res = utils._add_timestamp(orig) - assert res == orig - - def test_get_etag_empty(self): - tf = tempfile.NamedTemporaryFile() - hd = utils._get_etag(tf.name) - assert hd == hashlib.md5().hexdigest() - - def test_get_etag(self): - tf = tempfile.NamedTemporaryFile() - tf.file.write('123' * utils.CHUNK_SIZE) - tf.file.flush() - hd = utils._get_etag(tf.name) - tf.file.seek(0) - md5 = hashlib.md5() - while True: - chunk = tf.file.read(utils.CHUNK_SIZE) - if not chunk: - break - md5.update(chunk) - assert hd == md5.hexdigest() - - def test_get_object_metadata_dne(self): - md = utils.get_object_metadata("/tmp/doesNotEx1st") - assert md == {} - - def test_get_object_metadata_err(self): - tf = tempfile.NamedTemporaryFile() - try: - md = utils.get_object_metadata(os.path.join(tf.name,"doesNotEx1st")) - except OSError as e: - assert e.errno != errno.ENOENT - else: - self.fail("Expected exception") - - obj_keys = (utils.X_TIMESTAMP, utils.X_CONTENT_TYPE, utils.X_ETAG, - utils.X_CONTENT_LENGTH, utils.X_TYPE, utils.X_OBJECT_TYPE) - - def test_get_object_metadata_file(self): - tf = tempfile.NamedTemporaryFile() - tf.file.write('123'); tf.file.flush() - md = utils.get_object_metadata(tf.name) - for key in self.obj_keys: - assert key in md, "Expected key %s in %r" % (key, md) - assert md[utils.X_TYPE] == utils.OBJECT - assert md[utils.X_OBJECT_TYPE] == utils.FILE - assert md[utils.X_CONTENT_TYPE] == utils.FILE_TYPE - assert md[utils.X_CONTENT_LENGTH] == os.path.getsize(tf.name) - assert md[utils.X_TIMESTAMP] == normalize_timestamp(os.path.getctime(tf.name)) - assert md[utils.X_ETAG] == utils._get_etag(tf.name) - - def test_get_object_metadata_dir(self): - td = tempfile.mkdtemp() - try: - md = utils.get_object_metadata(td) - for key in self.obj_keys: - assert key in md, "Expected key %s in %r" % (key, md) - assert md[utils.X_TYPE] == utils.OBJECT - assert md[utils.X_OBJECT_TYPE] == utils.DIR - assert md[utils.X_CONTENT_TYPE] == utils.DIR_TYPE - assert md[utils.X_CONTENT_LENGTH] == 0 - assert md[utils.X_TIMESTAMP] == normalize_timestamp(os.path.getctime(td)) - assert md[utils.X_ETAG] == hashlib.md5().hexdigest() - finally: - os.rmdir(td) - - def test_create_object_metadata_file(self): - tf = tempfile.NamedTemporaryFile() - tf.file.write('4567'); tf.file.flush() - r_md = utils.create_object_metadata(tf.name) - - xkey = _xkey(tf.name, utils.METADATA_KEY) - assert len(_xattrs.keys()) == 1 - assert xkey in _xattrs - assert _xattr_op_cnt['get'] == 1 - assert _xattr_op_cnt['set'] == 1 - md = pickle.loads(_xattrs[xkey]) - assert r_md == md - - for key in self.obj_keys: - assert key in md, "Expected key %s in %r" % (key, md) - assert md[utils.X_TYPE] == utils.OBJECT - assert md[utils.X_OBJECT_TYPE] == utils.FILE - assert md[utils.X_CONTENT_TYPE] == utils.FILE_TYPE - assert md[utils.X_CONTENT_LENGTH] == os.path.getsize(tf.name) - assert md[utils.X_TIMESTAMP] == normalize_timestamp(os.path.getctime(tf.name)) - assert md[utils.X_ETAG] == utils._get_etag(tf.name) - - def test_create_object_metadata_dir(self): - td = tempfile.mkdtemp() - try: - r_md = utils.create_object_metadata(td) - - xkey = _xkey(td, utils.METADATA_KEY) - assert len(_xattrs.keys()) == 1 - assert xkey in _xattrs - assert _xattr_op_cnt['get'] == 1 - assert _xattr_op_cnt['set'] == 1 - md = pickle.loads(_xattrs[xkey]) - assert r_md == md - - for key in self.obj_keys: - assert key in md, "Expected key %s in %r" % (key, md) - assert md[utils.X_TYPE] == utils.OBJECT - assert md[utils.X_OBJECT_TYPE] == utils.DIR - assert md[utils.X_CONTENT_TYPE] == utils.DIR_TYPE - assert md[utils.X_CONTENT_LENGTH] == 0 - assert md[utils.X_TIMESTAMP] == normalize_timestamp(os.path.getctime(td)) - assert md[utils.X_ETAG] == hashlib.md5().hexdigest() - finally: - os.rmdir(td) - - def test_get_container_metadata(self): - def _mock_get_container_details(path, memcache=None): - o_list = [ 'a', 'b', 'c' ] - o_count = 3 - b_used = 47 - return o_list, o_count, b_used - td = tempfile.mkdtemp() - orig_gcd = utils.get_container_details - utils.get_container_details = _mock_get_container_details - try: - exp_md = { - utils.X_TYPE: (utils.CONTAINER, 0), - utils.X_TIMESTAMP: (normalize_timestamp(os.path.getctime(td)), 0), - utils.X_PUT_TIMESTAMP: (normalize_timestamp(os.path.getmtime(td)), 0), - utils.X_OBJECTS_COUNT: (3, 0), - utils.X_BYTES_USED: (47, 0), - } - md = utils.get_container_metadata(td) - assert md == exp_md - finally: - utils.get_container_details = orig_gcd - os.rmdir(td) - - def test_get_account_metadata(self): - def _mock_get_account_details(path, memcache=None): - c_list = [ '123', 'abc' ] - c_count = 2 - return c_list, c_count - td = tempfile.mkdtemp() - orig_gad = utils.get_account_details - utils.get_account_details = _mock_get_account_details - try: - exp_md = { - utils.X_TYPE: (utils.ACCOUNT, 0), - utils.X_TIMESTAMP: (normalize_timestamp(os.path.getctime(td)), 0), - utils.X_PUT_TIMESTAMP: (normalize_timestamp(os.path.getmtime(td)), 0), - utils.X_OBJECTS_COUNT: (0, 0), - utils.X_BYTES_USED: (0, 0), - utils.X_CONTAINER_COUNT: (2, 0), - } - md = utils.get_account_metadata(td) - assert md == exp_md - finally: - utils.get_account_details = orig_gad - os.rmdir(td) - - cont_keys = [utils.X_TYPE, utils.X_TIMESTAMP, utils.X_PUT_TIMESTAMP, - utils.X_OBJECTS_COUNT, utils.X_BYTES_USED] - - def test_create_container_metadata(self): - td = tempfile.mkdtemp() - try: - r_md = utils.create_container_metadata(td) - - xkey = _xkey(td, utils.METADATA_KEY) - assert len(_xattrs.keys()) == 1 - assert xkey in _xattrs - assert _xattr_op_cnt['get'] == 1 - assert _xattr_op_cnt['set'] == 1 - md = pickle.loads(_xattrs[xkey]) - assert r_md == md - - for key in self.cont_keys: - assert key in md, "Expected key %s in %r" % (key, md) - assert md[utils.X_TYPE] == (utils.CONTAINER, 0) - assert md[utils.X_TIMESTAMP] == (normalize_timestamp(os.path.getctime(td)), 0) - assert md[utils.X_PUT_TIMESTAMP] == (normalize_timestamp(os.path.getmtime(td)), 0) - assert md[utils.X_OBJECTS_COUNT] == (0, 0) - assert md[utils.X_BYTES_USED] == (0, 0) - finally: - os.rmdir(td) - - acct_keys = [val for val in cont_keys] - acct_keys.append(utils.X_CONTAINER_COUNT) - - def test_create_account_metadata(self): - td = tempfile.mkdtemp() - try: - r_md = utils.create_account_metadata(td) - - xkey = _xkey(td, utils.METADATA_KEY) - assert len(_xattrs.keys()) == 1 - assert xkey in _xattrs - assert _xattr_op_cnt['get'] == 1 - assert _xattr_op_cnt['set'] == 1 - md = pickle.loads(_xattrs[xkey]) - assert r_md == md - - for key in self.acct_keys: - assert key in md, "Expected key %s in %r" % (key, md) - assert md[utils.X_TYPE] == (utils.ACCOUNT, 0) - assert md[utils.X_TIMESTAMP] == (normalize_timestamp(os.path.getctime(td)), 0) - assert md[utils.X_PUT_TIMESTAMP] == (normalize_timestamp(os.path.getmtime(td)), 0) - assert md[utils.X_OBJECTS_COUNT] == (0, 0) - assert md[utils.X_BYTES_USED] == (0, 0) - assert md[utils.X_CONTAINER_COUNT] == (0, 0) - finally: - os.rmdir(td) - - def test_container_details_uncached(self): - the_path = "/tmp/bar" - def mock_get_container_details_from_fs(cont_path): - bu = 5 - oc = 1 - ol = ['foo',] - dl = [('a',100),] - return utils.ContainerDetails(bu, oc, ol, dl) - orig_gcdff = utils._get_container_details_from_fs - utils._get_container_details_from_fs = mock_get_container_details_from_fs - try: - retval = utils.get_container_details(the_path) - cd = mock_get_container_details_from_fs(the_path) - assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) - finally: - utils._get_container_details_from_fs = orig_gcdff - - def test_container_details_cached_hit(self): - mc = SimMemcache() - the_path = "/tmp/bar" - def mock_get_container_details_from_fs(cont_path, bu_p=5): - bu = bu_p - oc = 1 - ol = ['foo',] - dl = [('a',100),] - return utils.ContainerDetails(bu, oc, ol, dl) - def mock_do_stat(path): - class MockStat(object): - def __init__(self, mtime): - self.st_mtime = mtime - return MockStat(100) - cd = mock_get_container_details_from_fs(the_path, bu_p=6) - mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path, cd) - orig_gcdff = utils._get_container_details_from_fs - utils._get_container_details_from_fs = mock_get_container_details_from_fs - orig_ds = utils.do_stat - utils.do_stat = mock_do_stat - try: - retval = utils.get_container_details(the_path, memcache=mc) - # If it did not properly use memcache, the default mocked version - # of get details from fs would return 5 bytes used instead of the - # 6 we specified above. - cd = mock_get_container_details_from_fs(the_path, bu_p=6) - assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) - finally: - utils._get_container_details_from_fs = orig_gcdff - utils.do_stat = orig_ds - - def test_container_details_cached_miss_key(self): - mc = SimMemcache() - the_path = "/tmp/bar" - def mock_get_container_details_from_fs(cont_path, bu_p=5): - bu = bu_p - oc = 1 - ol = ['foo',] - dl = [('a',100),] - return utils.ContainerDetails(bu, oc, ol, dl) - def mock_do_stat(path): - # Be sure we don't miss due to mtimes not matching - self.fail("do_stat should not have been called") - cd = mock_get_container_details_from_fs(the_path + "u", bu_p=6) - mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path + "u", cd) - orig_gcdff = utils._get_container_details_from_fs - utils._get_container_details_from_fs = mock_get_container_details_from_fs - orig_ds = utils.do_stat - utils.do_stat = mock_do_stat - try: - retval = utils.get_container_details(the_path, memcache=mc) - cd = mock_get_container_details_from_fs(the_path) - assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) - mkey = utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path - assert mkey in mc._d - finally: - utils._get_container_details_from_fs = orig_gcdff - utils.do_stat = orig_ds - - def test_container_details_cached_miss_dir_list(self): - mc = SimMemcache() - the_path = "/tmp/bar" - def mock_get_container_details_from_fs(cont_path, bu_p=5): - bu = bu_p - oc = 1 - ol = ['foo',] - dl = [] - return utils.ContainerDetails(bu, oc, ol, dl) - def mock_do_stat(path): - # Be sure we don't miss due to mtimes not matching - self.fail("do_stat should not have been called") - cd = mock_get_container_details_from_fs(the_path, bu_p=6) - mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path, cd) - orig_gcdff = utils._get_container_details_from_fs - utils._get_container_details_from_fs = mock_get_container_details_from_fs - orig_ds = utils.do_stat - utils.do_stat = mock_do_stat - try: - retval = utils.get_container_details(the_path, memcache=mc) - cd = mock_get_container_details_from_fs(the_path) - assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) - mkey = utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path - assert mkey in mc._d - assert 5 == mc._d[mkey].bytes_used - finally: - utils._get_container_details_from_fs = orig_gcdff - utils.do_stat = orig_ds - - def test_container_details_cached_miss_mtime(self): - mc = SimMemcache() - the_path = "/tmp/bar" - def mock_get_container_details_from_fs(cont_path, bu_p=5): - bu = bu_p - oc = 1 - ol = ['foo',] - dl = [('a',100),] - return utils.ContainerDetails(bu, oc, ol, dl) - def mock_do_stat(path): - # Be sure we miss due to mtimes not matching - class MockStat(object): - def __init__(self, mtime): - self.st_mtime = mtime - return MockStat(200) - cd = mock_get_container_details_from_fs(the_path, bu_p=6) - mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path, cd) - orig_gcdff = utils._get_container_details_from_fs - utils._get_container_details_from_fs = mock_get_container_details_from_fs - orig_ds = utils.do_stat - utils.do_stat = mock_do_stat - try: - retval = utils.get_container_details(the_path, memcache=mc) - cd = mock_get_container_details_from_fs(the_path) - assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) - mkey = utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path - assert mkey in mc._d - assert 5 == mc._d[mkey].bytes_used - finally: - utils._get_container_details_from_fs = orig_gcdff - utils.do_stat = orig_ds - - def test_account_details_uncached(self): - the_path = "/tmp/bar" - def mock_get_account_details_from_fs(acc_path, acc_stats): - mt = 100 - cc = 2 - cl = ['a', 'b'] - return utils.AccountDetails(mt, cc, cl) - orig_gcdff = utils._get_account_details_from_fs - utils._get_account_details_from_fs = mock_get_account_details_from_fs - try: - retval = utils.get_account_details(the_path) - ad = mock_get_account_details_from_fs(the_path, None) - assert retval == (ad.container_list, ad.container_count) - finally: - utils._get_account_details_from_fs = orig_gcdff - - def test_account_details_cached_hit(self): - mc = SimMemcache() - the_path = "/tmp/bar" - def mock_get_account_details_from_fs(acc_path, acc_stats): - mt = 100 - cc = 2 - cl = ['a', 'b'] - return utils.AccountDetails(mt, cc, cl) - def mock_do_stat(path): - class MockStat(object): - def __init__(self, mtime): - self.st_mtime = mtime - return MockStat(100) - ad = mock_get_account_details_from_fs(the_path, None) - ad.container_list = ['x', 'y'] - mc.set(utils.MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX + the_path, ad) - orig_gcdff = utils._get_account_details_from_fs - orig_ds = utils.do_stat - utils._get_account_details_from_fs = mock_get_account_details_from_fs - utils.do_stat = mock_do_stat - try: - retval = utils.get_account_details(the_path, memcache=mc) - assert retval == (ad.container_list, ad.container_count) - wrong_ad = mock_get_account_details_from_fs(the_path, None) - assert wrong_ad != ad - finally: - utils._get_account_details_from_fs = orig_gcdff - utils.do_stat = orig_ds - - def test_account_details_cached_miss(self): - mc = SimMemcache() - the_path = "/tmp/bar" - def mock_get_account_details_from_fs(acc_path, acc_stats): - mt = 100 - cc = 2 - cl = ['a', 'b'] - return utils.AccountDetails(mt, cc, cl) - def mock_do_stat(path): - class MockStat(object): - def __init__(self, mtime): - self.st_mtime = mtime - return MockStat(100) - ad = mock_get_account_details_from_fs(the_path, None) - ad.container_list = ['x', 'y'] - mc.set(utils.MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX + the_path + 'u', ad) - orig_gcdff = utils._get_account_details_from_fs - orig_ds = utils.do_stat - utils._get_account_details_from_fs = mock_get_account_details_from_fs - utils.do_stat = mock_do_stat - try: - retval = utils.get_account_details(the_path, memcache=mc) - correct_ad = mock_get_account_details_from_fs(the_path, None) - assert retval == (correct_ad.container_list, correct_ad.container_count) - assert correct_ad != ad - finally: - utils._get_account_details_from_fs = orig_gcdff - utils.do_stat = orig_ds - - def test_account_details_cached_miss_mtime(self): - mc = SimMemcache() - the_path = "/tmp/bar" - def mock_get_account_details_from_fs(acc_path, acc_stats): - mt = 100 - cc = 2 - cl = ['a', 'b'] - return utils.AccountDetails(mt, cc, cl) - def mock_do_stat(path): - class MockStat(object): - def __init__(self, mtime): - self.st_mtime = mtime - return MockStat(100) - ad = mock_get_account_details_from_fs(the_path, None) - ad.container_list = ['x', 'y'] - ad.mtime = 200 - mc.set(utils.MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX + the_path, ad) - orig_gcdff = utils._get_account_details_from_fs - orig_ds = utils.do_stat - utils._get_account_details_from_fs = mock_get_account_details_from_fs - utils.do_stat = mock_do_stat - try: - retval = utils.get_account_details(the_path, memcache=mc) - correct_ad = mock_get_account_details_from_fs(the_path, None) - assert retval == (correct_ad.container_list, correct_ad.container_count) - assert correct_ad != ad - finally: - utils._get_account_details_from_fs = orig_gcdff - utils.do_stat = orig_ds - - def test_get_container_details_from_fs(self): - td = tempfile.mkdtemp() - tf = tarfile.open("plugins/data/account_tree.tar.bz2", "r:bz2") - orig_cwd = os.getcwd() - os.chdir(td) - tf.extractall() - try: - ad = utils._get_account_details_from_fs(td, None) - assert ad.mtime == os.path.getmtime(td) - assert ad.container_count == 3 - assert set(ad.container_list) == set(['c1', 'c2', 'c3']) - finally: - os.chdir(orig_cwd) - shutil.rmtree(td) - - def test_get_container_details_from_fs_notadir(self): - tf = tempfile.NamedTemporaryFile() - cd = utils._get_container_details_from_fs(tf.name) - assert cd.bytes_used == 0 - assert cd.object_count == 0 - assert cd.obj_list == [] - assert cd.dir_list == [] - - def test_get_account_details_from_fs(self): - td = tempfile.mkdtemp() - tf = tarfile.open("plugins/data/container_tree.tar.bz2", "r:bz2") - orig_cwd = os.getcwd() - os.chdir(td) - tf.extractall() - try: - cd = utils._get_container_details_from_fs(td) - assert cd.bytes_used == 30, repr(cd.bytes_used) - assert cd.object_count == 8, repr(cd.object_count) - assert cd.obj_list == ['file1', 'file3', 'file2', - 'dir3', 'dir1', 'dir2', - 'dir1/file1', 'dir1/file2' - ], repr(cd.obj_list) - full_dir1 = os.path.join(td, 'dir1') - full_dir2 = os.path.join(td, 'dir2') - full_dir3 = os.path.join(td, 'dir3') - exp_dir_dict = { td: os.path.getmtime(td), - full_dir1: os.path.getmtime(full_dir1), - full_dir2: os.path.getmtime(full_dir2), - full_dir3: os.path.getmtime(full_dir3), - } - for d,m in cd.dir_list: - assert d in exp_dir_dict - assert exp_dir_dict[d] == m - finally: - os.chdir(orig_cwd) - shutil.rmtree(td) - - def test_get_account_details_from_fs_notadir_w_stats(self): - tf = tempfile.NamedTemporaryFile() - ad = utils._get_account_details_from_fs(tf.name, os.stat(tf.name)) - assert ad.mtime == os.path.getmtime(tf.name) - assert ad.container_count == 0 - assert ad.container_list == [] - - def test_get_account_details_from_fs_notadir(self): - tf = tempfile.NamedTemporaryFile() - ad = utils._get_account_details_from_fs(tf.name, None) - assert ad.mtime == os.path.getmtime(tf.name) - assert ad.container_count == 0 - assert ad.container_list == [] -- cgit