Rebase to OpenStack Swift v1.10.0.172.g9fe7748v1.10.2-0v1.10.2

* Updated Proxy unit test
* Updated Functional tests
* Updated Tox to point to the new swift snapshot available
  on http://launchpad.net/gluster-swift

Change-Id: Ia91593c6a28d5a3fe70715ddc60546931ae71635
Signed-off-by: Luis Pabon <lpabon@redhat.com>
Reviewed-on: http://review.gluster.org/6445

2 files changed, 385 insertions, 25 deletions

diff --git a/test/functional/swift_test_client.py b/test/functional/swift_test_client.py
index 9c4766e..d407a33 100644
--- a/test/functional/swift_test_client.py
+++ b/test/functional/swift_test_client.py
@@ -1,3 +1,4 @@
+# Copyright (c) 2010-2012 OpenStack Foundation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -39,16 +40,28 @@ class RequestError(Exception):
 
 
 class ResponseError(Exception):
-    def __init__(self, response):
+    def __init__(self, response, method, path):
         self.status = response.status
         self.reason = response.reason
-        Exception.__init__(self)
+        self.method = method
+        self.path = path
+        self.headers = response.getheaders()
+
+        for name, value in self.headers:
+            if name.lower() == 'x-trans-id':
+                self.txid = value
+                break
+        else:
+            self.txid = None
+
+        super(ResponseError, self).__init__()
 
     def __str__(self):
-        return '%d: %s' % (self.status, self.reason)
+        return repr(self)
 
     def __repr__(self):
-        return '%d: %s' % (self.status, self.reason)
+        return '%d: %r (%r %r) txid=%s' % (
+            self.status, self.reason, self.method, self.path, self.txid)
 
 
 def listing_empty(method):
@@ -266,10 +279,6 @@ class Connection(object):
                           for (x, y) in parms.items()]
             path = '%s?%s' % (path, '&'.join(query_args))
 
-            query_args = ['%s=%s' % (urllib.quote(x),
-                          urllib.quote(str(y))) for (x, y) in parms.items()]
-            path = '%s?%s' % (path, '&'.join(query_args))
-
         self.connection = self.conn_class(self.storage_host,
                                           port=self.storage_port)
         #self.connection.set_debuglevel(3)
@@ -355,7 +364,8 @@ class Account(Base):
         elif status == 204:
             return []
 
-        raise ResponseError(self.conn.response)
+        raise ResponseError(self.conn.response, 'GET',
+                            self.conn.make_path(self.path))
 
     def delete_containers(self):
         for c in listing_items(self.containers):
@@ -369,7 +379,8 @@ class Account(Base):
         if self.conn.make_request('HEAD', self.path, hdrs=hdrs,
                                   parms=parms, cfg=cfg) != 204:
 
-            raise ResponseError(self.conn.response)
+            raise ResponseError(self.conn.response, 'HEAD',
+                                self.conn.make_path(self.path))
 
         fields = [['object_count', 'x-account-object-count'],
                   ['container_count', 'x-account-container-count'],
@@ -457,7 +468,8 @@ class Container(Base):
         elif status == 204:
             return []
 
-        raise ResponseError(self.conn.response)
+        raise ResponseError(self.conn.response, 'GET',
+                            self.conn.make_path(self.path))
 
     def info(self, hdrs={}, parms={}, cfg={}):
         self.conn.make_request('HEAD', self.path, hdrs=hdrs,
@@ -469,7 +481,8 @@ class Container(Base):
 
             return self.header_fields(fields)
 
-        raise ResponseError(self.conn.response)
+        raise ResponseError(self.conn.response, 'HEAD',
+                            self.conn.make_path(self.path))
 
     @property
     def path(self):
@@ -544,7 +557,8 @@ class File(Base):
         if self.conn.make_request('DELETE', self.path, hdrs=hdrs,
                                   parms=parms) != 204:
 
-            raise ResponseError(self.conn.response)
+            raise ResponseError(self.conn.response, 'DELETE',
+                                self.conn.make_path(self.path))
 
         return True
 
@@ -552,7 +566,8 @@ class File(Base):
         if self.conn.make_request('HEAD', self.path, hdrs=hdrs,
                                   parms=parms, cfg=cfg) != 200:
 
-            raise ResponseError(self.conn.response)
+            raise ResponseError(self.conn.response, 'HEAD',
+                                self.conn.make_path(self.path))
 
         fields = [['content_length', 'content-length'],
                   ['content_type', 'content-type'],
@@ -572,7 +587,8 @@ class File(Base):
         if status == 404:
             return False
         elif (status < 200) or (status > 299):
-            raise ResponseError(self.conn.response)
+            raise ResponseError(self.conn.response, 'HEAD',
+                                self.conn.make_path(self.path))
 
         for hdr in self.conn.response.getheaders():
             if hdr[0].lower() == 'content-type':
@@ -607,7 +623,7 @@ class File(Base):
         return data
 
     def read(self, size=-1, offset=0, hdrs=None, buffer=None,
-             callback=None, cfg={}):
+             callback=None, cfg={}, parms={}):
 
         if size > 0:
             range_string = 'bytes=%d-%d' % (offset, (offset + size) - 1)
@@ -617,10 +633,11 @@ class File(Base):
                 hdrs = {'Range': range_string}
 
         status = self.conn.make_request('GET', self.path, hdrs=hdrs,
-                                        cfg=cfg)
+                                        cfg=cfg, parms=parms)
 
-        if(status < 200) or (status > 299):
-            raise ResponseError(self.conn.response)
+        if (status < 200) or (status > 299):
+            raise ResponseError(self.conn.response, 'GET',
+                                self.conn.make_path(self.path))
 
         for hdr in self.conn.response.getheaders():
             if hdr[0].lower() == 'content-type':
@@ -643,8 +660,9 @@ class File(Base):
     def read_md5(self):
         status = self.conn.make_request('GET', self.path)
 
-        if(status < 200) or (status > 299):
-            raise ResponseError(self.conn.response)
+        if (status < 200) or (status > 299):
+            raise ResponseError(self.conn.response, 'GET',
+                                self.conn.make_path(self.path))
 
         checksum = hashlib.md5()
 
@@ -677,7 +695,8 @@ class File(Base):
             self.conn.make_request('POST', self.path, hdrs=headers, cfg=cfg)
 
             if self.conn.response.status not in (201, 202):
-                raise ResponseError(self.conn.response)
+                raise ResponseError(self.conn.response, 'POST',
+                                    self.conn.make_path(self.path))
 
         return True
 
@@ -735,8 +754,13 @@ class File(Base):
 
         if (self.conn.response.status < 200) or \
            (self.conn.response.status > 299):
-            raise ResponseError(self.conn.response)
+            raise ResponseError(self.conn.response, 'PUT',
+                                self.conn.make_path(self.path))
 
+        try:
+            data.seek(0)
+        except IOError:
+            pass
         self.md5 = self.compute_md5sum(data)
 
         return True
@@ -744,6 +768,7 @@ class File(Base):
     def write_random(self, size=None, hdrs={}, parms={}, cfg={}):
         data = self.random_data(size)
         if not self.write(data, hdrs=hdrs, parms=parms, cfg=cfg):
-            raise ResponseError(self.conn.response)
+            raise ResponseError(self.conn.response, 'PUT',
+                                self.conn.make_path(self.path))
         self.md5 = self.compute_md5sum(StringIO.StringIO(data))
         return data
diff --git a/test/functional/tests.py b/test/functional/tests.py
index e5ccda2..808208d 100644
--- a/test/functional/tests.py
+++ b/test/functional/tests.py
@@ -18,6 +18,8 @@
 
 from datetime import datetime
 import os
+import hashlib
+import json
 import locale
 import random
 import StringIO
@@ -916,7 +918,7 @@ class TestFile(Base):
     set_up = False
 
     def testCopy(self):
-        # makes sure to test encoded characters"
+        # makes sure to test encoded characters
         source_filename = 'dealde%2Fl04 011e%204c8df/flash.png'
         file_item = self.env.container.file(source_filename)
 
@@ -1570,6 +1572,116 @@ class TestFileUTF8(Base2, TestFile):
     set_up = False
 
 
+class TestDloEnv(object):
+    @classmethod
+    def setUp(cls):
+        cls.conn = Connection(config)
+        cls.conn.authenticate()
+        cls.account = Account(cls.conn, config.get('account',
+                                                   config['username']))
+        cls.account.delete_containers()
+
+        cls.container = cls.account.container(Utils.create_name())
+
+        if not cls.container.create():
+            raise ResponseError(cls.conn.response)
+
+        # avoid getting a prefix that stops halfway through an encoded
+        # character
+        prefix = Utils.create_name().decode("utf-8")[:10].encode("utf-8")
+        cls.segment_prefix = prefix
+
+        for letter in ('a', 'b', 'c', 'd', 'e'):
+            file_item = cls.container.file("%s/seg_lower%s" % (prefix, letter))
+            file_item.write(letter * 10)
+
+            file_item = cls.container.file("%s/seg_upper%s" % (prefix, letter))
+            file_item.write(letter.upper() * 10)
+
+        man1 = cls.container.file("man1")
+        man1.write('man1-contents',
+                   hdrs={"X-Object-Manifest": "%s/%s/seg_lower" %
+                         (cls.container.name, prefix)})
+
+        man1 = cls.container.file("man2")
+        man1.write('man2-contents',
+                   hdrs={"X-Object-Manifest": "%s/%s/seg_upper" %
+                         (cls.container.name, prefix)})
+
+        manall = cls.container.file("manall")
+        manall.write('manall-contents',
+                     hdrs={"X-Object-Manifest": "%s/%s/seg" %
+                           (cls.container.name, prefix)})
+
+
+class TestDlo(Base):
+    env = TestDloEnv
+    set_up = False
+
+    def test_get_manifest(self):
+        file_item = self.env.container.file('man1')
+        file_contents = file_item.read()
+        self.assertEqual(
+            file_contents,
+            "aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeee")
+
+        file_item = self.env.container.file('man2')
+        file_contents = file_item.read()
+        self.assertEqual(
+            file_contents,
+            "AAAAAAAAAABBBBBBBBBBCCCCCCCCCCDDDDDDDDDDEEEEEEEEEE")
+
+        file_item = self.env.container.file('manall')
+        file_contents = file_item.read()
+        self.assertEqual(
+            file_contents,
+            ("aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeee" +
+             "AAAAAAAAAABBBBBBBBBBCCCCCCCCCCDDDDDDDDDDEEEEEEEEEE"))
+
+    def test_get_manifest_document_itself(self):
+        file_item = self.env.container.file('man1')
+        file_contents = file_item.read(parms={'multipart-manifest': 'get'})
+        self.assertEqual(file_contents, "man1-contents")
+
+    def test_get_range(self):
+        file_item = self.env.container.file('man1')
+        file_contents = file_item.read(size=25, offset=8)
+        self.assertEqual(file_contents, "aabbbbbbbbbbccccccccccddd")
+
+        file_contents = file_item.read(size=1, offset=47)
+        self.assertEqual(file_contents, "e")
+
+    def test_get_range_out_of_range(self):
+        file_item = self.env.container.file('man1')
+
+        self.assertRaises(ResponseError, file_item.read, size=7, offset=50)
+        self.assert_status(416)
+
+    def test_copy(self):
+        # Adding a new segment, copying the manifest, and then deleting the
+        # segment proves that the new object is really the concatenated
+        # segments and not just a manifest.
+        f_segment = self.env.container.file("%s/seg_lowerf" %
+                                            (self.env.segment_prefix))
+        f_segment.write('ffffffffff')
+        try:
+            man1_item = self.env.container.file('man1')
+            man1_item.copy(self.env.container.name, "copied-man1")
+        finally:
+            # try not to leave this around for other tests to stumble over
+            f_segment.delete()
+
+        file_item = self.env.container.file('copied-man1')
+        file_contents = file_item.read()
+        self.assertEqual(
+            file_contents,
+            "aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffff")
+
+
+class TestDloUTF8(Base2, TestDlo):
+    set_up = False
+
+
 class TestFileComparisonEnv:
     @classmethod
     def setUp(cls):
@@ -1656,5 +1768,228 @@ class TestFileComparison(Base):
 class TestFileComparisonUTF8(Base2, TestFileComparison):
     set_up = False
 
+
+class TestSloEnv(object):
+    slo_enabled = None  # tri-state: None initially, then True/False
+
+    @classmethod
+    def setUp(cls):
+        cls.conn = Connection(config)
+        cls.conn.authenticate()
+        cls.account = Account(cls.conn, config.get('account',
+                                                   config['username']))
+        cls.account.delete_containers()
+
+        cls.container = cls.account.container(Utils.create_name())
+
+        if not cls.container.create():
+            raise ResponseError(cls.conn.response)
+
+        # TODO(seriously, anyone can do this): make this use the /info API once
+        # it lands, both for detection of SLO and for minimum segment size
+        if cls.slo_enabled is None:
+            test_file = cls.container.file(".test-slo")
+            try:
+                # If SLO is enabled, this'll raise an error since
+                # X-Static-Large-Object is a reserved header.
+                #
+                # If SLO is not enabled, then this will get the usual 2xx
+                # response.
+                test_file.write(
+                    "some contents",
+                    hdrs={'X-Static-Large-Object': 'true'})
+            except ResponseError as err:
+                if err.status == 400:
+                    cls.slo_enabled = True
+                else:
+                    raise
+            else:
+                cls.slo_enabled = False
+                return
+
+        seg_info = {}
+        for letter, size in (('a', 1024 * 1024),
+                             ('b', 1024 * 1024),
+                             ('c', 1024 * 1024),
+                             ('d', 1024 * 1024),
+                             ('e', 1)):
+            seg_name = "seg_%s" % letter
+            file_item = cls.container.file(seg_name)
+            file_item.write(letter * size)
+            seg_info[seg_name] = {
+                'size_bytes': size,
+                'etag': file_item.md5,
+                'path': '/%s/%s' % (cls.container.name, seg_name)}
+
+        file_item = cls.container.file("manifest-abcde")
+        file_item.write(
+            json.dumps([seg_info['seg_a'], seg_info['seg_b'],
+                        seg_info['seg_c'], seg_info['seg_d'],
+                        seg_info['seg_e']]),
+            parms={'multipart-manifest': 'put'})
+
+        file_item = cls.container.file('manifest-cd')
+        cd_json = json.dumps([seg_info['seg_c'], seg_info['seg_d']])
+        file_item.write(cd_json, parms={'multipart-manifest': 'put'})
+        cd_etag = hashlib.md5(seg_info['seg_c']['etag'] +
+                              seg_info['seg_d']['etag']).hexdigest()
+
+        file_item = cls.container.file("manifest-bcd-submanifest")
+        file_item.write(
+            json.dumps([seg_info['seg_b'],
+                        {'etag': cd_etag,
+                         'size_bytes': (seg_info['seg_c']['size_bytes'] +
+                                        seg_info['seg_d']['size_bytes']),
+                         'path': '/%s/%s' % (cls.container.name,
+                                             'manifest-cd')}]),
+            parms={'multipart-manifest': 'put'})
+        bcd_submanifest_etag = hashlib.md5(
+            seg_info['seg_b']['etag'] + cd_etag).hexdigest()
+
+        file_item = cls.container.file("manifest-abcde-submanifest")
+        file_item.write(
+            json.dumps([
+                seg_info['seg_a'],
+                {'etag': bcd_submanifest_etag,
+                 'size_bytes': (seg_info['seg_b']['size_bytes'] +
+                                seg_info['seg_c']['size_bytes'] +
+                                seg_info['seg_d']['size_bytes']),
+                 'path': '/%s/%s' % (cls.container.name,
+                                     'manifest-bcd-submanifest')},
+                seg_info['seg_e']]),
+            parms={'multipart-manifest': 'put'})
+
+
+class TestSlo(Base):
+    env = TestSloEnv
+    set_up = False
+
+    def setUp(self):
+        super(TestSlo, self).setUp()
+        if self.env.slo_enabled is False:
+            raise SkipTest("SLO not enabled")
+        elif self.env.slo_enabled is not True:
+            # just some sanity checking
+            raise Exception(
+                "Expected slo_enabled to be True/False, got %r" %
+                (self.env.slo_enabled,))
+
+    def test_slo_get_simple_manifest(self):
+        file_item = self.env.container.file('manifest-abcde')
+        file_contents = file_item.read()
+        self.assertEqual(4 * 1024 * 1024 + 1, len(file_contents))
+        self.assertEqual('a', file_contents[0])
+        self.assertEqual('a', file_contents[1024 * 1024 - 1])
+        self.assertEqual('b', file_contents[1024 * 1024])
+        self.assertEqual('d', file_contents[-2])
+        self.assertEqual('e', file_contents[-1])
+
+    def test_slo_get_nested_manifest(self):
+        file_item = self.env.container.file('manifest-abcde-submanifest')
+        file_contents = file_item.read()
+        self.assertEqual(4 * 1024 * 1024 + 1, len(file_contents))
+        self.assertEqual('a', file_contents[0])
+        self.assertEqual('a', file_contents[1024 * 1024 - 1])
+        self.assertEqual('b', file_contents[1024 * 1024])
+        self.assertEqual('d', file_contents[-2])
+        self.assertEqual('e', file_contents[-1])
+
+    def test_slo_ranged_get(self):
+        file_item = self.env.container.file('manifest-abcde')
+        file_contents = file_item.read(size=1024 * 1024 + 2,
+                                       offset=1024 * 1024 - 1)
+        self.assertEqual('a', file_contents[0])
+        self.assertEqual('b', file_contents[1])
+        self.assertEqual('b', file_contents[-2])
+        self.assertEqual('c', file_contents[-1])
+
+    def test_slo_ranged_submanifest(self):
+        file_item = self.env.container.file('manifest-abcde-submanifest')
+        file_contents = file_item.read(size=1024 * 1024 + 2,
+                                       offset=1024 * 1024 * 2 - 1)
+        self.assertEqual('b', file_contents[0])
+        self.assertEqual('c', file_contents[1])
+        self.assertEqual('c', file_contents[-2])
+        self.assertEqual('d', file_contents[-1])
+
+    def test_slo_etag_is_hash_of_etags(self):
+        expected_hash = hashlib.md5()
+        expected_hash.update(hashlib.md5('a' * 1024 * 1024).hexdigest())
+        expected_hash.update(hashlib.md5('b' * 1024 * 1024).hexdigest())
+        expected_hash.update(hashlib.md5('c' * 1024 * 1024).hexdigest())
+        expected_hash.update(hashlib.md5('d' * 1024 * 1024).hexdigest())
+        expected_hash.update(hashlib.md5('e').hexdigest())
+        expected_etag = expected_hash.hexdigest()
+
+        file_item = self.env.container.file('manifest-abcde')
+        self.assertEqual(expected_etag, file_item.info()['etag'])
+
+    def test_slo_etag_is_hash_of_etags_submanifests(self):
+
+        def hd(x):
+            return hashlib.md5(x).hexdigest()
+
+        expected_etag = hd(hd('a' * 1024 * 1024) +
+                           hd(hd('b' * 1024 * 1024) +
+                              hd(hd('c' * 1024 * 1024) +
+                                 hd('d' * 1024 * 1024))) +
+                           hd('e'))
+
+        file_item = self.env.container.file('manifest-abcde-submanifest')
+        self.assertEqual(expected_etag, file_item.info()['etag'])
+
+    def test_slo_etag_mismatch(self):
+        file_item = self.env.container.file("manifest-a-bad-etag")
+        try:
+            file_item.write(
+                json.dumps([{
+                    'size_bytes': 1024 * 1024,
+                    'etag': 'not it',
+                    'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]),
+                parms={'multipart-manifest': 'put'})
+        except ResponseError as err:
+            self.assertEqual(400, err.status)
+        else:
+            self.fail("Expected ResponseError but didn't get it")
+
+    def test_slo_size_mismatch(self):
+        file_item = self.env.container.file("manifest-a-bad-size")
+        try:
+            file_item.write(
+                json.dumps([{
+                    'size_bytes': 1024 * 1024 - 1,
+                    'etag': hashlib.md5('a' * 1024 * 1024).hexdigest(),
+                    'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]),
+                parms={'multipart-manifest': 'put'})
+        except ResponseError as err:
+            self.assertEqual(400, err.status)
+        else:
+            self.fail("Expected ResponseError but didn't get it")
+
+    def test_slo_copy(self):
+        file_item = self.env.container.file("manifest-abcde")
+        file_item.copy(self.env.container.name, "copied-abcde")
+
+        copied = self.env.container.file("copied-abcde")
+        copied_contents = copied.read(parms={'multipart-manifest': 'get'})
+        self.assertEqual(4 * 1024 * 1024 + 1, len(copied_contents))
+
+    def test_slo_copy_the_manifest(self):
+        file_item = self.env.container.file("manifest-abcde")
+        file_item.copy(self.env.container.name, "copied-abcde",
+                       parms={'multipart-manifest': 'get'})
+
+        copied = self.env.container.file("copied-abcde")
+        copied_contents = copied.read(parms={'multipart-manifest': 'get'})
+        try:
+            json.loads(copied_contents)
+        except ValueError:
+            self.fail("COPY didn't copy the manifest (invalid json on GET)")
+
+
+class TestSloUTF8(Base2, TestSlo):
+    set_up = False
+
+
 if __name__ == '__main__':
     unittest.main()